diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a1d194..16d71a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,17 +17,26 @@ jobs: - { name: "Ubuntu GCC 13", os: ubuntu-latest, - cxx: g++-13 + cxx: g++-13, + preset: gcc } - { name: "Ubuntu Clang 18", os: ubuntu-latest, - cxx: clang++-18 + cxx: clang++-18, + preset: clang + } + - { + name: "Ubuntu Clang 18 (ASan+UBSan)", + os: ubuntu-latest, + cxx: clang++-18, + preset: asan-ubsan } - { name: "Windows MSVC", os: windows-latest, - cxx: cl + cxx: cl, + preset: msvc } steps: @@ -48,18 +57,48 @@ jobs: - name: Configure CMake run: | - if [ "${{ matrix.config.os }}" = "windows-latest" ]; then - cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON + if [ "${{ runner.os }}" = "Windows" ]; then + cmake --preset ${{ matrix.config.preset }} else - cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} + cmake --preset ${{ matrix.config.preset }} -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} fi shell: bash - name: Build - run: cmake --build build --config Release + run: cmake --build --preset ${{ matrix.config.preset }} + shell: bash - name: Run Tests run: | - cd build - ctest -C Release --output-on-failure + # Treat any sanitizer finding as a hard failure. + export ASAN_OPTIONS=halt_on_error=1:abort_on_error=1 + export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 + ctest --preset ${{ matrix.config.preset }} shell: bash + + amalgamation: + name: Amalgamated single-header compiles + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install GCC 13 + run: | + sudo apt-get update + sudo apt-get install -y g++-13 + + - name: Generate and compile the amalgamated headers + shell: bash + run: | + set -euo pipefail + + # Core only, and the full bundle with every opt-in add-on inlined. + python3 scripts/amalgamate.py --output dist/soatable_core.hpp + python3 scripts/amalgamate.py --with all --output dist/soatable_full.hpp + + # Each generated header must compile as a standalone translation unit. + for hdr in dist/soatable_core.hpp dist/soatable_full.hpp; do + printf '#include "%s"\nint main() { return 0; }\n' "$hdr" > tu.cpp + g++-13 -std=c++23 -Wall -Wextra -c tu.cpp -o tu.o + echo "OK: ${hdr} compiles as a single header" + done diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..88c1a5a --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,45 @@ +name: Docs + +on: + push: + branches: [ main ] + +# Allow one concurrent deployment. +concurrency: + group: pages + cancel-in-progress: true + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + + - name: Install Doxygen + run: sudo apt-get update && sudo apt-get install -y doxygen + + - name: Generate API documentation + run: | + cmake -S . -B build-docs \ + -DSOATABLE_BUILD_DOCUMENTATION=ON \ + -DSOATABLE_BUILD_TESTS=OFF \ + -DSOATABLE_BUILD_EXAMPLES=OFF \ + -DSOATABLE_BUILD_BENCHMARKS=OFF + cmake --build build-docs --target docs + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: build-docs/doxygen/html + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 3a6474d..a37e6f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,16 @@ .venv/ .env build/ +build-docs/ install/ .cache/ CMakeUserPresets.json *-workspace /.vscode/*.json +prompts.txt +# Generated artifacts +dist/ +bench/ + +__pycache__ +*.pyc diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ca8a03f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,89 @@ +# Modern C++ Best Practices + +## Language & Compiler + +- Target C++17 minimum; prefer C++20/23. Set `CMAKE_CXX_STANDARD` explicitly; disable extensions (`-pedantic`). +- Enable `-Wall -Wextra -Wconversion -Wsign-conversion` and treat warnings as errors (`-Werror`). + +## Style & Naming + +- `m_` prefix for private members; `snake_case` for variables/functions; `PascalCase` for types; `SCREAMING_SNAKE_CASE` for macros and Constants. +- All symbols (variable, parameters, functions) must be at least 3 characters long. +- Alwayse Use `std::print`/`std::println`/`std::format` for output. +- No magic literals, replace with named `constexpr` values. Declare variables close to first use. +- Keep functions short and flat; remove unused variables, parameters, and includes. +- Use `#pragma once`; keep headers to declarations and inline/template definitions only. +- Do not use mdash (`--`) in code, comments, or documentation. + +## Variables & Types + +- Always initialize variables at declaration; prefer brace initialization `T x{value}` to prevent narrowing. +- Use `auto` when type is obvious; `const` by default; `constexpr` for compile-time constants. +- Avoid `using namespace std` in headers. Use `enum class`; use `std::string_view` for read-only strings. +- Never mix signed/unsigned arithmetic; never use C-style casts, use `static_cast`, `dynamic_cast`, etc. +- Never use raw pointers for ownership; use references or smart pointers instead. Avoid `void*` and `reinterpret_cast`. +- Never use `goto` or `longjmp`; prefer structured control flow and RAII for cleanup. +- Never use C-style arrays or C-style strings; prefer `std::array`, `std::vector`, and `std::string` for safety and convenience. + +## Memory Management + +- Follow RAII: tie every resource (memory, file, mutex) to an object's lifetime. +- Use `std::unique_ptr`/`std::shared_ptr`/`std::weak_ptr` via `make_unique`/`make_shared`, never `new`/`delete` directly. +- Follow Rule of Zero (all RAII members) or Rule of Five (when owning a raw resource). +- Detect leaks with AddressSanitizer; use standard containers over raw arrays. + +## Functions & Classes + +- One logical operation per function; always use trailing return types. +- Pass non-trivial types as `const T&`; return by value (rely on NRVO/RVO). +- Mark `[[nodiscard]]` where return values must not be ignored; mark `noexcept` when applicable. +- Single-argument constructors must be `explicit`; keep members `private`; use `override`/`final` on overrides. +- Prefer composition over inheritance; declare base destructors `virtual` (or `protected` non-virtual). +- Do not call virtual functions from constructors or destructors. + +## Error Handling + +- Use exceptions for errors (not for normal control flow); catch by `const` reference to avoid slicing. +- Maintain a consistent exception-safety guarantee: basic, strong, or no-throw. +- Use `static_assert` for compile-time invariants; `assert()` for debug-only preconditions, never on user input. +- Consider `std::expected` (C++23) for predictable, performance-sensitive failure paths. + +## Templates & Concurrency + +- Constrain templates with C++20 concepts; use `if constexpr` over SFINAE/`enable_if`. +- Use variadic templates and fold expressions instead of recursive specializations. +- Protect shared mutable state with `std::mutex`; prefer `std::jthread` over `std::thread`. +- Use `std::atomic` for lock-free single-variable operations; never `volatile` for inter-thread communication. +- Minimize lock duration; avoid callbacks or virtual calls while holding a lock; prefer `std::async` and parallel algorithms. + +## Performance & Build + +- Profile before optimizing; prefer `std::vector` for cache-friendly access; use `reserve()` when size is known. +- Use move semantics to avoid deep copies; prefer standard algorithms (`std::sort`, `std::transform`) over hand-written loops. +- Use CMake (≥ 3.25) with target-centric paradigm: `target_include_directories`, `target_compile_options`, `target_link_libraries`. +- Use `FetchContent` or vcpkg/Conan for dependencies; use out-of-source builds and CMake Presets. +- Enable `CMAKE_EXPORT_COMPILE_COMMANDS=ON` for tooling; use `ccache`/`sccache` to speed up rebuilds. + +## Testing & Tooling + +- Use GoogleTest/Catch2 with CTest; write tests before or alongside implementation. +- Run tests under AddressSanitizer and UndefinedBehaviourSanitizer; use fuzz testing for parsers and untrusted input. +- Enforce clang-format in CI; use clang-tidy with `cppcoreguidelines-*`, `modernize-*`, `readability-*` checks. +- Use debuggers (GDB, LLDB) actively rather than `std::cout`; use `compile_commands.json` for accurate tooling. +- Use real implementations; only mock external dependencies (LLM APIs, cloud services). +- Test public interface behavior, not implementation details, keeps tests resilient to refactoring. +- Tests must be fast, isolated, and have descriptive names; cover edge cases and error conditions. +- Location: `tests/unittests/` following source structure. + +## CI/CD & Maintenance + +## Versioning & Breaking Changes + +- Follow Semantic Versioning 2.0.0 (`MAJOR.MINOR.PATCH`). +- A breaking change is any backward-incompatible modification to the public API, including data schemas, CLI, and server communication formats. +- CI must build and test before merging; build across GCC, Clang, and MSVC. +- Document the _why_, not the _what_; use Doxygen (`///`) for API docs and `//` for implementation notes. +- Keep public API surface minimal; mark deprecated APIs with `[[deprecated("reason")]]` and include a migration path. +- Follow the Boy Scout Rule; track technical debt explicitly; maintain a `CHANGELOG.md` and `CONTRIBUTING.md`. + +--- diff --git a/CMakeLists.txt b/CMakeLists.txt index 914c7d8..ddea3b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,18 +1,32 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) -project(soatable VERSION 0.1.0 LANGUAGES CXX) +file(READ "VERSION.txt" RAW_VERSION) +string(STRIP "${RAW_VERSION}" APP_VERSION) +project(SOATABLE VERSION ${APP_VERSION} LANGUAGES CXX) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(FETCHCONTENT_BASE_DIR "${CMAKE_SOURCE_DIR}/.cache/_deps") +include(${CMAKE_SOURCE_DIR}/cmake/Helpers.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/Versions.cmake) -include(GNUInstallDirs) -include(CMakePackageConfigHelpers) +if(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS) + add_coverage_analysis(${CMAKE_SOURCE_DIR}) +endif() -option(SOATABLE_BUILD_EXAMPLES "Build the SoaTable example program" ON) -option(SOATABLE_BUILD_BENCHMARKS "Build the SoaTable benchmark program" ON) -option(SOATABLE_BUILD_TESTS "Build the SoaTable tests" ON) +if(${PROJECT_NAME}_APPLY_FORMATING) + apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING ${CMAKE_SOURCE_DIR}/include) + apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING ${CMAKE_SOURCE_DIR}/example) + apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING ${CMAKE_SOURCE_DIR}/benchmarks) + apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING ${CMAKE_SOURCE_DIR}/tests) +endif() +if(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY) + apply_clang_tidy_globaly() +endif() + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) add_library(soatable INTERFACE) add_library(soatable::soatable ALIAS soatable) @@ -23,8 +37,19 @@ target_include_directories(soatable INTERFACE $ ) +# Propagate the MSVC debugger visualizer to consumers. +target_sources(soatable INTERFACE + $ + $ +) + install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +# Ship the debugger visualizers (MSVC .natvis and GDB pretty-printers) alongside the package. +install(DIRECTORY tools/visualizers/ DESTINATION ${CMAKE_INSTALL_DATADIR}/soatable/visualizers + PATTERN "__pycache__" EXCLUDE +) + install(TARGETS soatable EXPORT soatableTargets ) @@ -53,15 +78,19 @@ install(FILES DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/soatable ) -if(SOATABLE_BUILD_EXAMPLES AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/example) +if(SOATABLE_BUILD_EXAMPLES AND EXISTS ${CMAKE_SOURCE_DIR}/example) add_subdirectory(example) endif() -if(SOATABLE_BUILD_BENCHMARKS AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks) +if(SOATABLE_BUILD_BENCHMARKS AND EXISTS ${CMAKE_SOURCE_DIR}/benchmarks) add_subdirectory(benchmarks) endif() -if(SOATABLE_BUILD_TESTS AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests) +if(SOATABLE_BUILD_TESTS AND EXISTS ${CMAKE_SOURCE_DIR}/tests) enable_testing() add_subdirectory(tests) endif() + +if(SOATABLE_BUILD_DOCUMENTATION) + build_documenation() +endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..16f47b6 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,136 @@ +{ + "version": 6, + "cmakeMinimumRequired": { + "major": 3, + "minor": 25, + "patch": 0 + }, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "SOATABLE_CXX_STANDARD": "23", + "SOATABLE_BUILD_TESTS": "ON", + "SOATABLE_BUILD_EXAMPLES": "ON", + "SOATABLE_APPLY_CLANG_TIDY_GLOBALY": "ON", + "SOATABLE_APPLY_FORMATING": "ON" + } + }, + { + "name": "debug", + "inherits": "base", + "displayName": "Debug", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } + }, + { + "name": "release", + "inherits": "base", + "displayName": "Release", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } + }, + { + "name": "gcc", + "inherits": "release", + "displayName": "GCC Release", + "cacheVariables": { "CMAKE_CXX_COMPILER": "g++" } + }, + { + "name": "clang", + "inherits": "release", + "displayName": "Clang Release", + "cacheVariables": { "CMAKE_CXX_COMPILER": "clang++" } + }, + { + "name": "msvc", + "inherits": "base", + "displayName": "MSVC (multi-config)", + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "asan-ubsan", + "inherits": "debug", + "displayName": "Clang Address + UB Sanitizers", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "clang++", + "CMAKE_CXX_FLAGS": "-fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" + } + }, + { + "name": "bench", + "inherits": "release", + "displayName": "Benchmarks", + "cacheVariables": { + "SOATABLE_BUILD_BENCHMARKS": "ON", + "SOATABLE_BUILD_TESTS": "OFF", + "SOATABLE_BUILD_EXAMPLES": "OFF" + } + } + ], + "buildPresets": [ + { "name": "debug", "configurePreset": "debug", "configuration": "Debug" }, + { + "name": "release", + "configurePreset": "release", + "configuration": "Release" + }, + { "name": "gcc", "configurePreset": "gcc", "configuration": "Release" }, + { "name": "clang", "configurePreset": "clang", "configuration": "Release" }, + { "name": "msvc", "configurePreset": "msvc", "configuration": "Release" }, + { + "name": "asan-ubsan", + "configurePreset": "asan-ubsan", + "configuration": "Debug" + }, + { "name": "bench", "configurePreset": "bench", "configuration": "Release" } + ], + "testPresets": [ + { + "name": "common", + "hidden": true, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error", "stopOnFailure": false } + }, + { + "name": "debug", + "inherits": "common", + "configurePreset": "debug", + "configuration": "Debug" + }, + { + "name": "release", + "inherits": "common", + "configurePreset": "release", + "configuration": "Release" + }, + { + "name": "gcc", + "inherits": "common", + "configurePreset": "gcc", + "configuration": "Release" + }, + { + "name": "clang", + "inherits": "common", + "configurePreset": "clang", + "configuration": "Release" + }, + { + "name": "msvc", + "inherits": "common", + "configurePreset": "msvc", + "configuration": "Release" + }, + { + "name": "asan-ubsan", + "inherits": "common", + "configurePreset": "asan-ubsan", + "configuration": "Debug" + } + ] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f44cce2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,116 @@ +# Contributing to SoaTable + +## Why this library exists + +Most data structures are designed for the programmer. SoaTable is designed for the machine. It +takes data that you _think about_ one row at a time and lays it out the way the hardware actually +_reads_ it: each column its own dense, aligned array, touched only when a query needs it. The result +is a row-oriented API sitting on top of memory the cache loves. + +That single idea has a cost discipline behind it: **you pay only for what you use**. The core is one +header with no dependencies. Everything else, the compute layer, queries, mmap storage, +concurrency, units, lives in a header you include _only if you reach for it_. Nothing you do not use +should cost you a cycle, a byte, or a second of compile time. + +This is not an abstract preference. The people reading these columns are writing game-engine entity +worlds, trading systems that cannot pause for a garbage collector, and flight software that cannot +throw. For them, "fast" and "correct" are not features to be added later; they are the reason the +library exists at all. If you want the shape of the problem and where SoaTable wins, the README's +"idea in 30 seconds" and "when SoaTable is the right tool" say it well. This document is about +something else: how to think while you work on it. + +## Why contribute + +Work like this is rare. There is no framework to hide behind, no allocator handed to you, no +dependency to blame. It is header-only C++23, zero core dependencies, and a public surface small +enough that one person can hold the whole thing in their head. That smallness is the gift: you can +understand a change _completely_ before you make it, and you can see the consequences immediately. + +The feedback loops here are honest. Sanitizers tell you when you are wrong about memory. +Compile-fail tests tell you when an interface promises something it should reject. Benchmarks tell +you whether your clever idea was actually clever. Few codebases give you instruments this sharp, and +working against them is how you get genuinely good at systems programming. + +You do not need to land a new subsystem to matter here. A failing edge case turned into a test, a +benchmark that exposes a regression, a header that compiles a little cleaner, a comment that finally +explains the _why_, these are real contributions. The bar is high, but the unit of progress is +small. + +## The principles that guide a good contribution + +These are not rules to pass a checklist; they are the judgment the codebase is built on. A good +change is one you can defend in these terms. + +- **The core stays small; everything optional is opt-in.** New capability earns its own header that + a user includes only when they want it, the way `compute`, `query`, `mmap`, and the rest already + do. Adding to `soatable.hpp` is a decision to make _everyone_ carry your feature. Make that case + deliberately, or keep it out of the core. + +- **You pay only for what you use.** A feature nobody calls should cost nothing, not in runtime, not + in binary size, not in compile time. Templates that instantiate lazily, abstractions that vanish + under optimization: zero-overhead is the contract, not the aspiration. + +- **Predictability over cleverness.** The no-exceptions build, the `std::expected` hot paths, the + deterministic allocators, none of these exist for elegance. They exist because this code runs + where a surprise is a crash, a missed trade, or a failed mission. When a clever path and a + predictable path disagree, the predictable one wins. + +- **Mechanical sympathy, not guesswork.** Understand the layout and the cache before you optimize, + then measure. The smallest-driver join heuristic is the house example: it is not faster because it + feels faster, it is faster because driving iteration off the smallest column scans far fewer rows, + and the benchmark suite proves it. Profile, then change. + +- **Correctness is demonstrated, not asserted.** A feature without tests is not finished, it is a + hypothesis. Trust in this library is built out of unit tests, sanitizer-clean runs, and + compile-fail contracts that prove the interface rejects what it should. If you cannot show it is + correct, it is not. + +- **Leave the public API as quiet as you found it.** The names are std-style and lowercase, the + surface is intentionally minimal, and breaking changes follow SemVer with a deprecation alias and + a migration note. The best API change is the one a user never has to notice. + +## How to proceed + +1. **Understand before you add.** Read the code you intend to touch and find the invariant it + protects, the generational handle that defeats the ABA problem, the independence of columns, the + per-row signature bitset that makes selective joins cheap. Most mistakes here are not typos; they + are a change that quietly violates one of these. Know which one your change lives next to. + +2. **Discuss direction early.** For anything beyond a fix, open an issue first and describe the + problem and the shape of your approach. Design is cheaper in prose than in a rejected PR, and a + short conversation often reveals the opt-in header your feature really wants to live in. + +3. **Build and prove it locally.** The presets mirror CI, so a green local run should mean a green + pipeline. At minimum, build and test on one compiler and run the sanitizer leg: + + ```bash + cmake --preset clang # or: gcc, msvc + cmake --build --preset clang + ctest --preset clang + + cmake --preset asan-ubsan # Address + UndefinedBehavior sanitizers + cmake --build --preset asan-ubsan + ctest --preset asan-ubsan + ``` + + Keep the public header warning-clean (the `soatable_warning_check` target enforces + `-Werror`/`/WX`) and run `cmake --build build --target format` before you push. The full compiler + matrix and docs build live in the README's "Building, testing, docs". + +4. **Make the change provable.** Add tests alongside it under `tests/unittests/`, one file per + surface area, real implementations, deterministic inputs, never random seeds. Contracts that must + _fail_ to compile belong in `tests/compile_fail/`. The sanitizer preset must stay clean. + +5. **Keep the PR focused and honest.** One idea per pull request, branched from `main`, with a + description of the motivation and the approach, not just the diff. If you touch the public API, + follow SemVer and ship a `[[deprecated]]` alias with a migration note rather than a silent break. + New public API gets Doxygen `///` comments. The full coding standard, naming, initialization, + casting, the lot, lives in [CLAUDE](CLAUDE.md); write code that already reads like the code + around it. + +## A note on the bar + +The standard here is high on purpose, and review can be exacting. None of it is about you; it is +about the code, which strangers will trust in places where mistakes are expensive. A small, +well-reasoned change that respects the principles above is always more welcome than a large one that +does not. Bring the small, sharp thing. We are glad you are here. diff --git a/README.md b/README.md index 8904d53..3f8a803 100644 --- a/README.md +++ b/README.md @@ -2,201 +2,671 @@ [![Build](https://github.com/bbalouki/soatable/actions/workflows/ci.yaml/badge.svg)](https://github.com/bbalouki/soatable/actions/workflows/ci.yaml) ![C++](https://img.shields.io/badge/C%2B%2B-23-blue.svg) -![CMake](https://img.shields.io/badge/CMake-3.20%2B-green.svg) +![CMake](https://img.shields.io/badge/CMake-3.25%2B-green.svg) ![License](https://img.shields.io/badge/License-MIT-yellow.svg) ![Header-only](https://img.shields.io/badge/Header--only-Yes-blue) -![Dependencies](https://img.shields.io/badge/Dependencies-None-brightgreen) -![Namespace](https://img.shields.io/badge/Namespace-soatable-lightgrey) -![Performance](https://img.shields.io/badge/Speedup-5x--15x-orange) - -SoaTable is a high-performance, header-only C++23 library providing a sparse column-oriented data structure. It is designed for workloads where data is naturally row-addressed but frequently queried by a subset of fields, offering significant performance gains through cache-friendly memory layout and efficient joins. - -## Table of Contents - -- [Core Concepts](#core-concepts) -- [Why Use SoaTable?](#why-use-soatable) -- [Features](#features) -- [Quick Start](#quick-start) -- [Real-World Example](#real-world-example) +![Dependencies](https://img.shields.io/badge/Core%20dependencies-None-brightgreen) + +**SoaTable is a header-only C++23 library for storing tables of data so that the computer can read +them as fast as physically possible.** + +You still think in terms of rows, "here is one trade", "here +is one spaceship", "here is one game character", but underneath, SoaTable stores the data in the +shape the hardware actually prefers. The result is a familiar, row-shaped way of working that runs +dramatically faster on the kinds of questions real programs ask. + +This document is written to be read start to finish, by anyone. You do not need to open a single +source file to understand what SoaTable does, why it is built the way it is, or how to use every one +of its features. Where code appears, the paragraph above it already explains the idea in plain +words, so you can read the prose and skip the code if you like. + +## Table of contents + +- [Start here: the problem, in plain words](#start-here-the-problem-in-plain-words) +- [The mental model](#the-mental-model) +- [The philosophy](#the-philosophy) +- [Is SoaTable the right tool for you?](#is-soatable-the-right-tool-for-you) +- [Quick start](#quick-start) +- [The guided tour](#the-guided-tour) + - [1. The table: rows, handles, and sparse columns](#1-the-table-rows-handles-and-sparse-columns) + - [2. Reading values safely](#2-reading-values-safely) + - [3. Asking questions: views and selects](#3-asking-questions-views-and-selects) + - [4. A friendlier handle: `row_handle`](#4-a-friendlier-handle-row_handle) + - [5. Sorting and bulk loading](#5-sorting-and-bulk-loading) + - [6. Zero-copy columns and validity bitmaps](#6-zero-copy-columns-and-validity-bitmaps) + - [7. Compact storage helpers](#7-compact-storage-helpers) + - [8. The compute layer: math over columns](#8-the-compute-layer-math-over-columns) + - [9. Queries and group-by](#9-queries-and-group-by) + - [10. Choosing how memory is laid out](#10-choosing-how-memory-is-laid-out) + - [11. Sharing a table across threads](#11-sharing-a-table-across-threads) + - [12. Time-series helpers](#12-time-series-helpers) + - [13. Units that catch mistakes at compile time](#13-units-that-catch-mistakes-at-compile-time) + - [14. Runtime schemas: the dynamic table](#14-runtime-schemas-the-dynamic-table) + - [15. Saving and loading](#15-saving-and-loading) + - [16. Building a table from a struct](#16-building-a-table-from-a-struct) + - [17. Running without exceptions](#17-running-without-exceptions) +- [The opt-in headers at a glance](#the-opt-in-headers-at-a-glance) +- [Domain cookbooks](#domain-cookbooks) - [Installation](#installation) -- [Benchmarks](#benchmarks) +- [Benchmarks: what the speed actually is](#benchmarks-what-the-speed-actually-is) +- [Building, testing, and docs](#building-testing-and-docs) +- [Versioning and naming](#versioning-and-naming) +- [Contributing](#contributing) - [License](#license) -## Core Concepts +## Start here: the problem, in plain words -SoaTable combines three powerful ideas: +Picture a filing cabinet where every drawer holds folders, and each folder is one record, one +person, one trade, one sensor reading. To find the average age of everyone in the cabinet, you have +to pull out every single folder, flip past the name, the address, the phone number, and finally read +the one line you care about. You touched everything to use almost nothing. -- **Stable Generational Row Handles:** Use `row_id` to refer to rows without worrying about the ABA problem or pointer invalidation. -- **Sparse Column Storage:** Only pay for the columns a row actually has. Memory is managed densely per column while appearing sparse per row. -- **Efficient Joins:** The `select<...>()` mechanism automatically uses the smallest requested column as a "driver," minimizing iteration overhead during sparse joins. +Computers have exactly this problem, and it is the single biggest reason programs run slower than +they should. When a program reads even one number from memory, the hardware does not fetch just that +number, it fetches a whole **cache line**, a fixed-size chunk of nearby bytes, on the bet that you +will want the neighbours too. If your records are stored as whole folders sitting next to each other +(the normal way, called **Array of Structures**, or AoS), then reading one field drags all the +other fields of that record along for the ride, wasting most of every fetch. -## Why Use SoaTable? +```cpp +// The normal layout: each record is a folder; folders sit back to back. +struct Particle { double x, y, vx, vy; std::string name; }; +std::vector particles; // x,y,vx,vy,name | x,y,vx,vy,name | ... +``` -Most applications start with an **Array of Structures (AoS)** layout: +SoaTable turns the cabinet on its side. Instead of one drawer per record, it keeps **one drawer per +field**: all the names together in one drawer, all the ages together in another, all the prices in a +third. This is called a **Structure of Arrays**, or SoA. Now "average age" reads a single tight +drawer end to end, and every byte the hardware fetches is one you actually wanted. + +```text + Array of Structures (folders) Structure of Arrays (SoaTable) + ----------------------------- ------------------------------ + [name age city][name age city] ... names: [name][name][name] ... + [name age city][name age city] ... ages: [age ][age ][age ] ... + cities: [city][city][city] ... +``` + +That one change is worth a lot, but SoaTable goes further with two refinements that matter in real +programs: + +- **The drawers are independent and sparse.** Not every record has every field. A free game + character might have a position but no weapon; a trade might have a price but no settlement note. + In SoaTable, a field's drawer only stores the records that actually have that field, so you never + pay memory for the blanks. +- **Every record keeps a stable ticket.** When you add a record you get back a small, permanent + **handle**. You can delete records, add new ones, and re-sort the whole table, and your handle + still points at the right record, or correctly tells you it is gone. No dangling indices, no + surprises. + +## The mental model + +If you remember four ideas, you understand SoaTable: + +1. **A column is a field's drawer.** You declare your columns up front as small types + (`struct Price { double value; };`). Each becomes its own dense, tightly packed array. +2. **A row is a record, reached by a handle.** `insert()` creates an empty record and hands you a + `row_id`. You then `assign` whatever fields that record happens to have. +3. **Columns are optional per row.** A record carries only the fields you gave it. Asking for a + field a record does not have is a well-defined "not present", never a crash or garbage. +4. **Questions start from the smallest drawer.** When you ask for "every record that has both a + price and a quantity", SoaTable looks at the smaller of the two drawers and checks each of those + against the other. Starting small means it examines far fewer records, which is where most of the + speed comes from. + +## The philosophy + +SoaTable is opinionated, and the opinions are what make it fast and trustworthy. They are worth +stating plainly, because they explain every design choice you will meet below. + +- **You pay only for what you use.** The core is a single header with no dependencies. Everything + beyond the table itself, math, queries, on-disk storage, threading, units, lives in a separate + header you include _only if you reach for it_. A feature you never touch costs you nothing: not + memory, not binary size, not compile time. +- **The row API never leaves you.** No matter which advanced storage or layout you pick underneath, + the way you insert, assign, read, and query stays identical. You choose performance trade-offs + without rewriting your program. +- **Predictability over cleverness.** SoaTable is used in places where a surprise is unacceptable, + trading systems that cannot stall, flight software that cannot throw. So it offers exception-free + error paths, deterministic memory strategies, and a build mode with no exceptions at all. When a + clever shortcut and a predictable one disagree, predictability wins. +- **Stable identity is sacred.** Handles survive deletion, insertion, and sorting. A stale handle is + _detected_, never silently pointed at the wrong record. + +## Is SoaTable the right tool for you? + +**Reach for it when:** + +- **Games and simulations (Entity-Component-System worlds).** Thousands of entities, each with a + sparse mix of components, stable identity across frames, and systems that walk a few components at + a time. +- **Finance.** Tick and portfolio tables, columnar analytics, group-by aggregation, compact storage, + and fast snapshots. +- **Engineering and aerospace telemetry.** Frame stores with optional channels, smoothly changing + signals, recompute-only-what-changed workflows, unit safety, and a no-exceptions build for flight + software. +- **Scientific and data work.** Hand a whole column straight to a math kernel or a numerical library + with no copying, run vectorized operations, or memory-map columns that are larger than RAM. + +**Prefer a plain `std::vector` when** the table is tiny, every record always has every +field, and every pass over the data touches every field anyway. SoaTable's advantage grows with +size, with sparsity, and with how selective your questions are. + +## Quick start + +Here is a complete, runnable program. Read the comments and you have the whole core idea in your +hands. ```cpp -struct Particle { - double x, y, vx, vy; - std::string name; -}; -std::vector particles; +#include +#include + +// Columns are tiny named types. The name is what makes a column distinct. +struct Health { float value; }; +struct Name { std::string value; }; +struct Poisoned {}; // A field with no data: just a yes/no tag. + +int main() { + soatable::soa_table table; + + const auto id = table.insert(); // Create a record; get back a stable handle. + table.assign(id, "Warrior"); // This record has a Name ... + table.assign(id, 100.0f); // ... and a Health, but no Poisoned tag. + + // Walk every record that has both Health and Name. The structured binding + // gives you real references: writing through them edits the table directly. + for (auto [row, health, name] : table.view()) { + health.value -= 10.0f; + std::println("{} has {} health", name.value, health.value); + } + + // Optional columns: include Poisoned when present, without requiring it. + for (auto [row, name, poison] : table.select>()) { + if (poison) std::println("{} is poisoned!", name.get().value); + } +} ``` -While simple, AoS becomes inefficient when: +## The guided tour + +Each section below opens with the everyday idea, then shows it in code. Together they cover every +feature SoaTable offers. -1. **Workloads touch only a few fields:** Scanning the whole array pulls unrelated members into cache lines. -2. **Schema is sparse:** Many rows have optional fields, leading to wasted memory or complex pointer-heavy structures. -3. **Identity stability is required:** Deleting elements from a vector invalidates indices or requires expensive shifting. +### 1. The table: rows, handles, and sparse columns -**SoaTable** solves these by storing data in a **Structure of Arrays (SoA)** format internally, while providing a row-oriented abstraction for insertion and handles. +**The idea.** A table is a fixed set of columns. You add records with `insert()`, attach fields with +`assign`, drop a field with `unassign`, and remove a record with `erase`. The handle you get back +(`row_id`) is a _generational_ ticket: it remembers not just _where_ a record lives but _which_ +record, so if a slot is reused by a later record, your old handle correctly reports itself invalid +instead of silently pointing at the newcomer. (This defeats a classic bug known as the ABA problem.) -### Performance Benefits +```cpp +soatable::soa_table table; + +const auto id = table.insert(); +table.assign(id, Age{30}); +table.contains(id); // true: this record has an Age +table.unassign(id); // drop just that field +table.erase(id); // remove the whole record +table.is_valid(id); // false: the handle knows its record is gone +``` -In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** than traditional AoS branch-and-scan methods for sparse data. By focusing iteration on the most selective column, SoaTable avoids checking every single row for presence. +A column with no data members (like `Poisoned` above) is a **tag**: it answers a yes/no question and +costs almost nothing. This is the natural way to mark state, "is frozen", "is poisoned", "needs +review", without inventing a boolean field. -## Features +### 2. Reading values safely -- **C++23 Standard:** Leverages modern features like `std::views`, `std::print`, and advanced template metaprogramming. -- **Stable IDs:** `row_id` handles remain valid until the row is explicitly erased, even if other rows are added or reordered. -- **Sparse Projection:** `select>()` allows joining required and optional columns seamlessly. -- **Batch Operations:** `insert_batch` and `assign_batch` for high-throughput data loading. -- **Parallel Sorting:** Physically reorder columns in parallel based on a comparison criteria. -- **Multi-Column Sorting:** Sort by primary, secondary, etc., keys. -- **Data Compression Utilities:** Includes `quantized_float`, `packed_bits`, and `DeltaValue` for compact storage. +**The idea.** Different situations want different answers to "what if the field isn't there?". +SoaTable gives you three readers so you can pick the failure mode that fits, rather than forcing one +on you. -## Quick Start +| Reader | When the field is missing | Use it when | +| --------------------- | ---------------------------------------- | --------------------------------------- | +| `get(id)` | throws an exception | you are confident it is there | +| `try_get(id)` | returns `nullptr` | you want a cheap "is it there?" check | +| `get_expected(id)` | returns a `std::expected` value-or-error | hot paths, or builds with no exceptions | ```cpp -#include -#include +if (auto* hp = table.try_get(id)) { // no exception, just a pointer check + hp->value -= 5.0f; +} +``` -struct Health { float value; }; -struct Name { std::string value; }; -struct Poisoned {}; // Component with no data +Reading through a `const` table gives you `const` references all the way down, so a table is safe to +share read-only across a program without anyone accidentally mutating it. -int main() { - soatable::SoaTable table; +### 3. Asking questions: views and selects - // Insertion - auto id = table.insert(); - table.assign(id, "Warrior"); - table.assign(id, 100.0f); +**The idea.** Most real work is "for every record that has _these_ fields, do something". That is +what `view` and `select` are for. Both walk only the records that have _all_ the requested fields, +and both start from the smallest of those columns so they examine as few records as possible. - // Efficient Selection - for (auto [row, health, name] : table.select()) { - std::println("Row {}: {} has {} health", row.index, name.get().value, health.get().value); - } +- **`view()`** is the everyday workhorse. Its structured bindings hand you **real + references**, so you read and write fields directly with no ceremony. +- **`select()`** is the more flexible form. It is the one that understands + `std::optional`, letting a column be included _when present_ without being required. - // Optional columns - for (auto [row, name, poison] : table.select>()) { - if (poison) { - std::println("{} is poisoned!", name.get().value); - } - } +```cpp +// Move every entity that has both a Position and a Velocity. +for (auto [id, pos, vel] : world.view()) { + pos.x += vel.x; + pos.y += vel.y; +} + +// Include Age when the record has it; the record is still listed if it doesn't. +for (auto [id, name, age] : table.select>()) { + if (age) { /* this record has an Age */ } } ``` -## Real-World Example +With `select`, fields come wrapped so you read them with `.get()`; with `view`, the binding is the +reference itself. -This example demonstrates how to use SoaTable for a vehicle fleet management system, utilizing batch operations, sparse joins, multi-column sorting, and data compression. +### 4. A friendlier handle: `row_handle` + +**The idea.** Carrying a `row_id` plus the table it belongs to everywhere can be tedious. +`row_handle` bundles the two together so you can act on a record directly, and a default-constructed +handle is safely "unbound" (it reports itself invalid rather than misbehaving). ```cpp -#include -#include -#include -#include -#include -#include -#include -#include +soatable::row_handle handle{ table.insert(), table }; +handle.assign("Dora"); +handle.assign(40); +handle.get().value; // "Dora" +handle.contains(); // true +handle.erase(); // remove the record through the handle +``` + +### 5. Sorting and bulk loading + +**The idea.** Sometimes order matters, ticks by timestamp, players by score. SoaTable can physically +reorder every column together so the data stays meaningfully ordered (which later lets time-series +windows and column views be correct). And when you are loading lots of records at once, batch +operations avoid the per-record overhead. -// 1. Data Compression Utilities -struct Status { - uint8_t flags = 0; - // packed_bits: [0-1] Engine State, [2-2] Lights, [3-3] Wiper - using EngineState = soatable::packed_bits; - using Lights = soatable::packed_bits; +```cpp +// Sort by one column. +table.sort_by_column([](const Age& a, const Age& b) { return a.value < b.value; }); + +// Sort by a primary key, breaking ties with a secondary key. +table.sort_by_multi( + std::pair>{ + {}, [](const Name& a, const Name& b) { return a.value < b.value; } }, + std::pair>{ + {}, [](const Age& a, const Age& b) { return a.value < b.value; } }); + +// Sort in parallel above a size threshold (serial below it, where threads would not pay off). +table.sort_by_column_parallel([](const Age& a, const Age& b) { return a.value < b.value; }); + +// Bulk-load many records, then fill a column from a sequence. +const auto ids = table.insert_batch(3); +std::array ages{ Age{20}, Age{30}, Age{40} }; +table.assign_batch(ids, ages.begin()); +``` + +### 6. Zero-copy columns and validity bitmaps + +**The idea.** Because each column is one dense, contiguous, aligned array, SoaTable can hand it to +you as a `std::span`, a borrowed window over the real storage, with **no copying**. That column can +go straight into a SIMD kernel, an FFT, a linear-algebra library, or any code that wants a plain +array of numbers. Alongside it, a **validity bitmap** is a compact yes/no record of which rows have +the column, perfect for fast "how many", "which ones", and branch-free masked loops. (It is also the +natural bridge to the Apache Arrow data format.) - void set_engine(uint8_t state) { EngineState::set(flags, state); } -}; +```cpp +std::span prices = table.column(); // borrowed, aligned, no copy +const auto valid = table.validity(); // packed presence bitmap +std::println("{} prices, {} rows actually priced", prices.size(), valid.count()); +``` -struct Position { float x, y; }; +### 7. Compact storage helpers -// Using quantized_float for Speed (0-200 km/h) and Fuel (0-100%) -using Speed = soatable::quantized_float; -using Fuel = soatable::quantized_float; +**The idea.** Some data does not need a full-size number. A confidence score between 0 and 1, a few +boolean flags, a value that only ever drifts a little, these can be squeezed into far fewer bytes, +which means more of them fit in cache and everything runs faster. These helpers live in the core +header and can be used directly as column types. -struct Driver { std::string name; }; -struct Maintenance { bool urgent; }; +- **`quantized_float`** stores a floating-point value, within a known range, in a small integer (for + example 8 bits), trading a little precision for a lot of space. +- **`packed_bits`** packs several small fields into the bits of one integer. +- **`delta_value`** tracks a value that changes gently by storing scaled _deltas_ rather than full + numbers, ideal for smooth signals like altitude. +- **`dirty_mask`** is a set of on/off flags, the clean way to record "which parts of this record + changed" so you can recompute only those. -int main() { - // Define the table with various column types - soatable::SoaTable, Driver, Maintenance> fleet; - - // 2. Batch Operations: High-throughput loading - auto ids = fleet.insert_batch(1000); - std::vector vids(1000); - for(int i=0; i<1000; ++i) vids[i] = 1000 + i; - fleet.assign_batch(ids, vids.begin()); - - // 3. Stable IDs: Keep a handle to a specific vehicle - auto my_truck_id = ids[50]; - fleet.assign(my_truck_id, "Alice"); - - // 4. Sparse Projection: Efficiently join required and optional columns - fleet.assign(ids[10], Maintenance{true}); - fleet.assign(ids[50], Maintenance{false}); - - // Only iterates over rows that HAVE Maintenance. Driver is optional. - for (auto [row, maint, driver] : fleet.select>()) { - std::string name = driver ? driver->get().name : "Unknown"; - std::println("Vehicle ID {}: Maintenance (Urgent: {}), Driver: {}", row.index, maint.get().urgent, name); - } +```cpp +// A confidence in [0, 1] kept in a single byte. +using Confidence = soatable::quantized_float; +Confidence c{0.75}; +double back = c.get(); // ~0.75, dequantized on read + +// Smoothly varying altitude tracked as deltas. +soatable::delta_value altitude{0.0F}; +altitude.apply_delta(altitude.get_delta(101.0F)); // nudge toward 101 m +float metres = altitude.get(); + +// Per-record flags. +enum class Subsystem : std::uint32_t { avionics = 1, propulsion = 2, thermal = 4 }; +soatable::dirty_mask flags; +flags.mark_dirty(Subsystem::thermal); +bool changed = flags.is_dirty(Subsystem::thermal); // true +``` + +### 8. The compute layer: math over columns + +**The idea.** Once your data is in columns, doing math over it should be one line, and it should be +fast because the compiler can vectorize a tight loop over an aligned array. Include +`` for that. It covers single-column transforms and reductions, scalar +broadcasting, masked and strided variants, chunk-parallel versions, and, crucially, **cross-column** +math like `pnl = price * qty` that walks the join of the inputs and only touches records that have +all of them. - // 5. Multi-Column Sorting: Physically reorder all columns - // Sort by Fuel (ascending), then by Speed (descending) - fleet.sort_by_multi( - std::make_pair(Fuel{}, [](const Fuel& a, const Fuel& b) { return a.get() < b.get(); }), - std::make_pair(Speed{}, [](const Speed& a, const Speed& b) { return a.get() > b.get(); }) - ); +```cpp +#include + +// notional = price * qty, computed for every record that has both. +soatable::compute::assign_from( + book, [](const Price& p, const Qty& q) { return Notional{ p.value * q.value }; }); + +// Sum a column with a reduction. +double gross = soatable::compute::reduce_column( + book, 0.0, [](double acc, const Notional& n) { return acc + n.value; }); +``` - // 6. Parallel Sorting: Reorder columns in parallel for large datasets - fleet.sort_by_column_parallel([](int a, int b) { return a < b; }); +### 9. Queries and group-by - // 7. Data Compression: DeltaValue for smooth tracking - auto altitude_id = ids[0]; - soatable::DeltaValue alt(100.0f); - alt.apply_delta(alt.get_delta(105.0f)); - fleet.assign>(altitude_id, alt); +**The idea.** Analytics ergonomics, "filter, then group, then total", without dragging in a database +or query engine. Include ``. `select_where` filters the join by any condition you +write; `group_sum`, `group_count`, and `group_reduce` aggregate records into buckets keyed by +whatever you project out of a column. All of it composes on top of the smallest-drawer scan, so it +stays fast. - return 0; +```cpp +#include + +// Total notional volume per symbol. +auto volume = soatable::query::group_sum( + ticks, + [](const Symbol& s){ return s.value; }, // group key + [](const Notional& n){ return n.value; }); // value to sum + +// Only the large trades. +for (auto row : soatable::query::select_where( + ticks, [](auto trade){ return trade.template get().value > 10000.0; })) { + // row.template get().value ... } ``` +### 10. Choosing how memory is laid out + +**The idea.** The default layout, one flat, contiguous, SIMD-aligned array per column, is best for +analytics. But other workloads want other trade-offs, and SoaTable lets you pick the layout _without +changing a line of your row code_. The choice is part of the table's type: + +- **`soa_table<...>`** (default): flat, contiguous, 64-byte aligned. Best for scanning and number + crunching. +- **`aosoa_table` / `chunked_soa_table`**: storage split into fixed-size tiles + or chunks, so growth never has to copy the existing data. Good for streaming and bounded latency. + `column_tiles()` views the data as per-tile aligned spans. +- **`custom_soa_table`**: bring your own allocator. The opt-in `` + gives you `pmr_soa_table`, which draws memory from a `std::pmr` arena, monotonic buffer, or pool, + for deterministic, fragmentation-free allocation in real-time code. +- **`` `mmap_soa_table`**: column storage is memory-mapped and paged in by the + operating system on demand, so a single column can be far larger than physical RAM. + +```cpp +// Tiled storage in chunks of 4; growth never copies existing chunks. +soatable::aosoa_table<4, Price> tiled; +for (int i = 0; i < 10; ++i) + tiled.assign(tiled.insert(), Price{ static_cast(i) }); +auto chunks = tiled.column_tiles(); // per-tile aligned spans +``` + +### 11. Sharing a table across threads + +**The idea.** A common pattern is "many readers, one writer", lots of worker threads reading a market +data table while one thread ingests updates. `` wraps any table in a +`synchronized_table` that lets many readers in at once but gives writers exclusive access, all +through scoped callbacks so the locking is automatic and correct. + +```cpp +#include + +soatable::synchronized_table shared; +shared.write([](Table& t){ t.insert(); }); // exclusive +auto n = shared.read([](const Table& t){ return t.size(); }); // shared with other readers +``` + +### 12. Time-series helpers + +**The idea.** When a column is ordered by time, you often want rolling windows and differences, a +20-tick moving average, the change since the last reading, and you want to recompute only the rows +that actually changed. `` provides exactly these, operating on a column's +dense order (so sort by your time key first). + +```cpp +#include + +// 2-period rolling mean of the Price column, in its stored order. +auto means = soatable::timeseries::rolling_mean_column( + book, [](const Price& p){ return p.value; }, 2); + +// Visit only the records flagged changed, and clear the flags as you go. +using Status = soatable::dirty_mask; // a dirty_mask used directly as a column +soatable::timeseries::for_each_dirty(frames, [](soatable::row_id, Status& s){ + // recompute this record, then s.reset(); +}); +``` + +There are also free-function forms (`rolling_sum`, `rolling_mean`, `deltas`) that work on any span of +values. + +### 13. Units that catch mistakes at compile time + +**The idea.** Adding metres to seconds is always a bug, so it should never compile. `` +gives you `quantity` types where multiplication and division combine units correctly +(distance / time gives a velocity) while adding mismatched units is a **compile error**. These can be +used directly as column types, so unit safety follows your data into the table. + +```cpp +#include +namespace un = soatable::units; + +auto speed = un::length<>{100.0} / un::duration<>{4.0}; // a velocity, by construction +double v = speed.value(); // 25.0 +// un::length<>{1.0} + un::duration<>{1.0}; // <-- would not compile, on purpose +``` + +### 14. Runtime schemas: the dynamic table + +**The idea.** Sometimes you do not know the columns until the program is running, loading a CSV whose +headers you have not seen, or letting a user define fields. `` gives a +`dynamic_table` whose columns are added and removed by name at runtime, type-checked on every access +(without RTTI), and each column can carry string metadata such as a unit label. + +```cpp +#include + +soatable::dynamic_table dyn; +dyn.add_column("altitude"); +dyn.set_metadata("altitude", "unit", "metres"); + +const auto row = dyn.insert_row(); +dyn.set(row, "altitude", 1280.0); +double a = *dyn.get(row, "altitude"); // 1280.0 +``` + +### 15. Saving and loading + +**The idea.** To checkpoint a simulation or snapshot a tick store, you want to write the whole table +to bytes and read it back exactly, handles, generations, and column order included. +`` does this for tables of plain, trivially-copyable columns, wrapping the +data in a versioned, schema-checked header so a load can _reject_ a buffer that does not match, +rather than silently corrupting your data. (The format is native byte order, meant for snapshots read +back on the same kind of machine.) + +```cpp +#include + +auto bytes = soatable::save(pod_table); // -> std::vector +soatable::soa_table restored; +auto status = soatable::load(restored, bytes); +bool ok = (status == soatable::serialize_status::ok); // also: bad_magic, schema_mismatch, ... +``` + +### 16. Building a table from a struct + +**The idea.** You may already have a plain struct describing a record and would rather not list its +fields twice. `` lets you derive the table type from the struct. Until C++26 +reflection ships in compilers, you write a single one-line mapping; once a compiler supports +reflection, even that line becomes unnecessary and the mapping is automatic. + +```cpp +#include + +struct ParticleSchema {}; // a tag naming your schema +template <> struct soatable::columns_of + : soatable::column_list {}; + +using ParticleTable = soatable::table_for; // == soa_table +``` + +### 17. Running without exceptions + +**The idea.** Embedded, kernel, and flight targets often forbid C++ exceptions entirely. Define +`SOATABLE_NO_EXCEPTIONS` and every place that would `throw` instead calls a terminating handler; you +lean on the non-throwing readers (`try_get`, `get_expected`) for normal error handling. SoaTable uses +no RTTI either, and a dedicated CI leg builds the whole library with `-fno-exceptions` to keep this +path honest. + +## The opt-in headers at a glance + +The core, `soatable/soatable.hpp`, gives you the table, spans, validity bitmaps, and the compression +helpers, with no extra dependencies. Everything else is a header you add only when you want it. + +| Header | What it adds | Extra standard headers it pulls in | +| ------------------------- | ------------------------------------------------- | ---------------------------------- | +| `soatable/soatable.hpp` | the table, spans, validity, compression helpers | nothing extra | +| `soatable/compute.hpp` | column math, broadcasting, chunk-parallel compute | ``, `` | +| `soatable/query.hpp` | `select_where`, group-by aggregation | `` | +| `soatable/serialize.hpp` | versioned `save()` / `load()` | nothing extra | +| `soatable/pmr.hpp` | `pmr_soa_table` (arena / pool allocation) | `` | +| `soatable/mmap.hpp` | `mmap_soa_table` (larger-than-RAM columns) | `` / `` | +| `soatable/concurrent.hpp` | `synchronized_table` (many readers, one writer) | `` | +| `soatable/timeseries.hpp` | rolling windows, deltas, dirty scans | nothing extra | +| `soatable/units.hpp` | dimensional `quantity` types | nothing extra | +| `soatable/dynamic.hpp` | runtime `dynamic_table` | `` | +| `soatable/reflect.hpp` | `table_for` | nothing extra | + +## Domain cookbooks + +These are complete, runnable programs (built as `soatable_cookbook_*` targets) that put the pieces +together for a real domain: + +- [Finance, a tick store](example/cookbook/finance_tick_store.cpp): ingest trades, derive P&L per + record, total notional volume per symbol, and filter the large trades. +- [Game / ECS](example/cookbook/game_ecs.cpp): a sparse-component world with a motion system over + `view()` and a data-less tag for frozen entities. +- [Aerospace, telemetry](example/cookbook/aerospace_telemetry.cpp): `delta_value` altitude tracking, + `dirty_mask` per-frame flags, and a validity bitmap over temperature readings. +- [Scientific data](example/cookbook/scientific_table.cpp): zero-copy span compute and + `quantized_float` storage. + +A broader [feature tour](example/soatable_example.cpp) exercises most headers in one program. + ## Installation -### CMake +SoaTable is header-only and needs a C++23 compiler (GCC 13+, Clang 18+, or MSVC from VS 2022) and +CMake 3.25+. -Add the repository as a subdirectory or install it: +### CMake ```cmake find_package(soatable CONFIG REQUIRED) target_link_libraries(your_target PRIVATE soatable::soatable) ``` -### vcpkg +### Conan / vcpkg + +```sh +conan create . # header-only recipe at the repo root +vcpkg install soatable --overlay-ports=packaging/vcpkg/ports # overlay port +``` + +### Single-header drop-in + +The core `soatable.hpp` is self-contained. To emit a stamped, vendorable copy: + +```sh +python scripts/amalgamate.py # writes dist/soatable.hpp +``` + +The opt-in add-on headers are kept separate on purpose, so you pay only for what you include. Pass +`--with compute,query` to bundle specific ones next to the core. + +### Debugger visualizers + +SoaTable ships pretty-printers that render its types (tables, handles, `delta_value`, +`quantized_float`, ...) as readable summaries in a debugger. When you consume the library through +CMake, the Visual Studio (`.natvis`) visualizer is embedded into your program automatically, with no +setup. The GDB and LLDB printers are opt-in: follow the short setup in +[Visualizers](tools/visualizers/README.md). + +## Benchmarks: what the speed actually is + +The whole point of SoaTable is speed on selective questions, so the claim is measured, not asserted. +The suite covers insertion, erase churn, single-column and parallel sort, sparse selection across a +sweep of densities, the smallest-drawer heuristic, SoA-versus-tiled selection, and compute +throughput, each against an Array-of-Structures baseline and a hand-written columnar baseline. + +```sh +./scripts/run_benchmarks.sh # builds the `bench` preset, writes bench/results.json +``` + +A representative result, a selective join over 250,000 rows in a Release build: + +| Method | Time | +| ------------------------------------------------- | ------------- | +| SoaTable `select` (starts from smallest drawer) | ~168,000 ns | +| Same, but forced to start from the largest drawer | ~1,547,000 ns | +| Hand-rolled columnar scan | ~1,058,000 ns | +| Array-of-Structures branch scan | ~1,303,000 ns | + +The roughly 9x gap between starting from the smallest drawer and being forced to start from the +largest is the whole thesis of the library in one number: examine fewer records, finish sooner. +Numbers are machine dependent; reproduce them with the harness above. + +## Building, testing, and docs + +```sh +cmake --preset clang # or: gcc, msvc, asan-ubsan, bench +cmake --build --preset clang +ctest --preset clang +``` + +CI builds GCC 13, Clang 18, and MSVC, plus an AddressSanitizer + UndefinedBehaviorSanitizer leg, all +through these same presets, so a green local run should match CI. Build the Doxygen API site locally +with: + +```sh +cmake -S . -B build-docs -DSOATABLE_BUILD_DOCUMENTATION=ON +cmake --build build-docs --target docs +``` + +## Versioning and naming -SoaTable is designed to be easily integrated into vcpkg. You can add it to your `vcpkg.json` once published or use it as an overlay. +SoaTable follows [Semantic Versioning](https://semver.org). The public API uses std-style lowercase +names (`soa_table`, `row_handle`, `column_vector`, `delta_value`, `dirty_mask`); the earlier +PascalCase spellings remain as `[[deprecated]]` aliases for one migration window so existing code +keeps compiling while you move. -## Benchmarks +## Contributing -Benchmark results (250,000 rows, selective join): -| Method | Time | -|-------------------|----------------| -| SoaTable `select` | ~162,000 ns | -| AoS Branch Scan | ~2,035,000 ns | -_Run on 4x 2300 MHz CPU, Release Build._ +Contributions are welcome. [CONTRIBUTING](CONTRIBUTING.md) explains the philosophy behind the +project and how to make a change that fits it, the principles, the local build and sanitizer +workflow, and what a good pull request looks like. ## License -MIT +MIT, see [LICENSE](LICENSE). diff --git a/VERSION.txt b/VERSION.txt new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION.txt @@ -0,0 +1 @@ +0.1.0 diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 8ddfaa3..8c8b88b 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -5,7 +5,7 @@ if(NOT benchmark_FOUND) FetchContent_Declare( benchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.5 + GIT_TAG "v${BENCHMARCK_VERSION}" ) set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) diff --git a/benchmarks/soatable_benchmark.cpp b/benchmarks/soatable_benchmark.cpp index f70b69c..f2bec26 100644 --- a/benchmarks/soatable_benchmark.cpp +++ b/benchmarks/soatable_benchmark.cpp @@ -1,11 +1,22 @@ +/// @file soatable_benchmark.cpp +/// @brief SoaTable benchmark suite. +/// @author Bertin Balouki SIMYELI +/// +/// Covers the axes where regressions hide: insertion, erase-churn, single-column and parallel sort, +/// sparse selection across density sweeps, and the select driver heuristic. Baselines compare +/// against an Array-of-Structures scan and a hand-rolled columnar scan. + #include -#include +#include #include #include +#include "soatable/compute.hpp" #include "soatable/soatable.hpp" +namespace { + struct Temperature { double celsius = 0.0; }; @@ -16,35 +27,124 @@ struct RegionId { std::uint32_t value = 0; }; -struct AOSRecord { - bool has_temperature = false; - bool has_pressure = false; - bool has_region = false; - Temperature temperature {}; - Pressure pressure {}; - RegionId region {}; -}; +using SensorTable = soatable::soa_table; -static void BM_SoaTableSelect(benchmark::State& state) { - const std::size_t row_count = static_cast(state.range(0)); - soatable::SoaTable table; +// Deterministic per-row values so sort benchmarks do real work without consuming the presence RNG. +constexpr double temperature_value(std::size_t i) { + return static_cast((i * 48271U) % 1000U); +} +constexpr double pressure_value(std::size_t i) { return static_cast((i * 16807U) % 1000U); } + +// Build a table of the given size with the given per-column presence probabilities (fixed seed). +SensorTable build_table(std::size_t row_count, double temp_p, double pressure_p, double region_p) { + SensorTable table; table.reserve(row_count); std::mt19937_64 rng(0x5A0A7EULL); - std::bernoulli_distribution temp_present(0.70); - std::bernoulli_distribution pressure_present(0.40); - std::bernoulli_distribution region_present(0.05); + std::bernoulli_distribution temp_present(temp_p); + std::bernoulli_distribution pressure_present(pressure_p); + std::bernoulli_distribution region_present(region_p); for (std::size_t i = 0; i < row_count; ++i) { const auto row = table.insert(); - if (temp_present(rng)) - table.assign(row, 25.0); - if (pressure_present(rng)) - table.assign(row, 101.3); - if (region_present(rng)) - table.assign(row, 1); + if (temp_present(rng)) { + table.assign(row, Temperature {temperature_value(i)}); + } + if (pressure_present(rng)) { + table.assign(row, Pressure {pressure_value(i)}); + } + if (region_present(rng)) { + table.assign(row, RegionId {static_cast(i % 64U)}); + } + } + return table; +} + +} // namespace + +// Insertion +static void BM_Insert(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + for (auto _ : state) { + SensorTable table; + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + } + benchmark::DoNotOptimize(table.size()); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} +BENCHMARK(BM_Insert)->Arg(100'000); + +// Erase churn +static void BM_EraseChurn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + for (auto _ : state) { + SensorTable table; + std::vector ids; + ids.reserve(row_count); + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + ids.push_back(row); + } + for (const auto id : ids) { + table.erase(id); + } + benchmark::DoNotOptimize(table.size()); + } + state.SetItemsProcessed(state.iterations() * state.range(0) * 2); +} +BENCHMARK(BM_EraseChurn)->Arg(100'000); + +// Sorting - +static void BM_SortByColumn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 1.0, 1.0, 1.0); + bool ascending = true; + for (auto _ : state) { + if (ascending) { + table.sort_by_column([](const Temperature& a, const Temperature& b) { + return a.celsius < b.celsius; + }); + } else { + table.sort_by_column([](const Temperature& a, const Temperature& b) { + return a.celsius > b.celsius; + }); + } + ascending = !ascending; + benchmark::ClobberMemory(); } +} +BENCHMARK(BM_SortByColumn)->Arg(100'000); +static void BM_SortByColumnParallel(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 1.0, 1.0, 1.0); + bool ascending = true; + for (auto _ : state) { + if (ascending) { + table.sort_by_column_parallel( + [](const Temperature& a, const Temperature& b) { return a.celsius < b.celsius; } + ); + } else { + table.sort_by_column_parallel( + [](const Temperature& a, const Temperature& b) { return a.celsius > b.celsius; } + ); + } + ascending = !ascending; + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_SortByColumnParallel)->Arg(100'000); + +// Selection: SoaTable vs baselines +static void BM_SoaTableSelect(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.70, 0.40, 0.05); for (auto _ : state) { double sum = 0.0; for (auto [id, temperature, pressure, region] : @@ -57,28 +157,36 @@ static void BM_SoaTableSelect(benchmark::State& state) { BENCHMARK(BM_SoaTableSelect)->Arg(100'000)->Arg(250'000); static void BM_AoSBranchScan(benchmark::State& state) { + struct AosRecord { + bool has_temperature = false; + bool has_pressure = false; + bool has_region = false; + double celsius = 0.0; + double kpa = 0.0; + std::uint32_t region = 0; + }; + const std::size_t row_count = static_cast(state.range(0)); - std::vector aos; + std::vector aos; aos.reserve(row_count); std::mt19937_64 rng(0x5A0A7EULL); std::bernoulli_distribution temp_present(0.70); std::bernoulli_distribution pressure_present(0.40); std::bernoulli_distribution region_present(0.05); - for (std::size_t i = 0; i < row_count; ++i) { - AOSRecord record; + AosRecord record; if (temp_present(rng)) { - record.has_temperature = true; - record.temperature.celsius = 25.0; + record.has_temperature = true; + record.celsius = temperature_value(i); } if (pressure_present(rng)) { record.has_pressure = true; - record.pressure.kpa = 101.3; + record.kpa = pressure_value(i); } if (region_present(rng)) { - record.has_region = true; - record.region.value = 1; + record.has_region = true; + record.region = static_cast(i % 64U); } aos.push_back(record); } @@ -87,7 +195,7 @@ static void BM_AoSBranchScan(benchmark::State& state) { double sum = 0.0; for (const auto& record : aos) { if (record.has_temperature && record.has_pressure && record.has_region) { - sum += record.temperature.celsius + record.pressure.kpa + record.region.value; + sum += record.celsius + record.kpa + record.region; } } benchmark::DoNotOptimize(sum); @@ -95,4 +203,176 @@ static void BM_AoSBranchScan(benchmark::State& state) { } BENCHMARK(BM_AoSBranchScan)->Arg(100'000)->Arg(250'000); +// Density sweep: select cost should track the smallest required column +static void BM_SelectDensitySweep(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + const double region_density = static_cast(state.range(1)) / 100.0; + SensorTable table = build_table(row_count, 0.90, 0.90, region_density); + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure, region] : + table.select()) { + sum += temperature.get().celsius + pressure.get().kpa + region.get().value; + } + benchmark::DoNotOptimize(sum); + } +} +// Same row count, increasing density of the driver column (1%, 5%, 20%, 50%). +BENCHMARK(BM_SelectDensitySweep) + ->Args({250'000, 1}) + ->Args({250'000, 5}) + ->Args({250'000, 20}) + ->Args({250'000, 50}); + +// Driver heuristic validation: smallest column should drive iteration - +// RegionId is sparse and Temperature is dense; the auto driver picks RegionId. The forced variant +// drives off the dense Temperature column and filters manually, modelling a wrong driver choice. +static void BM_SelectAutoSmallestDriver(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.95, 0.95, 0.02); + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure, region] : + table.select()) { + sum += temperature.get().celsius + pressure.get().kpa + region.get().value; + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectAutoSmallestDriver)->Arg(250'000); + +static void BM_SelectForcedLargestDriver(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.95, 0.95, 0.02); + for (auto _ : state) { + double sum = 0.0; + // Drive off the dense Temperature column, then filter the other columns by hand. + for (auto [id, temperature] : table.select()) { + const auto* pressure = table.try_get(id); + const auto* region = table.try_get(id); + if (pressure != nullptr && region != nullptr) { + sum += temperature.get().celsius + pressure->kpa + region->value; + } + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectForcedLargestDriver)->Arg(250'000); + +// Hand-rolled columnar (SoA) full scan baseline +static void BM_HandRolledSoAScan(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + std::vector celsius(row_count); + std::vector kpa(row_count); + std::vector region(row_count); + std::vector has_temp(row_count, 0); + std::vector has_pres(row_count, 0); + std::vector has_region(row_count, 0); + + std::mt19937_64 rng(0x5A0A7EULL); + std::bernoulli_distribution temp_present(0.70); + std::bernoulli_distribution pressure_present(0.40); + std::bernoulli_distribution region_present(0.05); + for (std::size_t i = 0; i < row_count; ++i) { + if (temp_present(rng)) { + has_temp[i] = 1; + celsius[i] = temperature_value(i); + } + if (pressure_present(rng)) { + has_pres[i] = 1; + kpa[i] = pressure_value(i); + } + if (region_present(rng)) { + has_region[i] = 1; + region[i] = static_cast(i % 64U); + } + } + + for (auto _ : state) { + double sum = 0.0; + for (std::size_t i = 0; i < row_count; ++i) { + if (has_temp[i] != 0 && has_pres[i] != 0 && has_region[i] != 0) { + sum += celsius[i] + kpa[i] + region[i]; + } + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_HandRolledSoAScan)->Arg(100'000)->Arg(250'000); + +// SoA vs AoSoA (tiled) selection +// A dense 2-column select, contiguous storage vs 1024-element tiles, to show the layout cost of a +// full-column scan is comparable while tiled storage bounds reallocation on growth. +static void BM_SelectSoaDense(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + soatable::soa_table table; + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + table.assign(row, Pressure {pressure_value(i)}); + } + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure] : table.select()) { + sum += temperature.get().celsius + pressure.get().kpa; + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectSoaDense)->Arg(250'000); + +static void BM_SelectAosoaDense(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + soatable::aosoa_table<1024, Temperature, Pressure> table; + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + table.assign(row, Pressure {pressure_value(i)}); + } + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure] : table.select()) { + sum += temperature.get().celsius + pressure.get().kpa; + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectAosoaDense)->Arg(250'000); + +// Compute throughput +static void BM_ComputeTransformColumn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + soatable::soa_table table; + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + table.assign(table.insert(), Temperature {temperature_value(i)}); + } + for (auto _ : state) { + soatable::compute::transform_column(table, [](Temperature t) { + return Temperature {t.celsius + 1.0}; + }); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} +BENCHMARK(BM_ComputeTransformColumn)->Arg(250'000); + +static void BM_ComputeReduceColumn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + soatable::soa_table table; + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + table.assign(table.insert(), Temperature {temperature_value(i)}); + } + for (auto _ : state) { + const double total = soatable::compute::reduce_column( + table, 0.0, [](double acc, const Temperature& t) { return acc + t.celsius; } + ); + benchmark::DoNotOptimize(total); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} +BENCHMARK(BM_ComputeReduceColumn)->Arg(250'000); + BENCHMARK_MAIN(); diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake new file mode 100644 index 0000000..cafde8b --- /dev/null +++ b/cmake/Helpers.cmake @@ -0,0 +1,223 @@ +cmake_minimum_required(VERSION 3.23) + +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(${PROJECT_NAME}_CXX_STANDARD 23 CACHE STRING "C++ standard for the project") +set(CMAKE_CXX_STANDARD ${${PROJECT_NAME}_CXX_STANDARD}) + +option(${PROJECT_NAME}_BUILD_TESTS "Add tests" OFF) +option(${PROJECT_NAME}_BUILD_BENCHMARKS "Add benchmark analysis" OFF) +option(${PROJECT_NAME}_BUILD_EXAMPLES "Build some examples" OFF) + +option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analisys" OFF) +option(${PROJECT_NAME}_APPLY_FORMATING "Apply formating with clang-format" OFF) +option(${PROJECT_NAME}_USE_PER_FILE_FORMATTING "For very large projects" OFF) +option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globaly" OFF) +option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documenation with Doxygen" OFF) +option(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) + +###################################################### +function(set_cxx_std target_name standard) + message(STATUS + "Setting C++${standard} standard for: ${target_name}" + ) + target_compile_features(${target_name} PUBLIC cxx_std_${standard}) +endfunction() + +###################################################### +function(configure_visibility target_name) + if (WIN32 AND BUILD_SHARED_LIBS) + set_target_properties(${target_name} PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS ON + ) + elseif (BUILD_SHARED_LIBS) + set_target_properties(${target_name} PROPERTIES + CXX_VISIBILITY_PRESET default + ) + endif() +endfunction() + +###################################################### +add_library(warnings INTERFACE) +add_library(warnings::strict ALIAS warnings) +if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(warnings INTERFACE + /W4 # Warning level 4 + /WX # Treat warnings as errors + /EHsc # Enable C++ exception handling (standard) + /permissive- # Enforce standard C++ conformance + /options:strict # unrecognized compiler options are an error + ) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU|AppleClang") + target_compile_options(warnings INTERFACE + -Wall # Enable most warning messages. + -Wpedantic # Enforces strict adherence to the C++ standard. + -Werror # Treat warnings as errors + -Wconversion # Warns about implicit type conversions that may cause loss of data. + -Wpedantic # Issue warnings needed for strict compliance to the standard. + -pedantic-errors # Like -pedantic but issue them as errors. + -Wsign-conversion # Warns about implicit conversions between signed and unsigned types. + -Wshadow # Warn when one variable shadows another (globally). + -Wshadow=local # Warn when one local variable shadows another local variable or parameter. + ) +endif() + +################################################### +function(apply_clang_tidy TARGET_NAME) + find_program(CLANG_TIDY_EXE "clang-tidy") + if(CLANG_TIDY_EXE) + message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") + else() + return() + endif() + cmake_parse_arguments(TIDY "" "EXTRA_ARGS" "" ${ARGN}) + set(LINTING_COMMAND ${CLANG_TIDY_EXE}) + if(TIDY_EXTRA_ARGS) + list(APPEND LINTING_COMMAND ${TIDY_EXTRA_ARGS}) + endif() + set_target_properties(${TARGET_NAME} PROPERTIES + CXX_CLANG_TIDY "${LINTING_COMMAND}" + ) + message(STATUS "Enabled clang-tidy for target: ${TARGET_NAME}") +endfunction() + +################################################### +function(apply_clang_tidy_globaly) + find_program(CLANG_TIDY_EXE "clang-tidy") + + if(CLANG_TIDY_EXE) + message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") + set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_EXE}) + else() + return() + endif() +endfunction() + +#################################################### +function(add_coverage_analysis path) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + message(STATUS "Code coverage enabled") + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "It's recommended to use Debug build type for code coverage") + endif() + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + add_compile_options( + -O0 -g + -fcoverage-mapping + -fcoverage-mcdc + -ftest-coverage + --coverage + ) + else() + add_compile_options(-O0 -ggdb --coverage) + endif() + add_link_options(--coverage) + endif() + + find_program(GCOVR_EXE gcovr) + if(GCOVR_EXE) + add_custom_target( + coverage + COMMAND ${GCOVR_EXE} + -r "${src_path}" + --filter "${src_path}/src/" + --html "coverage.html" + COMMENT "Generating code coverage report..." + ) + message(STATUS "Added 'coverage' target") + endif() +endfunction() + +#################################################### +function(build_documenation) + find_package(Doxygen) + if (Doxygen_FOUND) + message( + STATUS + "Found Doxygen Version " "${DOXYGEN_VERSION} " + "at ${DOXYGEN_EXECUTABLE}" + ) + configure_file( + "${PROJECT_SOURCE_DIR}/docs/Doxyfile.in" + "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + @ONLY + ) + doxygen_add_docs( + docs + CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + COMMENT "Generating docs with Doxygen ..." + ) + message(STATUS "Add 'docs' target") + message(STATUS "To build the documenation, run: cmake --build . --target docs") + endif() +endfunction() + +##################################################### +function(apply_formating USE_PER_FILE_LOGIC PATH) + find_program(CLANG_FORMAT_EXE clang-format) + + if (NOT CLANG_FORMAT_EXE) + return() + endif() + + file(GLOB_RECURSE SOURCES "${PATH}/*.cpp") + file(GLOB_RECURSE HEADERS "${PATH}/*.h" "${PATH}/*.hpp") + set(ALL_FILES ${SOURCES} ${HEADERS}) + + if(NOT TARGET format) + add_custom_target(format COMMENT "Formatting files with clang-format ...") + message(STATUS "Add 'format' target. Run: cmake --build \"${CMAKE_BINARY_DIR}\" --target format") + endif() + + if(NOT ALL_FILES) + return() + endif() + + if (USE_PER_FILE_LOGIC) + foreach(file ${ALL_FILES}) + add_custom_command( + TARGET format + POST_BUILD + COMMAND ${CLANG_FORMAT_EXE} -i "${file}" + COMMENT "Formatting ${file}" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + endforeach() + else() + add_custom_command( + TARGET format + POST_BUILD + COMMAND ${CLANG_FORMAT_EXE} -i ${ALL_FILES} + COMMENT "Formatting files in ${PATH} with clang-format ..." + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + endif() + +endfunction() + +##################################################### +function(anable_address_sanitizer) + set(SANITIZER_SUPPORTED OFF) + if (MINGW) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(SANITIZER_SUPPORTED OFF) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(SANITIZER_SUPPORTED ON) + endif() + elseif (MSVC OR CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + set(SANITIZER_SUPPORTED ON) + endif() + + if (SANITIZER_SUPPORTED) + message(STATUS "Enabling AddressSanitizer") + if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_options(/fsanitize=address /Zi) + add_link_options(/INCREMENTAL:NO) + else() + add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g) + add_link_options(-fsanitize=address) + endif() + else() + message(WARNING "Sanitizers are NOT supported with ${CMAKE_CXX_COMPILER_ID} on MinGW. Skipping.") + endif() +endfunction() diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake new file mode 100644 index 0000000..8a2eef5 --- /dev/null +++ b/cmake/Versions.cmake @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.20) + +set(GTEST_VERSION 1.17.0 CACHE STRING "") +set(BENCHMARCK_VERSION 1.9.5 CACHE STRING "") diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..921b958 --- /dev/null +++ b/conanfile.py @@ -0,0 +1,65 @@ +"""Conan recipe for SoaTable, a header-only C++23 sparse Structure-of-Arrays table.""" + +import os + +from conan import ConanFile +from conan.tools.cmake import cmake_layout +from conan.tools.files import copy + + +class SoaTableConan(ConanFile): + name = "soatable" + version = "0.3.0" + license = "MIT" + url = "https://github.com/bbalouki/soatable" + homepage = "https://github.com/bbalouki/soatable" + description = ( + "Header-only C++23 sparse Structure-of-Arrays table with generational handles." + ) + topics = ("soa", "ecs", "columnar", "header-only", "cpp23") + + settings = "os", "arch", "compiler", "build_type" + no_copy_source = True + exports_sources = ( + "include/*", + "LICENSE", + "tools/visualizers/*.natvis", + "tools/visualizers/*.py", + "tools/visualizers/README.md", + ) + + def package_id(self): + self.info.clear() + + def layout(self): + cmake_layout(self) + + def package(self): + copy( + self, + "*.hpp", + src=os.path.join(self.source_folder, "include"), + dst=os.path.join(self.package_folder, "include"), + ) + copy( + self, + "LICENSE", + src=self.source_folder, + dst=os.path.join(self.package_folder, "licenses"), + ) + + copy( + self, + "*", + src=os.path.join(self.source_folder, "tools", "visualizers"), + dst=os.path.join(self.package_folder, "res", "soatable", "visualizers"), + excludes=("__pycache__/*",), + ) + + def package_info(self): + self.cpp_info.bindirs = [] + self.cpp_info.libdirs = [] + self.cpp_info.resdirs = ["res"] + self.cpp_info.set_property("cmake_file_name", "soatable") + self.cpp_info.set_property("cmake_target_name", "soatable::soatable") + self.cpp_info.cppstd = "23" diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in new file mode 100644 index 0000000..0f9bd65 --- /dev/null +++ b/docs/Doxyfile.in @@ -0,0 +1,29 @@ +# Doxygen configuration template. CMake's build_documenation() helper substitutes the @VARS@ via +# configure_file(@ONLY) and drives generation through the `docs` target. + +PROJECT_NAME = "SoaTable" +PROJECT_NUMBER = @PROJECT_VERSION@ +PROJECT_BRIEF = "Header-only C++23 sparse Structure-of-Arrays table" + +OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@/doxygen +INPUT = @CMAKE_SOURCE_DIR@/include @CMAKE_SOURCE_DIR@/README.md +USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/README.md +RECURSIVE = YES + +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +JAVADOC_AUTOBRIEF = YES +MARKDOWN_SUPPORT = YES +SORT_MEMBER_DOCS = NO + +GENERATE_HTML = YES +HTML_OUTPUT = html +GENERATE_LATEX = NO + +QUIET = YES +WARN_IF_UNDOCUMENTED = NO +WARNINGS = YES + +# C++23 sources; help Doxygen parse modern constructs. +BUILTIN_STL_SUPPORT = YES +CPP_CLI_SUPPORT = NO diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index e073825..49d2c9d 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -1,9 +1,22 @@ -add_executable(soatable_example soatable_example.cpp) -target_compile_features(soatable_example PRIVATE cxx_std_23) - if(NOT TARGET soatable::soatable) find_package(soatable CONFIG REQUIRED) endif() +add_executable(soatable_example soatable_example.cpp) +target_compile_features(soatable_example PRIVATE cxx_std_23) target_link_libraries(soatable_example PRIVATE soatable::soatable) +# Domain cookbooks: one worked example per domain (finance, game/ECS, aerospace, scientific). +set(SOATABLE_COOKBOOKS + finance_tick_store + game_ecs + aerospace_telemetry + scientific_table +) +foreach(cookbook IN LISTS SOATABLE_COOKBOOKS) + add_executable(soatable_cookbook_${cookbook} cookbook/${cookbook}.cpp) + target_compile_features(soatable_cookbook_${cookbook} PRIVATE cxx_std_23) + target_include_directories(soatable_cookbook_${cookbook} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cookbook) + target_link_libraries(soatable_cookbook_${cookbook} PRIVATE soatable::soatable) +endforeach() + diff --git a/example/cookbook/aerospace_telemetry.cpp b/example/cookbook/aerospace_telemetry.cpp new file mode 100644 index 0000000..595f094 --- /dev/null +++ b/example/cookbook/aerospace_telemetry.cpp @@ -0,0 +1,63 @@ +/// @file aerospace_telemetry.cpp +/// @brief Aerospace cookbook: a telemetry frame store. +/// @author Bertin Balouki SIMYELI +/// +/// Track smoothly-varying altitude with delta_value, flag changed subsystems per frame with +/// dirty_mask, and report which frames carry a temperature reading via the validity bitmap. + +#include + +#include "output.hpp" +#include "soatable/soatable.hpp" + +namespace { +enum class Subsystem : std::uint32_t { + avionics = 1U << 0U, + propulsion = 1U << 1U, + thermal = 1U << 2U, +}; + +struct Altitude { + soatable::delta_value metres {0.0F}; +}; +struct Temperature { + float celsius = 0.0F; +}; +struct Status { + soatable::dirty_mask flags; +}; +} // namespace + +int main() { + soatable::soa_table frames; + + for (int frame = 0; frame < 5; ++frame) { + const auto id = frames.insert(); + + Altitude altitude; + altitude.metres.apply_delta(altitude.metres.get_delta(100.0F + static_cast(frame))); + frames.assign(id, altitude); + + Status status; + status.flags.mark_dirty(Subsystem::avionics); + if (frame % 2 == 0) { + status.flags.mark_dirty(Subsystem::thermal); + frames.assign(id, Temperature {20.0F + static_cast(frame)}); + } + frames.assign(id, status); + } + + // Report each frame and which subsystems it flagged dirty. + for (auto [id, altitude, status] : frames.select()) { + static_cast(id); + OUT_PRINTLN( + "altitude {:.1f} m, thermal dirty: {}", + altitude.get().metres.get(), + status.get().flags.is_dirty(Subsystem::thermal) + ); + } + + // Validity bitmap: how many frames carry a temperature reading. + OUT_PRINTLN("frames with temperature: {}", frames.validity().count()); + return 0; +} diff --git a/example/cookbook/finance_tick_store.cpp b/example/cookbook/finance_tick_store.cpp new file mode 100644 index 0000000..d5bc6ed --- /dev/null +++ b/example/cookbook/finance_tick_store.cpp @@ -0,0 +1,70 @@ +/// @file finance_tick_store.cpp +/// @brief Finance cookbook: a tick store. +/// @author Bertin Balouki SIMYELI +/// +/// Ingest trades, derive P&L per row with a cross-column op, and aggregate notional volume per +/// symbol with the query helpers. + +#include +#include + +#include "output.hpp" +#include "soatable/compute.hpp" +#include "soatable/query.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Symbol { + std::string value; +}; +struct Price { + double value = 0.0; +}; +struct Quantity { + double value = 0.0; +}; +struct Notional { + double value = 0.0; +}; +} // namespace + +int main() { + soatable::soa_table ticks; + + const auto add_tick = [&](const std::string& symbol, double price, double qty) { + const auto id = ticks.insert(); + ticks.assign(id, Symbol {symbol}); + ticks.assign(id, Price {price}); + ticks.assign(id, Quantity {qty}); + }; + add_tick("AAPL", 190.10, 100); + add_tick("AAPL", 190.50, 50); + add_tick("MSFT", 410.00, 30); + add_tick("MSFT", 409.50, 70); + + // notional = price * quantity, computed row-wise over the join of Price and Quantity. + soatable::compute::assign_from( + ticks, [](const Price& p, const Quantity& q) { return Notional {p.value * q.value}; } + ); + + // Total notional volume per symbol. + const auto volume = soatable::query::group_sum( + ticks, [](const Symbol& s) { return s.value; }, [](const Notional& n) { return n.value; } + ); + for (const auto& [symbol, total] : volume) { + OUT_PRINTLN("{}: notional volume {:.2f}", symbol, total); + } + + // Print the large trades (notional over 10k). + OUT_PRINTLN(" Large Trades "); + for (auto row : soatable::query::select_where(ticks, [](auto trade) { + return trade.template get().value > 10000.0; + })) { + OUT_PRINTLN( + "{} notional {:.2f}", + row.template get().value, + row.template get().value + ); + } + return 0; +} diff --git a/example/cookbook/game_ecs.cpp b/example/cookbook/game_ecs.cpp new file mode 100644 index 0000000..d3a50f5 --- /dev/null +++ b/example/cookbook/game_ecs.cpp @@ -0,0 +1,58 @@ +/// @file game_ecs.cpp +/// @brief Game / ECS cookbook: a world of entities with sparse components. +/// @author Bertin Balouki SIMYELI +/// +/// Integrate motion over the entities that have both Position and Velocity, and tag a subset with a +/// data-less component. + +#include + +#include "output.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Position { + float x = 0.0F; + float y = 0.0F; +}; +struct Velocity { + float x = 0.0F; + float y = 0.0F; +}; +struct Name { + std::string value; +}; +struct Frozen {}; // A data-less tag component. +} // namespace + +int main() { + soatable::soa_table world; + + const auto spawn = [&](const std::string& name, Position pos, Velocity vel) { + const auto entity = world.insert(); + world.assign(entity, Name {name}); + world.assign(entity, pos); + world.assign(entity, vel); + return entity; + }; + spawn("player", Position {0.0F, 0.0F}, Velocity {1.0F, 2.0F}); + const auto rock = spawn("rock", Position {5.0F, 5.0F}, Velocity {0.0F, 0.0F}); + spawn("enemy", Position {10.0F, 0.0F}, Velocity {-1.0F, 0.0F}); + world.assign(rock); // The rock does not move (data-less tag). + + // Motion system: advance every entity that has Position and Velocity and is not Frozen. + constexpr float time_step = 0.5F; + for (auto [id, position, velocity] : world.view()) { + if (world.contains(id)) { + continue; + } + position.x += velocity.x * time_step; + position.y += velocity.y * time_step; + } + + for (auto [id, name, position] : world.select()) { + static_cast(id); + OUT_PRINTLN("{} at ({:.1f}, {:.1f})", name.get().value, position.get().x, position.get().y); + } + return 0; +} diff --git a/example/cookbook/output.hpp b/example/cookbook/output.hpp new file mode 100644 index 0000000..5777dfc --- /dev/null +++ b/example/cookbook/output.hpp @@ -0,0 +1,16 @@ +/// @file output.hpp +/// @brief Portable println helper shared by the cookbook examples: uses std::println where +/// is available (per SOATABLE_HAS_PRINT), otherwise falls back to std::cout + std::format. +/// @author Bertin Balouki SIMYELI +#pragma once + +#include "soatable/soatable.hpp" + +#if SOATABLE_HAS_PRINT +#include +#define OUT_PRINTLN(...) std::println(__VA_ARGS__) +#else +#include +#include +#define OUT_PRINTLN(...) std::cout << std::format(__VA_ARGS__) << std::endl +#endif diff --git a/example/cookbook/scientific_table.cpp b/example/cookbook/scientific_table.cpp new file mode 100644 index 0000000..3f40536 --- /dev/null +++ b/example/cookbook/scientific_table.cpp @@ -0,0 +1,55 @@ +/// @file scientific_table.cpp +/// @brief Scientific cookbook: a measurement table. +/// @author Bertin Balouki SIMYELI +/// +/// Hand a column straight to a vectorized kernel via the zero-copy span, reduce it, and store a +/// compact quantized confidence per sample. + +#include +#include + +#include "output.hpp" +#include "soatable/compute.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Signal { + double value = 0.0; +}; +struct Energy { + double value = 0.0; +}; +// Confidence in [0, 1] stored in 8 bits. +using Confidence = soatable::quantized_float; +struct Sample { + Confidence confidence {}; +}; +} // namespace + +int main() { + soatable::soa_table data; + + for (int i = 0; i < 8; ++i) { + const auto id = data.insert(); + data.assign(id, Signal {std::sin(static_cast(i))}); + data.assign(id, Sample {Confidence {0.5 + 0.05 * i}}); + } + + // energy = signal^2, computed row-wise, then summed via a zero-copy span reduction. + soatable::compute::assign_from(data, [](const Signal& s) { + return Energy {s.value * s.value}; + }); + const double total_energy = soatable::compute::reduce_column( + data, 0.0, [](double acc, const Energy& e) { return acc + e.value; } + ); + OUT_PRINTLN("total energy: {:.4f}", total_energy); + + // Average confidence (dequantized on read). + double confidence_sum = 0.0; + for (auto [id, sample] : data.select()) { + static_cast(id); + confidence_sum += sample.get().confidence.get(); + } + OUT_PRINTLN("mean confidence: {:.3f}", confidence_sum / static_cast(data.size())); + return 0; +} diff --git a/example/soatable_example.cpp b/example/soatable_example.cpp index 34189a1..2d027de 100644 --- a/example/soatable_example.cpp +++ b/example/soatable_example.cpp @@ -1,166 +1,146 @@ +/// @file soatable_example.cpp +/// @brief A guided tour exercising most of SoaTable's headers in one runnable program. +/// @author Bertin Balouki SIMYELI +/// +/// Walks through the container basics, reference-semantic views, the non-throwing accessor, +/// zero-copy spans and validity bitmaps, the compute layer, query aggregation, binary serialization +/// round-trip, tiled (AoSoA) storage, dimensional units, time-series helpers, and the runtime +/// dynamic table. + #include -#include -#include +#include #include #include +#include "soatable/compute.hpp" +#include "soatable/dynamic.hpp" +#include "soatable/query.hpp" +#include "soatable/serialize.hpp" #include "soatable/soatable.hpp" +#include "soatable/timeseries.hpp" +#include "soatable/units.hpp" #if SOATABLE_HAS_PRINT #include #define OUT_PRINTLN(...) std::println(__VA_ARGS__) #else #include +#include #define OUT_PRINTLN(...) std::cout << std::format(__VA_ARGS__) << std::endl #endif -struct CashFlow { - double value = 0.0; -}; - -struct AccountId { - std::uint64_t value = 0; -}; - -struct RiskScore { - soatable::quantized_float value; -}; - -struct Mass { - double kg = 0.0; +namespace { +struct Symbol { + std::string value; }; - -struct Velocity { - double dx = 0.0; - double dy = 0.0; +struct Price { + double value = 0.0; }; - -struct Position { - double x = 0.0; - double y = 0.0; +struct Qty { + int value = 0; }; - -struct ObjectName { - std::string value; +struct Notional { + double value = 0.0; }; - -struct IsActive {}; // Component with no data (tag) - -using Table = soatable:: - SoaTable; -using Row = soatable:: - RowHandle; +using Book = soatable::soa_table; + +Book make_book() { + Book book; + const auto add = [&](const std::string& symbol, double price, int qty) { + const auto id = book.insert(); // Stable generational handle. + book.assign(id, Symbol {symbol}); + book.assign(id, Price {price}); + book.assign(id, Qty {qty}); + }; + add("AAPL", 190.0, 100); + add("AAPL", 191.0, 50); + add("MSFT", 410.0, 30); + return book; +} +} // namespace int main() { - Table database; - database.reserve(32); - - auto make_row = [&](Table& table) { return Row {table.insert(), table}; }; - - OUT_PRINTLN(" Basic Operations "); - auto transaction_a = make_row(database); - transaction_a.assign(99014522); - transaction_a.assign(1450000.50); - transaction_a.assign(12.50); - transaction_a.assign("High-Yield Corporate Bond Portfolio"); - transaction_a.assign(); - - auto transaction_b = make_row(database); - transaction_b.assign(88231011); - transaction_b.assign(-320000.00); - transaction_b.assign(45.80); - transaction_b.assign("Short-Selling Speculative Derivative Margin"); - - OUT_PRINTLN("Finance rows:"); - double net_balance = 0.0; - for (auto [id, flow, risk, name] : database.select()) { - OUT_PRINTLN(" row {}: {}", id.index, name.get().value); - OUT_PRINTLN(" flow={}, risk={}", flow.get().value, risk.get().value.get()); - net_balance += flow.get().value; + Book book = make_book(); + + // 1. Reference-semantic views: + // structured bindings yield real references (no .get()). + OUT_PRINTLN("== view =="); + for (auto [id, symbol, price] : book.view()) { + static_cast(id); + OUT_PRINTLN("{} @ {:.2f}", symbol.value, price.value); } - OUT_PRINTLN(" net balance = {}\n", net_balance); - OUT_PRINTLN(" Optional Columns Selection "); - // Using std::optional to join required and optional columns - for (auto [id, name, active] : database.select>()) { - OUT_PRINTLN(" row {}: {} (Active: {})", id.index, name.get().value, active.has_value()); - } + // 2. Compute: notional = price * qty over the Price/Qty join, then a span reduction. + soatable::compute::assign_from(book, [](const Price& p, const Qty& q) { + return Notional {p.value * q.value}; + }); + const double gross = soatable::compute::reduce_column( + book, 0.0, [](double acc, const Notional& n) { return acc + n.value; } + ); + OUT_PRINTLN("== compute == gross notional {:.2f}", gross); - OUT_PRINTLN("\n Physics Simulation Scenario "); - auto body_a = make_row(database); - body_a.assign(1200.0); - body_a.assign(0.0, 0.0); - body_a.assign(1.5, -0.5); - body_a.assign("Dynamic Projectile A"); - - auto body_b = make_row(database); - body_b.assign(50.0); - body_b.assign(10.0, 50.0); - body_b.assign(0.0, 9.81); - body_b.assign("Dynamic Projectile B"); - - const double dt = 0.016; - for (auto [id, position, velocity, mass, name] : - database.select()) { - position.get().x += velocity.get().dx * dt; - position.get().y += velocity.get().dy * dt; - OUT_PRINTLN( - " row {}: {} ({} kg) -> ({}, {})", - id.index, - name.get().value, - mass.get().kg, - position.get().x, - position.get().y - ); + // 3. Query: total notional per symbol. + OUT_PRINTLN("== query =="); + const auto volume = soatable::query::group_sum( + book, [](const Symbol& s) { return s.value; }, [](const Notional& n) { return n.value; } + ); + for (const auto& [symbol, total] : volume) { + OUT_PRINTLN("{}: {:.2f}", symbol, total); } - OUT_PRINTLN("\n Batch Operations "); - auto batch_ids = database.insert_batch(3); - std::vector names = {{"Batch 1"}, {"Batch 2"}, {"Batch 3"}}; - database.assign_batch(batch_ids, names.begin()); - OUT_PRINTLN(" Inserted and assigned 3 rows via batch operations."); + // 4. Zero-copy span + validity bitmap. + const std::span prices = book.column(); + OUT_PRINTLN( + "== span == {} prices, {} rows priced", prices.size(), book.validity().count() + ); - OUT_PRINTLN("\n Sorting "); - OUT_PRINTLN("Sorting by cash flow:"); - database.sort_by([](const CashFlow& lhs, const CashFlow& rhs) { - return lhs.value < rhs.value; - }); - for (auto [id, flow, name] : database.select()) { - OUT_PRINTLN(" row {}: {} :: {}", id.index, flow.get().value, name.get().value); + // 5. Serialization round-trip over a trivially-copyable schema. + soatable::soa_table pod; + for (auto [id, price, qty] : book.select()) { + static_cast(id); + const auto row = pod.insert(); + pod.assign(row, price.get()); + pod.assign(row, qty.get()); } - - OUT_PRINTLN("\n Handle Safety & Recycling "); - OUT_PRINTLN(" body_a valid before erase: {}", body_a.is_valid()); - body_a.erase(); - OUT_PRINTLN(" body_a valid after erase: {}", body_a.is_valid()); - - auto recycled = make_row(database); - recycled.assign(10500.0); - recycled.assign("Recycled Micro-Loan Settlement"); + const auto bytes = soatable::save(pod); + soatable::soa_table restored; + const auto status = soatable::load(restored, bytes); OUT_PRINTLN( - " recycled row = (index: {}, generation: {})", recycled.id.index, recycled.id.generation - ); - OUT_PRINTLN(" Stale body_a handle valid: {} (should be false)", body_a.is_valid()); - - OUT_PRINTLN("\n Edge Case: Multi-column sorting with missing values "); - // Sort by AccountId. Rows without AccountId should be handled gracefully. - database.sort_by_multi( - std::pair> { - {}, [](const AccountId& a, const AccountId& b) { return a.value < b.value; } - } + "== serialize == {} bytes, restored {} rows, ok={}", + bytes.size(), + restored.size(), + status == soatable::serialize_status::ok ); - OUT_PRINTLN(" Sorted by AccountId (rows without it should be last):"); - for (auto row : database.rows()) { - auto* acc = database.try_get(row); - auto* name = database.try_get(row); - OUT_PRINTLN( - " Row {}: Account={}, Name={}", - row.index, - acc ? std::to_string(acc->value) : "N/A", - name ? name->value : "N/A" - ); + // 6. Tiled (AoSoA) storage with per-chunk aligned spans. + soatable::aosoa_table<4, Price> tiled; + for (int i = 0; i < 10; ++i) { + tiled.assign(tiled.insert(), Price {static_cast(i)}); } + OUT_PRINTLN("== aosoa == {} price chunks", tiled.column_tiles().size()); + + // 7. Dimensional units: velocity = distance / time, checked at compile time. + namespace un = soatable::units; + const auto speed = un::length<> {100.0} / un::duration<> {4.0}; + OUT_PRINTLN("== units == speed {:.1f} (length/time)", speed.value()); + + // 8. Time-series rolling mean over the price column's dense order. + const auto means = soatable::timeseries::rolling_mean_column( + book, [](const Price& p) { return p.value; }, 2 + ); + OUT_PRINTLN("== timeseries == last 2-window mean {:.2f}", means.back()); + + // 9. Runtime dynamic schema with metadata. + soatable::dynamic_table dyn; + dyn.add_column("altitude"); + dyn.set_metadata("altitude", "unit", "metres"); + const auto sample = dyn.insert_row(); + dyn.set(sample, "altitude", 1280.0); + OUT_PRINTLN( + "== dynamic == altitude {:.0f} {}", + *dyn.get(sample, "altitude"), + *dyn.get_metadata("altitude", "unit") + ); return 0; } diff --git a/include/soatable/compute.hpp b/include/soatable/compute.hpp new file mode 100644 index 0000000..94a9670 --- /dev/null +++ b/include/soatable/compute.hpp @@ -0,0 +1,310 @@ +/// @file compute.hpp +/// @brief Opt-in vectorized column operations ("the numpy part") layered on the zero-copy column +/// spans from soatable.hpp. +/// @author Bertin Balouki SIMYELI +/// +/// Single-column ufuncs operate on std::span and rely on compiler auto-vectorization over the +/// 64-byte-aligned column storage; table-level helpers apply them through column_tiles(), so +/// they work uniformly for the contiguous (SoA) and tiled (AoSoA) storage policies. Cross-column +/// (row-wise) computation goes through the select/view join, since independent sparse columns are +/// not row-aligned in storage. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable::compute { + +// ========================= +// Single-column span ufuncs +// ========================= + +/// @brief Transform a column span in place. +/// @param values The span to modify. +/// @param op A callable mapping each element to its new value. +template +void transform(std::span values, UnaryOp op) { + std::transform(values.begin(), values.end(), values.begin(), op); +} + +/// @brief Transform an input span into an output span of equal length. +/// @param input The source span. +/// @param output The destination span (must be at least input.size()). +/// @param op A callable mapping an input element to an output element. +template +void transform(std::span input, std::span output, UnaryOp op) { + std::transform(input.begin(), input.end(), output.begin(), op); +} + +/// @brief Reduce a column span with an associative, commutative operation. +/// @param values The span to reduce. +/// @param init The identity / starting value. +/// @param op The reduction operation (defaults to addition). +/// @return The reduced value. +template > +[[nodiscard]] U reduce(std::span values, U init, BinaryOp op = {}) { + return std::reduce(values.begin(), values.end(), std::move(init), op); +} + +/// @brief Inclusive prefix scan of an input span into an output span. +/// @param input The source span. +/// @param output The destination span (must be at least input.size()). +/// @param op The combining operation (defaults to addition). +template > +void inclusive_scan(std::span input, std::span output, BinaryOp op = {}) { + std::inclusive_scan(input.begin(), input.end(), output.begin(), op); +} + +/// @brief Count the elements of a column span that satisfy a predicate. +/// @param values The span to scan. +/// @param pred The predicate. +/// @return The number of matching elements. +template +[[nodiscard]] std::size_t count_if(std::span values, Pred pred) { + return static_cast(std::count_if(values.begin(), values.end(), pred)); +} + +// ========================= +// Table column conveniences +// ========================= + +/// @brief Transform every value of a column in place (works for SoA and AoSoA storage). +/// @tparam T The column type. +/// @param table The table. +/// @param op A callable mapping each value to its new value. +template +void transform_column(basic_soa_table& table, UnaryOp op) { + for (std::span tile : table.template column_tiles()) { + compute::transform(tile, op); + } +} + +/// @brief Left-fold a column's values, allowing a projecting accumulator. +/// @tparam T The column type. +/// @param table The table. +/// @param init The starting accumulator value. +/// @param op A callable op(accumulator, const T&) returning the new accumulator. +/// @return The accumulated value. +template +[[nodiscard]] U reduce_column( + const basic_soa_table& table, U init, BinaryOp op +) { + U accumulator = std::move(init); + for (std::span tile : table.template column_tiles()) { + for (const T& value : tile) { + accumulator = op(std::move(accumulator), value); + } + } + return accumulator; +} + +/// @brief Count the rows whose value in column T satisfies a predicate. +/// @tparam T The column type. +/// @param table The table. +/// @param pred The predicate over const T&. +/// @return The number of matching values. +template +[[nodiscard]] std::size_t count_column_if( + const basic_soa_table& table, Pred pred +) { + std::size_t total = 0; + for (std::span tile : table.template column_tiles()) { + total += compute::count_if(tile, pred); + } + return total; +} + +// ===================================== +// Broadcasting, masked, and strided ops +// ===================================== + +/// @brief Broadcast a scalar over a column span in place: values[i] = op(values[i], scalar). +/// @param values The span to modify. +/// @param scalar The scalar operand applied to every element. +/// @param op The combining operation (defaults to addition). +template > +void broadcast(std::span values, const Scalar& scalar, BinaryOp op = {}) { + for (T& value : values) { + value = op(value, scalar); + } +} + +/// @brief Add a scalar to every element of a column span (a column bias). +template +void add_scalar(std::span values, const Scalar& scalar) { + broadcast(values, scalar, std::plus<> {}); +} + +/// @brief Multiply every element of a column span by a scalar. +template +void multiply_scalar(std::span values, const Scalar& scalar) { + broadcast(values, scalar, std::multiplies<> {}); +} + +/// @brief Transform only the elements that satisfy a predicate (a value-flagged subset). +/// @param values The span to modify. +/// @param pred A predicate over const T& selecting which elements to transform. +/// @param op A callable mapping a selected element to its new value. +template +void transform_if(std::span values, Pred pred, UnaryOp op) { + for (T& value : values) { + if (pred(value)) { + value = op(value); + } + } +} + +/// @brief Transform the elements flagged by a parallel byte mask of equal length. +/// @param values The span to modify. +/// @param mask A span where a non-zero entry flags the element at the same index. +/// @param op A callable mapping a flagged element to its new value. +template +void transform_masked(std::span values, std::span mask, UnaryOp op) { + const std::size_t count = std::min(values.size(), mask.size()); + for (std::size_t i = 0; i < count; ++i) { + if (mask[i] != 0) { + values[i] = op(values[i]); + } + } +} + +/// @brief Transform every stride-th element of a column span (a strided subset). +/// @param values The span to modify. +/// @param stride The step between transformed elements (must be positive). +/// @param op A callable mapping a selected element to its new value. +template +void transform_strided(std::span values, std::size_t stride, UnaryOp op) { + for (std::size_t i = 0; i < values.size(); i += stride) { + values[i] = op(values[i]); + } +} + +/// @brief Broadcast a scalar over a whole column in place (works for SoA and AoSoA storage). +/// @tparam T The column type. +/// @param table The table. +/// @param scalar The scalar operand. +/// @param op The combining operation (defaults to addition). +template < + typename T, + typename Storage, + typename... Columns, + typename Scalar, + typename BinaryOp = std::plus<>> +void broadcast_column( + basic_soa_table& table, const Scalar& scalar, BinaryOp op = {} +) { + for (std::span tile : table.template column_tiles()) { + compute::broadcast(tile, scalar, op); + } +} + +/// @brief Transform a column's flagged values in place across both storage policies. +/// @tparam T The column type. +/// @param table The table. +/// @param pred A predicate over const T& selecting which values to transform. +/// @param op A callable mapping a selected value to its new value. +template +void transform_column_if(basic_soa_table& table, Pred pred, UnaryOp op) { + for (std::span tile : table.template column_tiles()) { + compute::transform_if(tile, pred, op); + } +} + +// ==================================== +// Chunk-wise iteration and parallelism +// ==================================== + +/// @brief Number of elements below which chunk-parallel ops run serially (task-launch crossover). +inline constexpr std::size_t parallel_compute_threshold = 1U << 14; + +/// @brief Invoke a callback for each storage chunk of a column. +/// @tparam T The column type. +/// @param table The table. +/// @param func A callable taking a std::span over one chunk. +/// @note A single whole-column span for soa_layout; one span per tile for aosoa_layout. The hook +/// for custom per-chunk kernels. +template +void for_each_chunk(basic_soa_table& table, Func func) { + for (std::span chunk : table.template column_tiles()) { + func(chunk); + } +} + +/// @brief Transform a column in place, dispatching its chunks across threads when large enough. +/// @tparam T The column type. +/// @param table The table. +/// @param op A callable mapping each value to its new value. +/// @note Falls back to a serial transform below parallel_compute_threshold elements, with a single +/// chunk, or on a single hardware thread. Chunks are disjoint, so reordering carries no data race. +template +void transform_column_parallel(basic_soa_table& table, UnaryOp op) { + std::vector> chunks = table.template column_tiles(); + + std::size_t total = 0; + for (const std::span& chunk : chunks) { + total += chunk.size(); + } + + if (chunks.size() <= 1 || total < parallel_compute_threshold || + std::thread::hardware_concurrency() <= 1) { + for (std::span chunk : chunks) { + compute::transform(chunk, op); + } + return; + } + + // Dispatch all but the first chunk to async tasks; the caller transforms the first + // concurrently. + std::vector> futures; + futures.reserve(chunks.size() - 1); + for (std::size_t i = 1; i < chunks.size(); ++i) { + futures.push_back(std::async(std::launch::async, [chunk = chunks[i], op]() mutable { + compute::transform(chunk, op); + })); + } + compute::transform(chunks[0], op); + for (std::future& future : futures) { + future.get(); + } +} + +// =================================== +// Cross-column (row-wise) computation +// =================================== + +/// @brief Compute an output column from one or more input columns, per row. +/// +/// Iterates the rows that have every input column (the select/view join) and assigns the result of +/// @p op to the output column. Runs in two passes so assigning the output never invalidates the +/// in-progress join. +/// @tparam Out The output column type (must differ from every input). +/// @tparam In The input column types. +/// @param table The table. +/// @param op A callable op(const In&...) returning an Out value. +template +void assign_from(basic_soa_table& table, Op op) { + static_assert( + (!std::same_as && ...), "assign_from: the output column must differ from inputs." + ); + + std::vector> results; + for (auto row : table.template view()) { + results.emplace_back(row.id(), op(row.template get()...)); + } + for (auto& [id, value] : results) { + table.template assign(id, std::move(value)); + } +} + +} // namespace soatable::compute diff --git a/include/soatable/concurrent.hpp b/include/soatable/concurrent.hpp new file mode 100644 index 0000000..1e50653 --- /dev/null +++ b/include/soatable/concurrent.hpp @@ -0,0 +1,61 @@ +/// @file concurrent.hpp +/// @brief Opt-in thread-safe access to a table. +/// @author Bertin Balouki SIMYELI +/// +/// synchronized_table wraps any soa_table behind a std::shared_mutex, giving many concurrent +/// readers and exclusive writers via scoped callbacks: the lock is held for the duration of the +/// callback, so access is always synchronized. This matches the "many strategy threads read while +/// one thread ingests" use case. +/// +/// It is reader-writer synchronization, not lock-free MVCC or sharding; a snapshot/copy-on-write +/// read view and per-shard locking are natural future enhancements. References handed to a callback +/// must not escape it, since they are only valid while the lock is held. +#pragma once + +#include +#include +#include + +namespace soatable { + +/// @brief A thread-safe wrapper giving shared (reader) and exclusive (writer) access to a table. +/// @tparam Table The wrapped table type (e.g. soa_table<...>). +template +class synchronized_table { + public: + /// @brief The wrapped table type. + using table_type = Table; + + /// @brief Default-construct an empty wrapped table. + synchronized_table() = default; + + /// @brief Wrap an existing table by moving it in. + /// @param table The table to take ownership of. + explicit synchronized_table(Table table) : m_table(std::move(table)) {} + + /// @brief Run a callback with shared (read) access; many readers may proceed concurrently. + /// @tparam Func A callable taking const Table&. + /// @param func The callback; its return value is forwarded out. + /// @return Whatever @p func returns. + template + decltype(auto) read(Func&& func) const { + const std::shared_lock lock(m_mutex); + return std::forward(func)(m_table); + } + + /// @brief Run a callback with exclusive (write) access; no other reader or writer proceeds. + /// @tparam Func A callable taking Table&. + /// @param func The callback; its return value is forwarded out. + /// @return Whatever @p func returns. + template + decltype(auto) write(Func&& func) { + const std::unique_lock lock(m_mutex); + return std::forward(func)(m_table); + } + + private: + Table m_table; + mutable std::shared_mutex m_mutex; +}; + +} // namespace soatable diff --git a/include/soatable/dynamic.hpp b/include/soatable/dynamic.hpp new file mode 100644 index 0000000..6e71eeb --- /dev/null +++ b/include/soatable/dynamic.hpp @@ -0,0 +1,222 @@ +/// @file dynamic.hpp +/// @brief Opt-in runtime schema evolution: a type-erased table whose columns are added and removed +/// at runtime by name, with per-column string metadata. +/// @author Bertin Balouki SIMYELI +/// +/// It complements the statically-typed soa_table for schemas that are not known at compile time. +/// Columns are type-checked without RTTI (each type has a unique key address), so set/get +/// reject a type mismatch. Cells are stored sparsely per column, mirroring soa_table's "pay only +/// for columns a row has" model. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace soatable { + +namespace detail { +/// @brief A process-unique key for a type, used for type-erased checks without RTTI. +/// @tparam T The type. +/// @return A stable address unique to T. +template +inline const void* dynamic_type_key() noexcept { + static const char key = 0; + return &key; +} +} // namespace detail + +/// @brief A table whose columns can be added and removed at runtime, with per-column metadata. +class dynamic_table { + public: + /// @brief Row and column index type. + using size_type = std::size_t; + + /// @brief Add a typed column. + /// @tparam T The column value type. + /// @param name The column name (must be unique). + /// @return True if added, false if a column with that name already exists. + template + bool add_column(std::string name) { + if (m_columns.contains(name)) { + return false; + } + auto column = std::make_unique>(); + column->type_key = detail::dynamic_type_key(); + m_columns.emplace(std::move(name), std::move(column)); + return true; + } + + /// @brief Remove a column and all its cells. + /// @param name The column name. + /// @return True if a column was removed. + bool remove_column(const std::string& name) { return m_columns.erase(name) != 0; } + + /// @brief Whether a column with the given name exists. + [[nodiscard]] bool has_column(const std::string& name) const { + return m_columns.contains(name); + } + + /// @brief The number of columns. + [[nodiscard]] size_type column_count() const noexcept { return m_columns.size(); } + + /// @brief The names of all columns (unordered). + [[nodiscard]] std::vector column_names() const { + std::vector names; + names.reserve(m_columns.size()); + for (const auto& [name, column] : m_columns) { + static_cast(column); + names.push_back(name); + } + return names; + } + + /// @brief Insert a new alive row. + /// @return The new row index. + size_type insert_row() { + const size_type row = m_alive.size(); + m_alive.push_back(true); + ++m_alive_count; + return row; + } + + /// @brief Erase a row, dropping its cells from every column. + /// @param row The row index. + void erase_row(size_type row) { + if (row >= m_alive.size() || !m_alive[row]) { + return; + } + for (auto& [name, column] : m_columns) { + static_cast(name); + column->erase_row(row); + } + m_alive[row] = false; + --m_alive_count; + } + + /// @brief Whether a row index is alive. + [[nodiscard]] bool is_alive(size_type row) const { + return row < m_alive.size() && m_alive[row]; + } + + /// @brief The number of alive rows. + [[nodiscard]] size_type size() const noexcept { return m_alive_count; } + + /// @brief The number of row slots ever allocated. + [[nodiscard]] size_type row_slots() const noexcept { return m_alive.size(); } + + /// @brief Set a cell value. + /// @tparam T The column value type (must match the column's type). + /// @param row The row index. + /// @param name The column name. + /// @param value The value to store. + /// @throws std::out_of_range If the column or row is invalid. + /// @throws std::invalid_argument If T does not match the column's type. + template + void set(size_type row, const std::string& name, T value) { + const auto it = m_columns.find(name); + if (it == m_columns.end()) { + throw std::out_of_range("dynamic_table: unknown column '" + name + "'"); + } + if (it->second->type_key != detail::dynamic_type_key()) { + throw std::invalid_argument("dynamic_table: type mismatch for column '" + name + "'"); + } + if (row >= m_alive.size() || !m_alive[row]) { + throw std::out_of_range("dynamic_table: invalid row"); + } + static_cast*>(it->second.get())->cells[row] = std::move(value); + } + + /// @brief Get a pointer to a cell value, or nullptr if absent or the type mismatches. + /// @tparam T The column value type. + /// @param row The row index. + /// @param name The column name. + /// @return Pointer to the value, or nullptr. + template + [[nodiscard]] T* get(size_type row, const std::string& name) { + auto* column = typed(name); + if (column == nullptr) { + return nullptr; + } + const auto cell = column->cells.find(row); + return cell == column->cells.end() ? nullptr : &cell->second; + } + + /// @brief Const overload of get(). + template + [[nodiscard]] const T* get(size_type row, const std::string& name) const { + const auto* column = typed(name); + if (column == nullptr) { + return nullptr; + } + const auto cell = column->cells.find(row); + return cell == column->cells.end() ? nullptr : &cell->second; + } + + /// @brief Attach a string metadata key/value to a column (e.g. a unit label). + /// @param column The column name. + /// @param key The metadata key. + /// @param value The metadata value. + /// @throws std::out_of_range If the column does not exist. + void set_metadata(const std::string& column, std::string key, std::string value) { + const auto it = m_columns.find(column); + if (it == m_columns.end()) { + throw std::out_of_range("dynamic_table: unknown column '" + column + "'"); + } + it->second->metadata[std::move(key)] = std::move(value); + } + + /// @brief Read a column's metadata value, or nullptr if the column or key is absent. + [[nodiscard]] const std::string* get_metadata( + const std::string& column, const std::string& key + ) const { + const auto it = m_columns.find(column); + if (it == m_columns.end()) { + return nullptr; + } + const auto entry = it->second->metadata.find(key); + return entry == it->second->metadata.end() ? nullptr : &entry->second; + } + + private: + struct column_base { + const void* type_key = nullptr; + std::unordered_map metadata; + virtual ~column_base() = default; + virtual void erase_row(size_type row) = 0; + }; + + template + struct typed_column : column_base { + std::unordered_map cells; + void erase_row(size_type row) override { cells.erase(row); } + }; + + template + typed_column* typed(const std::string& name) { + const auto it = m_columns.find(name); + if (it == m_columns.end() || it->second->type_key != detail::dynamic_type_key()) { + return nullptr; + } + return static_cast*>(it->second.get()); + } + + template + const typed_column* typed(const std::string& name) const { + const auto it = m_columns.find(name); + if (it == m_columns.end() || it->second->type_key != detail::dynamic_type_key()) { + return nullptr; + } + return static_cast*>(it->second.get()); + } + + std::unordered_map> m_columns; + std::vector m_alive; + size_type m_alive_count = 0; +}; + +} // namespace soatable diff --git a/include/soatable/mmap.hpp b/include/soatable/mmap.hpp new file mode 100644 index 0000000..5f3ba3c --- /dev/null +++ b/include/soatable/mmap.hpp @@ -0,0 +1,107 @@ +/// @file mmap.hpp +/// @brief Opt-in memory-mapped column storage for very large, trivially-copyable columns. +/// @author Bertin Balouki SIMYELI +/// +/// mmap_allocator serves column storage from page-granular virtual memory (mmap on POSIX, +/// VirtualAlloc on Windows) that the OS pages in on demand, so a column can be far larger than the +/// resident set and only the touched pages occupy physical RAM. Mappings are page-aligned, so +/// columns stay SIMD-friendly. +/// +/// Composes with the Callocator support as mmap_soa_table. Reserve the column capacity up front +/// (table.reserve) so the backing region is mapped once; growth otherwise remaps and copies. +/// Backing is anonymous demand-paged virtual memory; persistent named-file mapping is a future +/// extension. +#pragma once + +#include +#include +#include + +#include "soatable/soatable.hpp" + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif + +namespace soatable { + +/// @brief A stateless allocator that backs storage with page-granular, demand-paged virtual memory. +/// @tparam T The element type. +template +struct mmap_allocator { + /// @brief The allocated value type. + using value_type = T; + + /// @brief Rebind helper for container node allocations. + template + struct rebind { + using other = mmap_allocator; + }; + + constexpr mmap_allocator() noexcept = default; + + /// @brief Converting constructor required by the Allocator concept. + template + constexpr mmap_allocator(const mmap_allocator&) noexcept {} + + /// @brief Map storage for @p count elements from demand-paged virtual memory. + /// @param count The number of elements. + /// @return Pointer to the page-aligned mapping. + [[nodiscard]] T* allocate(std::size_t count) { + if (count == 0) { + return nullptr; + } + if (count > std::numeric_limits::max() / sizeof(T)) { + throw std::bad_alloc(); + } + const std::size_t bytes = count * sizeof(T); +#if defined(_WIN32) + void* ptr = ::VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (ptr == nullptr) { + throw std::bad_alloc(); + } +#else + void* ptr = + ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + throw std::bad_alloc(); + } +#endif + return static_cast(ptr); + } + + /// @brief Release a mapping obtained from allocate(). + /// @param ptr The mapping base address. + /// @param count The element count passed to allocate(). + void deallocate(T* ptr, std::size_t count) noexcept { + if (ptr == nullptr) { + return; + } +#if defined(_WIN32) + static_cast(count); + ::VirtualFree(ptr, 0, MEM_RELEASE); +#else + ::munmap(ptr, count * sizeof(T)); +#endif + } + + template + constexpr bool operator==(const mmap_allocator&) const noexcept { + return true; + } +}; + +/// @brief A flat table whose columns are backed by demand-paged memory-mapped storage. +/// @tparam Columns The unique, trivially-copyable column types. +template +using mmap_soa_table = custom_soa_table; + +} // namespace soatable diff --git a/include/soatable/pmr.hpp b/include/soatable/pmr.hpp new file mode 100644 index 0000000..1baeed3 --- /dev/null +++ b/include/soatable/pmr.hpp @@ -0,0 +1,30 @@ +/// @file pmr.hpp +/// @brief Opt-in std::pmr (polymorphic memory resource) support for SoaTable. +/// @author Bertin Balouki SIMYELI +/// +/// Included separately so the core header does not pull in . A pmr_soa_table +/// allocates its column storage from the active std::pmr default memory resource, letting callers +/// route allocations to an arena, monotonic buffer, or pool resource for deterministic, +/// fragmentation-free behaviour. +#pragma once + +#include + +#include "soatable/soatable.hpp" + +namespace soatable { + +/// @brief A single-argument allocator template wrapping std::pmr::polymorphic_allocator. +/// @tparam T The element type. +template +using pmr_allocator = std::pmr::polymorphic_allocator; + +/// @brief A flat table whose columns allocate from the active std::pmr default memory resource. +/// @tparam Columns The unique column types. +/// +/// Set the resource with std::pmr::set_default_resource() (or wrap construction accordingly) before +/// populating the table to route its allocations through an arena / monotonic buffer / pool. +template +using pmr_soa_table = custom_soa_table; + +} // namespace soatable diff --git a/include/soatable/query.hpp b/include/soatable/query.hpp new file mode 100644 index 0000000..18eff0e --- /dev/null +++ b/include/soatable/query.hpp @@ -0,0 +1,107 @@ +/// @file query.hpp +/// @brief Opt-in query helpers layered on select<>() / view<>(): predicate filtering and group-by +/// aggregation. +/// @author Bertin Balouki SIMYELI +/// +/// These compose the existing smallest-driver scan with a predicate or a grouping pass, giving +/// analytics ergonomics without leaving C++. +#pragma once + +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable::query { + +/// @brief Select rows that have all of Cols and additionally satisfy a row predicate. +/// @tparam Cols The required column types (non-optional). +/// @param table The table to query. +/// @param predicate A callable taking a row_view and returning bool. +/// @return A range of row_view proxies for the matching rows (structured-binding friendly). +/// @note Lazy: it composes view() with std::views::filter. The table must outlive +/// iteration. +template +[[nodiscard]] auto select_where(basic_soa_table& table, Pred predicate) { + return table.template view() | + std::views::filter([predicate](auto row) { return predicate(row); }); +} + +/// @brief Group rows by a key derived from column Key and fold column Value within each group. +/// @tparam Key The key column type. +/// @tparam Value The value column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable, equality-comparable group key. +/// @param init The per-group starting accumulator value. +/// @param op A callable op(accumulator, const Value&) returning the new accumulator. +/// @return A map from group key to accumulated value, over rows that have both Key and Value. +template < + typename Key, + typename Value, + typename Storage, + typename... Columns, + typename KeyProj, + typename U, + typename Op> +[[nodiscard]] auto group_reduce( + const basic_soa_table& table, KeyProj key_of, U init, Op op +) { + using key_type = std::decay_t>; + std::unordered_map groups; + for (auto [id, key, value] : table.template select()) { + static_cast(id); + const key_type group_key = key_of(key.get()); + auto it = groups.find(group_key); + if (it == groups.end()) { + it = groups.emplace(group_key, init).first; + } + it->second = op(std::move(it->second), value.get()); + } + return groups; +} + +/// @brief Group rows by a key and sum a projection of column Value within each group. +/// @tparam Key The key column type. +/// @tparam Value The value column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable group key. +/// @param value_of A callable mapping const Value& to a summable number. +/// @return A map from group key to the summed projection. +template < + typename Key, + typename Value, + typename Storage, + typename... Columns, + typename KeyProj, + typename ValueProj> +[[nodiscard]] auto group_sum( + const basic_soa_table& table, KeyProj key_of, ValueProj value_of +) { + using sum_type = std::decay_t>; + return group_reduce( + table, key_of, sum_type {}, [value_of](sum_type accumulator, const Value& value) { + return accumulator + value_of(value); + } + ); +} + +/// @brief Count the rows in each group keyed by a projection of column Key. +/// @tparam Key The key column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable group key. +/// @return A map from group key to the number of rows that have Key in that group. +template +[[nodiscard]] auto group_count(const basic_soa_table& table, KeyProj key_of) { + using key_type = std::decay_t>; + std::unordered_map counts; + for (auto [id, key] : table.template select()) { + static_cast(id); + ++counts[key_of(key.get())]; + } + return counts; +} + +} // namespace soatable::query diff --git a/include/soatable/reflect.hpp b/include/soatable/reflect.hpp new file mode 100644 index 0000000..9ec7099 --- /dev/null +++ b/include/soatable/reflect.hpp @@ -0,0 +1,50 @@ +/// @file reflect.hpp +/// @brief Build a table directly from a plain struct. +/// @author Bertin Balouki SIMYELI +/// +/// C++26 static reflection (P2996) is not yet available in shipping toolchains, so this header +/// provides a portable fallback that works today and an automatic path that activates once a +/// compiler implements reflection: +/// +/// * Portable (now): specialize columns_of (via column_list) to list the struct's column +/// types, then use table_for. * Automatic (future): when SOATABLE_HAS_REFLECTION is 1, +/// columns_of is derived from the struct's members with no manual specialization. +#pragma once + +#include "soatable/soatable.hpp" + +#if defined(__cpp_reflection) && __cpp_reflection >= 202600L +#define SOATABLE_HAS_REFLECTION 1 +#else +#define SOATABLE_HAS_REFLECTION 0 +#endif + +namespace soatable { + +/// @brief A list of column types, exposing the corresponding soa_table type. +/// @tparam Columns The column types. +template +struct column_list { + /// @brief The table type holding these columns. + using table_type = soa_table; +}; + +/// @brief Maps a user struct to its column list. Specialize this (deriving from column_list) to use +/// table_for without reflection; the specialization becomes unnecessary once reflection +/// ships. +/// @tparam Struct The user struct identifying a schema. +template +struct columns_of; + +#if SOATABLE_HAS_REFLECTION +// When a toolchain implements C++26 reflection, a default definition of columns_of that +// enumerates the struct's data members into columns is provided here, making the manual +// specialization above optional. (Intentionally left for the reflection-capable build.) +#endif + +/// @brief The soa_table type for a struct, via its columns_of mapping. +/// @tparam Struct The user struct identifying a schema. +template +using table_for = typename columns_of::table_type; + +} // namespace soatable diff --git a/include/soatable/serialize.hpp b/include/soatable/serialize.hpp new file mode 100644 index 0000000..eb9026f --- /dev/null +++ b/include/soatable/serialize.hpp @@ -0,0 +1,318 @@ +/// @file serialize.hpp +/// @brief Opt-in binary serialization for soa_table. +/// @author Bertin Balouki SIMYELI +/// +/// Included separately so the core header pays nothing for it. Supports the trivially-copyable fast +/// path: each column's dense values are dumped/loaded as raw bytes, wrapped in a versioned, +/// schema-checked header that round-trips the full table state (handles, generations, free list, +/// and physical column order). +/// +/// Portability: the format is self-describing and versioned, but values are written in the host's +/// native byte order and ABI. It is intended for snapshots consumed by the same architecture (tick +/// stores, simulation checkpoints). Cross-endian / cross-ABI interchange is a future enhancement +/// and is the natural place for an Arrow bridge, built on column() spans and +/// validity().words(). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable { + +/// @brief Outcome of a load() call. +enum class serialize_status { + ok, + bad_magic, + version_mismatch, + schema_mismatch, + truncated, +}; + +namespace detail { + +/// @brief Friend accessor exposing a table's private state to the serializer only. +/// @tparam Storage The table's storage policy. +/// @tparam Columns The table's column types. +template +struct table_access { + using table_type = basic_soa_table; + + static auto& rows(table_type& t) noexcept { return t.m_rows; } + static const auto& rows(const table_type& t) noexcept { return t.m_rows; } + static auto& free_links(table_type& t) noexcept { return t.m_free_links; } + static const auto& free_links(const table_type& t) noexcept { return t.m_free_links; } + static std::uint32_t& free_head(table_type& t) noexcept { return t.m_free_head; } + static std::uint32_t free_head(const table_type& t) noexcept { return t.m_free_head; } + static std::size_t& alive_count(table_type& t) noexcept { return t.m_alive_count; } + static std::size_t alive_count(const table_type& t) noexcept { return t.m_alive_count; } + static auto& columns(table_type& t) noexcept { return t.m_columns; } + static const auto& columns(const table_type& t) noexcept { return t.m_columns; } +}; + +inline constexpr std::array serialize_magic = { + std::byte {'S'}, + std::byte {'O'}, + std::byte {'A'}, + std::byte {'T'}, + std::byte {'B'}, + std::byte {'L'}, + std::byte {'E'}, + std::byte {'\0'} +}; +inline constexpr std::uint32_t serialize_version = 1; + +/// @brief FNV-1a hash of the column layout so load() can reject a mismatched schema. +template +constexpr std::uint64_t schema_fingerprint() { + std::uint64_t hash = 1469598103934665603ULL; + const auto mix = [&hash](std::uint64_t value) { + hash ^= value; + hash *= 1099511628211ULL; + }; + mix(sizeof...(Columns)); + (mix(sizeof(Columns)), ...); + (mix(alignof(Columns)), ...); + return hash; +} + +/// @brief Append a trivially-copyable value to a byte buffer. +template +void append_pod(std::vector& out, const T& value) { + // Grow then memcpy rather than insert() over a reinterpreted byte range: the latter trips a + // GCC -Wstringop-overflow false positive when it cannot prove the source object's size. + const std::size_t offset = out.size(); + out.resize(offset + sizeof(T)); + std::memcpy(out.data() + offset, &value, sizeof(T)); +} + +/// @brief Append raw bytes of a contiguous range of trivially-copyable values. +template +void append_array(std::vector& out, std::span values) { + // An empty column yields a null data() pointer; skip the copy rather than form pointer + // arithmetic on null. + if (values.empty()) { + return; + } + const auto* bytes = reinterpret_cast(values.data()); + out.insert(out.end(), bytes, bytes + values.size_bytes()); +} + +/// @brief Sequential little-state reader over a byte span with bounds checking. +struct byte_reader { + std::span data; + std::size_t offset = 0; + bool ok = true; + + template + T read_pod() { + T value {}; + if (!ok || offset + sizeof(T) > data.size()) { + ok = false; + return value; + } + std::memcpy(&value, data.data() + offset, sizeof(T)); + offset += sizeof(T); + return value; + } + + template + void read_into(T* dst, std::size_t count) { + const std::size_t bytes = count * sizeof(T); + if (!ok || offset + bytes > data.size()) { + ok = false; + return; + } + // dst is null for an empty destination vector; a zero-length memcpy on a null pointer is + // undefined behavior, so make it a no-op. + if (bytes == 0) { + return; + } + std::memcpy(dst, data.data() + offset, bytes); + offset += bytes; + } +}; + +} // namespace detail + +/// @brief Serialize a table to a self-describing byte buffer (trivially-copyable columns only). +/// @tparam Columns The table's column types. +/// @param table The table to serialize. +/// @return A byte buffer that load() can reconstruct the table from. +template +[[nodiscard]] std::vector save(const basic_soa_table& table) { + static_assert( + (std::is_trivially_copyable_v && ...), + "save() requires every column type to be trivially copyable." + ); + static_assert( + basic_soa_table::column_count <= 64, + "save() supports up to 64 columns (the signature is stored as a 64-bit word)." + ); + + using access = detail::table_access; + std::vector out; + + const auto& rows = access::rows(table); + const auto& free_links = access::free_links(table); + + out.insert(out.end(), detail::serialize_magic.begin(), detail::serialize_magic.end()); + detail::append_pod(out, detail::serialize_version); + detail::append_pod(out, detail::schema_fingerprint()); + detail::append_pod(out, static_cast(rows.size())); + detail::append_pod(out, access::free_head(table)); + detail::append_pod(out, static_cast(access::alive_count(table))); + + for (const auto& meta : rows) { + detail::append_pod(out, meta.generation); + detail::append_pod(out, static_cast(meta.signature.to_ullong())); + detail::append_pod(out, static_cast(meta.alive ? 1 : 0)); + } + detail::append_array(out, std::span(free_links.data(), free_links.size())); + + // Each column: its dense row indices followed by its raw values, in declaration order. + std::apply( + [&out](const auto&... pool) { + ( + [&out](const auto& column_pool) { + const auto& dense = column_pool.dense_rows(); + const auto& data = column_pool.raw_data(); + detail::append_pod(out, static_cast(dense.size())); + detail::append_array( + out, std::span(dense.data(), dense.size()) + ); + using pool_type = std::decay_t; + if constexpr (pool_type::is_contiguous) { + using value_type = std::decay_t; + detail::append_array( + out, std::span(data.data(), data.size()) + ); + } else { + // Tiled storage is not contiguous; write each value individually. + for (std::size_t i = 0; i < data.size(); ++i) { + detail::append_pod(out, data[i]); + } + } + }(pool), + ...); + }, + access::columns(table) + ); + + return out; +} + +/// @brief Reconstruct a table from a buffer produced by save(). +/// @tparam Columns The table's column types (must match the schema that was saved). +/// @param table The table to overwrite (cleared first). +/// @param bytes The serialized buffer. +/// @return serialize_status::ok on success, or the reason the buffer was rejected. +template +[[nodiscard]] serialize_status load( + basic_soa_table& table, std::span bytes +) { + static_assert( + (std::is_trivially_copyable_v && ...), + "load() requires every column type to be trivially copyable." + ); + static_assert( + basic_soa_table::column_count <= 64, + "load() supports up to 64 columns (the signature is stored as a 64-bit word)." + ); + + using access = detail::table_access; + + detail::byte_reader reader {bytes}; + + std::array magic {}; + reader.read_into(magic.data(), magic.size()); + if (!reader.ok) { + return serialize_status::truncated; + } + if (magic != detail::serialize_magic) { + return serialize_status::bad_magic; + } + if (reader.read_pod() != detail::serialize_version) { + return serialize_status::version_mismatch; + } + if (reader.read_pod() != detail::schema_fingerprint()) { + return serialize_status::schema_mismatch; + } + + const auto row_slots = static_cast(reader.read_pod()); + const auto free_head = reader.read_pod(); + const auto alive_count = static_cast(reader.read_pod()); + if (!reader.ok) { + return serialize_status::truncated; + } + + table.clear(); + + auto& rows = access::rows(table); + rows.resize(row_slots); + for (auto& meta : rows) { + meta.generation = reader.read_pod(); + const auto sig = reader.read_pod(); + meta.signature = typename basic_soa_table::signature_type {sig}; + meta.alive = reader.read_pod() != 0; + } + + auto& free_links = access::free_links(table); + free_links.resize(row_slots); + reader.read_into(free_links.data(), row_slots); + + access::free_head(table) = free_head; + access::alive_count(table) = alive_count; + if (!reader.ok) { + table.clear(); + return serialize_status::truncated; + } + + // Replay each column's dense entries through emplace, rebuilding the sparse index. + bool columns_ok = true; + std::apply( + [&](auto&... pool) { + ( + [&](auto& column_pool) { + if (!columns_ok) { + return; + } + using value_type = std::decay_t; + column_pool.clear(); + const auto count = static_cast(reader.read_pod()); + if (!reader.ok) { + columns_ok = false; + return; + } + std::vector dense(count); + reader.read_into(dense.data(), count); + std::vector data(count); + reader.read_into(data.data(), count); + if (!reader.ok) { + columns_ok = false; + return; + } + for (std::size_t i = 0; i < count; ++i) { + column_pool.emplace(dense[i], data[i]); + } + }(pool), + ...); + }, + access::columns(table) + ); + + if (!columns_ok) { + table.clear(); + return serialize_status::truncated; + } + return serialize_status::ok; +} + +} // namespace soatable diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 9180af3..9d37067 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -1,6 +1,18 @@ +/// @file soatable.hpp +/// @brief Core header for SoaTable: a header-only C++23 sparse Structure-of-Arrays table. +/// @author Bertin Balouki SIMYELI +/// +/// Defines the foundational types: generational `row_id` handles, the sparse-to-dense +/// `column_vector` backing each column, the `basic_soa_table` container (aliased as `soa_table`, +/// `aosoa_table`, `chunked_soa_table`, `custom_soa_table`) with its `select`/`view` joins, +/// zero-copy `column()` spans, validity `bitmap`s, and the `quantized_float` / `packed_bits` / +/// `delta_value` / `dirty_mask` helpers. The compute, query, serialization, allocator, concurrency, +/// out-of-core, time-series, units, reflection, and dynamic-schema add-ons live in their own opt-in +/// headers and build on this one. #pragma once #include +#include #include #include #include @@ -9,11 +21,13 @@ #include #include #include -#include #include +#include #include #include +#include #include +#include #include #include #include @@ -26,14 +40,49 @@ #define SOATABLE_HAS_PRINT 0 #endif -/// @brief The main namespace for the SoaTable library. +// std::expected (C++23) is an optional enhancement: where the standard library does not provide it +// the feature-detected get_expected() accessors are simply not declared, and try_get() remains +// the always-available non-throwing accessor. Define SOATABLE_DISABLE_EXPECTED to force the +// fallback even on a capable toolchain. +#if __has_include() +#include +#endif +#if !defined(SOATABLE_DISABLE_EXPECTED) && defined(__cpp_lib_expected) && __cpp_lib_expected >= 202202L +#define SOATABLE_HAS_EXPECTED 1 +#else +#define SOATABLE_HAS_EXPECTED 0 +#endif + +// Freestanding / no-exceptions mode. Define SOATABLE_NO_EXCEPTIONS to compile without throwing: +// every internal `throw` is redirected to a terminating handler, and callers use the non-throwing +// get_expected() / try_get() accessors for error handling. The library uses no RTTI either. +#ifdef SOATABLE_NO_EXCEPTIONS +#include +#include +namespace soatable::detail { +/// @brief Terminating handler invoked in place of a throw under SOATABLE_NO_EXCEPTIONS. +/// @param message A human-readable description of the failed precondition. +[[noreturn]] inline void terminate_with(const char* message) noexcept { + std::fputs("soatable fatal: ", stderr); + std::fputs(message, stderr); + std::fputc('\n', stderr); + std::abort(); +} +} // namespace soatable::detail +#define SOATABLE_THROW(exception_type, message) ::soatable::detail::terminate_with(message) +#else +/// @brief Throw @p exception_type with @p message, or terminate under SOATABLE_NO_EXCEPTIONS. +#define SOATABLE_THROW(exception_type, message) throw exception_type(message) +#endif + +/// @brief The main namespace for the soa_table library. namespace soatable { -// ========================================= -// 1. Stable row identity and type helpers -// ========================================= +// ==================================== +// Stable row identity and type helpers +// ==================================== -/// @brief A stable identifier for a row in an SoaTable. +/// @brief A stable identifier for a row in an soa_table. /// /// Contains an index into the row metadata and a generation count to prevent /// the ABA problem (referencing a new row with an old handle). @@ -49,6 +98,14 @@ struct row_id { constexpr bool operator==(const row_id& other) const = default; }; +/// @brief Failure categories returned by the non-throwing std::expected accessors. +enum class access_error { + /// @brief The row_id is stale or out of range. + invalid_row, + /// @brief The row is valid but does not carry the requested column. + missing_column, +}; + /// @brief Helper concept to check if a type is present in a parameter pack. /// @tparam T The type to search for. /// @tparam Types The parameter pack to search in. @@ -88,7 +145,7 @@ struct index_of; template struct index_of> { static_assert( - sizeof...(Types) > 0, "SoaTable must contain at least one registered column type." + sizeof...(Types) > 0, "soa_table must contain at least one registered column type." ); /// @brief Internal function to find the index of type T in the pack. @@ -106,13 +163,179 @@ struct index_of> { /// @brief The compile-time index of type T in Types.... static constexpr std::size_t value = value_of(); static_assert( - value < sizeof...(Types), "The requested type is not registered inside the SoaTable." + value < sizeof...(Types), "The requested type is not registered inside the soa_table." ); }; +/// @brief Default over-alignment (in bytes) for dense column storage. +/// +/// Columns are over-aligned to this boundary (matching Apache Arrow's 64-byte recommendation) so +/// the spans returned by soa_table::column() are ready for aligned SIMD loads and cache-line +/// friendly. +inline constexpr std::size_t simd_alignment = 64; + +/// @brief The alignment used for a column of T: the larger of T's natural alignment and +/// simd_alignment. +/// @tparam T The column value type. +template +inline constexpr std::size_t column_alignment = + (alignof(T) > simd_alignment) ? alignof(T) : simd_alignment; + /// @brief Detail namespace for internal implementation details. namespace detail { +/// @brief Friend accessor granting the opt-in header access to the private +/// state of a basic_soa_table without widening its public surface. Defined only in that header. +/// @tparam Storage The table's storage policy. +/// @tparam Columns The table's column types. +template +struct table_access; + +/// @brief A standard-conforming allocator that over-aligns its allocations to Alignment bytes. +/// @tparam T The allocated value type. +/// @tparam Alignment The required alignment in bytes (defaults to column_alignment). +template > +struct aligned_allocator { + static_assert(Alignment >= alignof(T), "Alignment must be at least alignof(T)."); + + /// @brief The allocated value type. + using value_type = T; + /// @brief The over-alignment in bytes. + static constexpr std::size_t alignment = Alignment; + + /// @brief Rebind helper so container nodes can be allocated with the same alignment policy. + template + struct rebind { + using other = aligned_allocator>; + }; + + constexpr aligned_allocator() noexcept = default; + + /// @brief Converting constructor required by the Allocator concept. + template + constexpr aligned_allocator(const aligned_allocator&) noexcept {} + + /// @brief Allocate n over-aligned elements. + /// @param count The number of elements. + /// @return Pointer to the aligned storage. + [[nodiscard]] T* allocate(std::size_t count) { + if (count > std::numeric_limits::max() / sizeof(T)) { +#ifdef SOATABLE_NO_EXCEPTIONS + terminate_with("aligned_allocator: allocation size overflow."); +#else + throw std::bad_alloc(); +#endif + } + void* ptr = ::operator new(count * sizeof(T), std::align_val_t {Alignment}); + return static_cast(ptr); + } + + /// @brief Release storage obtained from allocate(). + /// @param ptr The pointer to release. + void deallocate(T* ptr, std::size_t) noexcept { + ::operator delete(ptr, std::align_val_t {Alignment}); + } + + template + constexpr bool operator==(const aligned_allocator&) const noexcept { + return Alignment == OtherAlignment; + } +}; + +/// @brief A vector whose storage is over-aligned to column_alignment for SIMD-friendly columns. +/// @tparam T The element type. +template +using aligned_vector = std::vector>; + +/// @brief A tiled (AoSoA-style) dense container: a sequence of fixed-size, individually +/// over-aligned tiles. Growth appends a tile and never copies existing tiles, bounding reallocation +/// cost. +/// @tparam T The element type. +/// @tparam TileSize The number of elements per tile. +/// +/// Exposes the subset of the std::vector interface that column_vector relies on, so it is a drop-in +/// alternative storage backend. Elements are not contiguous across tiles, so it is non-contiguous. +template +class tiled_vector { + static_assert(TileSize > 0, "tiled_vector requires a positive tile size."); + + public: + /// @brief Number of stored elements. + [[nodiscard]] std::size_t size() const noexcept { return m_size; } + /// @brief Whether the container is empty. + [[nodiscard]] bool empty() const noexcept { return m_size == 0; } + + /// @brief Mutable element access by flat index. + [[nodiscard]] T& operator[](std::size_t index) { + return m_tiles[index / TileSize][index % TileSize]; + } + /// @brief Const element access by flat index. + [[nodiscard]] const T& operator[](std::size_t index) const { + return m_tiles[index / TileSize][index % TileSize]; + } + + /// @brief Construct an element at the end, allocating a new tile when the last one is full. + template + T& emplace_back(Args&&... args) { + if (m_tiles.empty() || m_tiles.back().size() == TileSize) { + m_tiles.emplace_back(); + m_tiles.back().reserve(TileSize); + } + T& ref = m_tiles.back().emplace_back(std::forward(args)...); + ++m_size; + return ref; + } + + /// @brief Append a value at the end. + void push_back(const T& value) { emplace_back(value); } + /// @brief Append a value at the end (move). + void push_back(T&& value) { emplace_back(std::move(value)); } + + /// @brief The last element. + [[nodiscard]] T& back() { return m_tiles.back().back(); } + + /// @brief Remove the last element, dropping the trailing tile when it becomes empty. + void pop_back() { + m_tiles.back().pop_back(); + if (m_tiles.back().empty()) { + m_tiles.pop_back(); + } + --m_size; + } + + /// @brief Reserve capacity for at least the given number of elements (rounded up to tiles). + void reserve(std::size_t capacity) { m_tiles.reserve((capacity + TileSize - 1) / TileSize); } + + /// @brief Remove all elements. + void clear() noexcept { + m_tiles.clear(); + m_size = 0; + } + + /// @brief Release unused capacity. + void shrink_to_fit() { + for (auto& tile_data : m_tiles) { + tile_data.shrink_to_fit(); + } + m_tiles.shrink_to_fit(); + } + + /// @brief Number of tiles currently allocated. + [[nodiscard]] std::size_t tile_count() const noexcept { return m_tiles.size(); } + /// @brief A contiguous, over-aligned span over one tile's live elements. + [[nodiscard]] std::span tile(std::size_t tile_index) { + return std::span(m_tiles[tile_index].data(), m_tiles[tile_index].size()); + } + /// @brief A contiguous, over-aligned const span over one tile's live elements. + [[nodiscard]] std::span tile(std::size_t tile_index) const { + return std::span(m_tiles[tile_index].data(), m_tiles[tile_index].size()); + } + + private: + std::vector> m_tiles; + std::size_t m_size = 0; +}; + /// @brief Generates a mask with the specified number of low bits set. /// @tparam UInt The unsigned integer type. /// @param bits The number of bits to set. @@ -167,7 +390,7 @@ template using optional_value_type_t = typename optional_value_type::type; /// @brief The return type of a selection query. -/// @tparam Self The SoaTable type (const or non-const). +/// @tparam Self The soa_table type (const or non-const). /// @tparam ReqColumns The requested column types. template using select_result_t = std::tuple< @@ -185,9 +408,51 @@ using select_result_t = std::tuple< } // namespace detail -// ================================= -// 2. Compression and state helpers -// ================================= +// ======================= +// Column storage policies +// ======================= + +/// @brief Storage policy: a flat, contiguous column allocated with a caller-chosen allocator. +/// @tparam Allocator A single-argument allocator template applied per element type +/// (e.g. std::pmr::polymorphic_allocator). Compose with custom_soa_table. +template