diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 0000000..f638f64 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,19 @@ +# `cargo nextest run` runs each test in its own process and every binary at once. +[profile.default] +# A test running past 5 s is reported as slow. One still running at 30 s is stopped and fails. +slow-timeout = { period = "5s", terminate-after = 6 } +fail-fast = false + +# Timings run last, with nothing beside them: threaded correctness tests such as `linalg` would slow them. +[[profile.default.overrides]] +filter = 'binary(speed)' +threads-required = 'num-cpus' +priority = -100 + +# Optional selection excluding timings, used by focused cross-repo diagnostics beside basedpl's untimed selection. +[profile.untimed] +default-filter = 'not binary(speed)' + +# Optional timing-only selection. An explicit cross-repo ('untimed', 'timed', 'speed') run uses this after basedpl timings. Plain run_tests uses default and includes everything. +[profile.speed] +default-filter = 'binary(speed)' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8a4c50..c760b6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,10 +49,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - id: auth uses: rust-lang/crates-io-auth-action@v1 - - run: cargo publish -p rustymath --no-verify - env: - CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} - - run: cargo publish -p rustymath-buffer --no-verify + - run: cargo publish --workspace --no-verify env: CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} - uses: actions/download-artifact@v8 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a13ee6..af449f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@nextest # The repository has no `Cargo.lock`. Creating one first makes rust-cache's key change when a dependency publishes a new version. - run: cargo generate-lockfile - uses: Swatinem/rust-cache@v2 @@ -25,8 +26,8 @@ jobs: python-version: '3.12' - run: pip install -e '.[dev]' - run: cargo develop - - run: cargo test --release - - run: pytest -q + - run: cargo nextest run --cargo-profile release -E 'not binary(speed)' + - run: pytest -q --ignore=tests/test_speed.py if: ${{ !cancelled() }} - run: nbdev-test if: ${{ !cancelled() }} diff --git a/Cargo.toml b/Cargo.toml index d7c3421..569e094 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,32 @@ [workspace] -members = ["buffer", "py"] +members = ["core", "kernels", "timing", "buffer", "py"] default-members = [".", "buffer", "py"] [workspace.package] -version = "0.1.56" +version = "0.1.57" edition = "2024" [workspace.dependencies] -rustymath = { path = ".", version = ">=0.1.55, <1", default-features = false } +rustymath = { path = ".", version = ">=0.1.56, <1", default-features = false } rustymath-buffer = { path = "buffer", version = ">=0.1.39, <1" } pyo3 = ">=0.29" pyo3-build-config = ">=0.29" +rustymath-core = { path = "core", version = ">=0.1.56, <1", default-features = false } +rustymath-kernels = { path = "kernels", version = ">=0.1.56, <1" } +rustymath-timing = { path = "timing", version = ">=0.1.56, <1" } +half = { version = ">=2.7.1, <3", features = ["num-traits"] } +hashbrown = ">=0.15, <1" +foldhash = ">=0.2.0, <1" +num-bigint = ">=0.4.6, <0.5" +num-complex = ">=0.4.6, <1" +num-integer = ">=0.1.46, <1" +num-rational = ">=0.4.2, <1" +num-traits = ">=0.2.19, <1" +rand = { version = ">=0.10.2, <1", features = ["chacha"] } +strength_reduce = ">=0.2.4, <1" +rayon = ">=1.12.0, <2" +web-time = ">=1.1.0, <2" +libc = { version = ">=0.2.190, <1", features = ["extra_traits"] } [workspace.metadata.fastfmt] join_comments = true @@ -29,40 +45,28 @@ name = "rustymath" test = false doctest = false -# The timing suite: plain `cargo test` skips it; `cargo test --test speed` runs it. -[[test]] -name = "speed" -test = false - [features] default = ["native"] # Compiles rustymath's own linear algebra into macOS builds too, beside Accelerate, so `RUSTYMATH_NATIVE=1` can select it at run time. # Other targets always compile it. The Mac release builds turn it off with `--no-default-features`. -native = [] +native = ["rustymath-core/native"] [dependencies] -half = { version = ">=2.7.1, <3", features = ["num-traits"] } -hashbrown = ">=0.15, <1" -foldhash = ">=0.2.0, <1" -num-bigint = ">=0.4.6, <0.5" -num-complex = ">=0.4.6, <1" -num-integer = ">=0.1.46, <1" -num-rational = ">=0.4.2, <1" -num-traits = ">=0.2.19, <1" -rand = { version = ">=0.10.2, <1", default-features = false, features = ["alloc", "chacha"] } -strength_reduce = ">=0.2.4, <1" - -# The browser build has no threads. There `parallel` runs its pieces in turn on the calling thread. -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -rayon = ">=1.12.0, <2" +rustymath-core.workspace = true +rustymath-kernels.workspace = true +num-complex.workspace = true +num-traits.workspace = true -# The browser's clock, for the timing loop in `timing`, as basedpl's browser build uses it. -[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] -web-time = ">=1.1.0, <2" - -[target.'cfg(unix)'.dependencies] -# The memory guard calls `libc` to read physical memory and, on Linux, to ask for huge pages. On macOS, the timing lock also makes its QoS call through it, to favour the performance cores for a timing. -libc = ">=0.2.190, <1" +[dev-dependencies] +rustymath-timing.workspace = true +half.workspace = true +hashbrown.workspace = true +foldhash.workspace = true +num-bigint.workspace = true +num-integer.workspace = true +num-rational.workspace = true +rand.workspace = true +strength_reduce.workspace = true [profile.test] inherits = "release" @@ -77,6 +81,10 @@ incremental = true # Line tables change with every shifted line and recompile the code below it. debug = false +[profile.quick] +inherits = "release" +lto = "off" + [profile.dist] inherits = "release" lto = "thin" diff --git a/DEV.md b/DEV.md index aa53148..4eab798 100644 --- a/DEV.md +++ b/DEV.md @@ -16,7 +16,7 @@ Development and planning run on Syd1 under `/home/ubuntu/aai-ws/{basedpl,rustyma ```bash cargo develop -cargo test --release +cargo nextest run --cargo-profile release pytest -q RUSTYMATH_COSTS=1 cargo test --release --test speed @@ -25,11 +25,11 @@ nbdev-test nbdev-preview ``` -Use `cargo develop` to rebuild and install the Python extension after Rust changes. It builds the `release` profile, which development, tests and timings all use; `fastws.cargo.run_tests(root, ...)` builds it too. Bare `cargo test` uses Cargo's built-in `test` profile, which has `release`'s settings but builds again; pass `--release`. Cargo tests compile missing code but do not install it. An already-running Python process retains its loaded extension. +Use `cargo develop` to rebuild and install the Python extension after Rust changes. It builds the `release` profile, which development, tests and timings all use; `fastws.cargo.run_tests(root, ...)` builds it too. Bare `cargo test` uses Cargo's built-in `test` profile, which has `release`'s settings but builds again; pass `--release`. Cargo tests compile missing code but do not install it. An already-running Python process retains its loaded extension. `cargo nextest run --cargo-profile release` runs each test in its own process and every test binary at once. `.config/nextest.toml` reports a test as slow at 5 s, stops it at 30 s, and runs `speed` last with nothing beside it. Its `untimed` and `speed` profiles split the speed test from the rest. basedpl's default test command, `fastws.cargo.run_tests([basedpl, rustymath], profiles=('untimed', 'timed', 'speed'))`, runs each profile in turn, at once in each repository that defines it, so `speed` runs after basedpl's timings, with nothing beside it. -The timing suites run only by name: `cargo test --release --test speed` (`test = false` in `Cargo.toml`) and `pytest -q tests/test_speed.py` (`addopts` skips it otherwise). Use one test binary for a focused check. Run `pytest -q` at the end of a piece of work. Before reporting a failure in untouched code, rebuild with the same profile and rerun it. Builds have no execution timeout. Use the documented managed runners with a 30-second test execution budget. Cargo serializes builds in the same target directory. +`cargo nextest run --cargo-profile release` and `pytest -q` include the speed tests, `tests/speed.rs` and `tests/test_speed.py`. Their ordinary runs time each case beside builds and other tests, under a shared hold on the timing lock and on one thread. They also check each case's recorded counts of instructions and allocated bytes for each item, which don't depend on what else the machine runs. A case over its budget is timed again under an exclusive hold where no one else holds the lock, and is reported `unconfirmed`, without failing, where someone does. `RUSTYMATH_COSTS=1` makes either test a calibration run, as "Speed measurements" describes. Use one test binary for a focused check. Run `pytest -q` at the end of a piece of work. Before reporting a failure in untouched code, rebuild with the same profile and rerun it. Builds have no execution timeout. Use the documented managed runners with a 30-second test execution budget. Cargo serializes builds in the same target directory. -Before a stage's closing review and before a release, run the full checks, as CI runs them: `cargo develop`, `cargo test --release`, `pytest -q`, `nbdev-test`, then both timing suites with `RUSTYMATH_COSTS=1`. Then run `bpl-tests target_builds /tmp/_wasm` in basedpl. It builds this crate for Linux x86-64 and wasm32 as basedpl links it, with basedpl's wasm32 flags. basedpl's DEV.md describes it beside basedpl's own full checks. +Before a stage's closing review and before a release, run the full checks, as CI runs them: `cargo develop`, `cargo nextest run --cargo-profile release`, `pytest -q`, `nbdev-test`, then both speed tests as calibration runs, with `RUSTYMATH_COSTS=1`. Then run `bpl-tests target_builds /tmp/_wasm` in basedpl. It builds this crate for Linux x86-64 and wasm32 as basedpl links it, with basedpl's wasm32 flags. basedpl's DEV.md describes it beside basedpl's own full checks. `cargo develop` keeps each build it installs, read-only and with its Python package, under an id that hashes the build's inputs and files (`fastws.store`). Another agent's `cargo develop` can install another build at any time, so before your change, note the installed build's id, from `fastws.store.builds(rms)`, as the baseline for your timings and tests. `basedpl.bpltests.pytest_costs` and `run_pytest` take it as `build=`, and put that build's Python package before the installed one on `PYTHONPATH`. Don't restore old code in the shared tree for a baseline. @@ -45,13 +45,15 @@ Rust 1.98 or later supplies algebraic float operations. The `test` profile inher `[tool.fastws] dev-profile = "release"` in `pyproject.toml` selects `release` for development and managed tests, so one build in `target/release/` serves development, tests and timings. It enables cross-crate generic sharing. The compiler flags, Cargo-scoped bootstrap and tested toolchain are documented in basedpl's DEV.md, "Codegen and rebuilds". +`cargo check --all-targets --profile quick` and `cargo develop --profile quick` select optional O3, incremental builds without LTO. They use a separate `target/quick/` cache. Development defaults and performance validation profiles are unchanged. See basedpl's "Codegen and rebuilds" for the shared procedure. + Keep these profiles aligned with basedpl's. Cargo uses the consuming workspace's profiles for dependencies. Concrete function forms compile in rustymath, but their build still uses that workspace's profile and target flags. Incremental builds limit how much code an edit recompiles; debug line tables and inlining across units increase that work. ### Speed measurements -`tests/speed.txt` holds two budgets for each function, in copies of the same number of items: one for macOS on Apple silicon and one for Linux on Syd1. The speed tests compare costs with the one that `timing::NATIVE_COLUMN` picks, which a `# held` or `# busy` line above the function's line can raise, as basedpl's DEV.md describes. A case within 10% of its budget on a platform that checks it gets a `# held` line that raises that column's budget by 10% as a margin against noise, with the date. Such a hold goes stale under basedpl's rule, once the case costs at most 0.9 times its rule budget. `timing::budgeted` reads a budget file for both speed tests. They fail on it only on the authoritative CPU, `timing::BUDGET_CPU`: the development Mac's or Syd1's. On any other CPU, they print every cost, then each case still over its budget after the retries, marked `over`, and fail on none. They time a failing case again after the rest, after each pause in `timing::RETRY_PAUSES` (0.1, 1 and 10 s), and `timing::warm` keeps the thread busy for 0.1 s after each pause, because a core that has been idle runs slower at first. A cost in copies depends on a CPU's caches as well as its speed. On the 9950X that joined CI's runner pool in October 2026, a 4-byte copy took under a third of the time of an 8-byte one, where the 7950X3D's takes half. 24 of the 28 cases over budget there were f32, though every one of them ran faster than on the 7950X3D. On machines with no budgets, such as Linux on Arm, the speed tests report costs with `budget -`. A copy is the time to write that many items of the same type, each negated, into a second buffer, with both in cache. `timing::copy_time` gives it: this machine's entry in `timing::COPIES`, by CPU, item size and number of items, or else a copy measured once, with a line on standard error to add the entry. `RUSTYMATH_COSTS` also prints each cost's time per item and a copy line for each item size. A row whose footprint, 65,536 items of its type and the most bytes it holds at once, passes 1 MiB adds that footprint and the time of a copy of it, measured beside the row, as `timing::footprint` describes. basedpl's DEV.md says how the stored times were chosen. The standard-library comparisons are informational. A budget failure often identifies a loop that stopped vectorizing. `RUSTYMATH_COSTS` prints every cost and the comparison table in `src/lib.md`'s "Accuracy and speed". +`tests/speed.txt` holds two budgets for each function, in copies of the same number of items: one for macOS on Apple silicon and one for Linux on Syd1. The speed tests compare costs with the one that `timing::NATIVE_COLUMN` picks, which a `# held` or `# busy` line above the function's line can raise, as basedpl's DEV.md describes. A case within 10% of its budget on a platform that checks it gets a `# held` line that raises that column's budget by 10% as a margin against noise, with the date. Such a hold goes stale under basedpl's rule, once the case costs at most 0.9 times its rule budget. `timing::budgeted` reads a budget file for both speed tests. They fail on it only on the authoritative CPU, `timing::BUDGET_CPU`: the development Mac's or Syd1's. On any other CPU, they print every cost, then each case still over its budget after the retries, marked `over`, and fail on none. An ordinary run times a failing case again once, under `timing::try_exclusive`, and a calibration run after each pause in `timing::RETRY_PAUSES` (0.1, 1 and 10 s). `timing::warm` keeps the thread busy for 0.1 s before each retry, because a core that has been idle runs slower at first. Each row also counts, for each item, the instructions that its first timed call retires and the bytes that it allocates. `tests/speed.txt` records them for each platform, and a count more than `timing::COUNT_MARGIN` (1.25) times its record fails, as `timing::Count::over` decides. A count changes only when the code does, so counts catch a slower loop on a busy machine too. Only a calibration run times the standard-library comparisons. A cost in copies depends on a CPU's caches as well as its speed. On the 9950X that joined CI's runner pool in October 2026, a 4-byte copy took under a third of the time of an 8-byte one, where the 7950X3D's takes half. 24 of the 28 cases over budget there were f32, though every one of them ran faster than on the 7950X3D. On machines with no budgets, such as Linux on Arm, the speed tests report costs with `budget -`. A copy is the time to write that many items of the same type, each negated, into a second buffer, with both in cache. `timing::copy_time` gives it: this machine's entry in `timing::COPIES`, by CPU, item size and number of items, or else a copy measured once, with a line on standard error to add the entry. `RUSTYMATH_COSTS` also prints each cost's time per item and a copy line for each item size. A row whose footprint, 65,536 items of its type and the most bytes it holds at once, passes 1 MiB adds that footprint and the time of a copy of it, measured beside the row, as `timing::footprint` describes. basedpl's DEV.md says how the stored times were chosen. The standard-library comparisons are informational. A budget failure often identifies a loop that stopped vectorizing. `RUSTYMATH_COSTS` prints every cost and the comparison table in `src/lib.md`'s "Accuracy and speed". -`tests/test_speed.py` times rustymath's Python API the same way, in the fresh process that `pytest` runs. Its cases are the Python expressions in `tests/speed_python.txt`, each with two budgets in copies of 65,536 items of its section's type, as in `tests/speed.txt`. It reads the file through `rm.timing()`'s `budgets`, which gives each case's `Budget` with its hold, and `RUSTYMATH_COSTS` prints the same copy lines and cost lines as the speed test, through `copy`, `peak_held`, `footprint` and `line`. To time a change against the build before it, time the baseline build that “Commands and builds” notes and your build with `basedpl.bpltests.pytest_costs`: +`tests/test_speed.py` times rustymath's Python API the same way, in the fresh process that `pytest` runs. Its cases are the Python expressions in `tests/speed_python.txt`, each with two budgets in copies of 65,536 items of its section's type, as in `tests/speed.txt`. It reads the file through `rm.timing()`'s `budgets`, which gives each case's `Budget` with its hold and the `Count` that it records for this platform. `RUSTYMATH_COSTS` prints the same copy lines and cost lines as the speed test, through `copy`, `peak_held`, `footprint`, `instructions` and `line`. Its ordinary run holds `rm.timing(shared=True)` and confirms a case over its budget under `rm.try_timing()`. It counts instructions but not allocated bytes, since the Python extension's allocator doesn't count them. To time a change against the build before it, time the baseline build that “Commands and builds” notes and your build with `basedpl.bpltests.pytest_costs`: ```python pytest_costs(rms, build=base), pytest_costs(rms) @@ -65,9 +67,11 @@ set_budgets(expected, path=rms/'tests/speed.txt', sections=sections) Here `rms` is this repo's `Path`, `expected` maps each function's name to its expected cost in copies, and `sections` names the item types whose budgets change. `path=rms/'tests/speed_python.txt'` sets the Python cases' budgets. Speed measurements use `parallel::set_threads(1)`. Measure multicore scaling separately. +A calibration run's log also gives each row's counts. `basedpl.bpltests.set_counts` records them in the budget file's column for the platform that ran it. + On macOS, a native timing measures Accelerate wherever the Linear algebra section says Accelerate replaces rustymath's own code. To time rustymath's own versions natively on the Mac, pass `native=True` to basedpl's `time_cases`, `speed_costs`, `cargo_costs` or `pytest_costs`. That sets `RUSTYMATH_NATIVE=1` for the process they time, which dev builds honour. basedpl's wasm build, timed with `speed_costs(wasm=DIR)`, also runs rustymath's own versions. Run the existing native speed suites on Syd1 for x86-64 costs. -Both speed tests and basedpl's timing tools take the machine's timing lock exclusively. Rust code takes it through `rustymath::timing`, whose docs give its waits and messages, and Python code through `rustymath.timing()`, a `with` block around the same hold. Shared and exclusive acquisition both fail after 5 minutes; builds and tests never bypass a timing. A thread's nested shared hold reuses its file; any nesting involving exclusive fails immediately. Use one lock API for a hold and its nested calls. Both speed tests time each case over its budget again after each pause in `rustymath::timing::RETRY_PAUSES`, and fail only for a case that's over every time. `fastws.timelock` takes the same lock in Python code that doesn't import rustymath, and every rustc call takes it shared through the rustc wrapper that `ws-sync` writes. basedpl's DEV.md lists the builds and tests that take it shared, and the rules for agents. +Calibration runs of both speed tests, and basedpl's timing tools, take the machine's timing lock exclusively. Ordinary runs of the speed tests take it shared, on one thread, and exclusively only through `try_exclusive`, which never waits. Rust code takes it through `rustymath_timing`, whose docs give its waits and messages, and Python code through `rustymath.timing()`, a `with` block around the same hold, or `rustymath.try_timing()`. Shared and exclusive acquisition both fail after 5 minutes; builds and tests never bypass a timing. A thread's nested shared hold reuses its file; any nesting involving exclusive fails immediately. Use one lock API for a hold and its nested calls. In a calibration run, both speed tests time each case over its budget again after each pause in `rustymath_timing::RETRY_PAUSES`, and fail only for a case that's over every time. `fastws.timelock` takes the same lock in Python code that doesn't import rustymath, and every rustc call takes it shared through the rustc wrapper that `ws-sync` writes. basedpl's DEV.md lists the builds and tests that take it shared, and the rules for agents. ### Build time and code size @@ -81,6 +85,8 @@ A generic function compiles once for each combination of types that callers give ## Code map +The dependency layers are `rustymath-core` → `rustymath-kernels` → the public `rustymath` facade; the facade also uses core directly. Core owns formulas, number facts, storage and geometry together, with no dependency on executable kernels. Kernels owns concrete typed execution. Existing public numerical paths are facade reexports, not duplicate implementations. Files below are relative to `core/src/` unless qualified. + | Files | Responsibility | |---|---| | `lib.rs`, `float.rs` | Public functions and objects; `Float`, `Native`, fused/unfused arithmetic and sealed implementation traits. | @@ -90,18 +96,21 @@ A generic function compiles once for each combination of types that callers give | `slice.rs`, `parallel.rs` | Scalar/slice object interfaces, output storage and elementwise scheduling. | | `reduce.rs`, `stats.rs` | Folds, scans, axis traversal and statistical summaries. | | `facts.rs` | What a planner knows about each function object before it reads an item: the `Range` of its results, from its arguments' ranges by interval rules, whether any result fails, and which argument a constant keeps (`Facts`). Also the one table of which reduction folds each function and which objects take characters, which rustymath, its Python crate and basedpl each expand. | -| `create.rs`, `structure.rs` | Typed construction, sequences and structural selection plans. `structure.rs` also holds each structural form's shape rule and the steps that read its result (`take_steps`, `rotate_steps`, `select_steps` and the rest), written once over the `Lengths` trait: `Known` computes on numbers, and `Bounded` on a trace's length variables, recording each decision for later runs to check (`evaluate`). Each concrete plan calls them with `Known`. A recording calls them with `Bounded` for a template, `Steps`, which each run fills at its lengths (`Steps::at`). | +| `structure.rs` | Structural selection plans. `structure.rs` also holds each structural form's shape rule and the steps that read its result (`take_steps`, `rotate_steps`, `select_steps` and the rest), written once over the `Lengths` trait: `Known` computes on numbers, and `Bounded` on a trace's length variables, recording each decision for later runs to check (`evaluate`). Each concrete plan calls them with `Known`. A recording calls them with `Bounded` for a template, `Steps`, which each run fills at its lengths (`Steps::at`). | | `arith.rs`, `integer.rs`, `convert.rs` | Wrapped/checked arithmetic, integer functions and vectorizable conversions. | | `distribution.rs`, `random.rs` | Checked distributions, function objects, sampling, rolls and deals. | | `number_theory.rs` | Primality, factorization, prime enumeration/counting and factorial. | | `field.rs`, `exact.rs`, `linalg/exact.rs`, `polynomial.rs` | Generic field algorithms, rational functions, exact matrices and polynomials. | | `search.rs`, `tolerant.rs`, `gather.rs` | Exact/tolerant search and classification, stable sorting, partial selection, comparisons and copying through conversions. | -| `error.rs`, `layout.rs`, `array.rs` | Errors that the functions on arrays return, with their kinds and messages; array shapes, axis keys and names, with their size limits; and the array type, a layout paired with the buffer that fills it. | +| `error.rs`, `layout.rs`, `array.rs` | Checked array errors; shapes, axis keys, names and size limits; and the borrowed `ArrayView` over source bindings. | | `optimize.rs` | Minimization by five methods behind one `Objective` trait, and solving behind one `Mapping` trait: Brent's root finder and Levenberg–Marquardt for nonlinear equations, and CG, MINRES and GMRES for linear ones. | | `linalg.rs`, `linalg/scalar.rs`, `linalg/factor/` | Packed products, scalar operations and decompositions. | | `signal.rs` | Correlations and convolutions, which slide a kernel along leading axes through `linalg`'s mapped products, and overlap-add. | +| `kernels/src/operation.rs`, `kernels/src/operation/` | Concrete `Kernel` resolution and typed map, fold, scan, scatter and contraction execution; depends only on core. | +| `src/array.rs`, `src/create.rs`, `src/linalg/stacked.rs` | Owned `Array`, typed construction and array-level matrix calls; the public `rustymath` facade reexports core mathematics and kernel execution. | | `py/src/`, `python/rustymath/` | The Python extension: the `Array` class, the converter of call arguments and each module's functions, with Python modules that name them. | | `buffer/` (`rustymath-buffer`) | The Python buffer protocol for compact buffers, selector decoding, dict keys and data frames, and the Python exception for each error kind. Basedpl's extension shares it. | +| `timing/` (`rustymath-timing`) | What the speed tests share: the timing lock, the timing loop, instruction and byte counts, copy times and budget files. It depends on core, and core, kernels and the facade don't depend on it, so editing it rebuilds only the tests, the Python extensions and basedpl's `testkit`. | ## Functions and slice forms @@ -253,7 +262,7 @@ With no `axis`, a reduction reduces the whole array and gives a Python number, a Statistical bindings in `py/src/functions.rs` share conversion and axis preparation through `stats::Axes`. Numerical loops live in `stats`, including two-pass variance, weighted averages, covariance and histogram counting. Float16 statistics accumulate and return float32. The 2-norm folds through `reduce::Squares`, which computes and sums the squares in float64, for float32 data too, with no buffer of squares. `linalg::norm`'s Frobenius norm is that 2-norm over every item. Complex variance and norms return real values. -`linalg::Stacked` (`src/linalg/stacked.rs`) runs rustymath's matrix functions on stacks of matrices, for rustymath's Python API and basedpl alike. A call computes in the dtype that the ladder (`DType::computes_in`) gives the common dtype of all its array arguments, singular values included, so no argument narrows. Exact numbers compute as floats of the width that the caller passes to `Stacked::new`: rustymath's Python API passes `operation::numpy_float`'s, float32 for Booleans, `uint8` and `int16` and float64 for `int32` and `int64`, and basedpl passes its exact width. Results take the common dtype where it's a float or complex dtype, and the computing float otherwise, so float16 computes in float32 and gives float16 results. `rustymath::agreement::Frames` pairs the arguments' frames, the axes before each argument's cell, by rustymath's broadcasting rule. It gives each argument's cell for each cell of the result. `linalg::cells` runs a function on each cell of the result through `parallel::each`, with scratch such as an `LuWork` for each thread, and an interrupt check between rounds. A function whose results are slices writes each cell straight into its outputs through `parallel::stack`. `linalg::paired` reads a right-hand side so that its rows pair with the matrix's rows by key, and `linalg::stored` lays a result out as the frame's axes, then the cell's. A right-hand side of one axis is one vector. Row positions read as index positions do (`structure::signed_positions`), so a float position must be a whole number. +`linalg::Stacked` (`src/linalg/stacked.rs`) runs rustymath's matrix functions on stacks of matrices, for rustymath's Python API and basedpl alike. A call computes in the dtype that the ladder (`DType::computes_in`) gives the common dtype of all its array arguments, singular values included, so no argument narrows. Exact numbers compute as floats of the width that the caller passes to `Stacked::new`: rustymath's Python API passes `DType::numpy_float`'s, float32 for Booleans, `uint8` and `int16` and float64 for `int32` and `int64`, and basedpl passes its exact width. Results take the common dtype where it's a float or complex dtype, and the computing float otherwise, so float16 computes in float32 and gives float16 results. `rustymath::agreement::Frames` pairs the arguments' frames, the axes before each argument's cell, by rustymath's broadcasting rule. It gives each argument's cell for each cell of the result. `linalg::cells` runs a function on each cell of the result through `parallel::each`, with scratch such as an `LuWork` for each thread, and an interrupt check between rounds. A function whose results are slices writes each cell straight into its outputs through `parallel::stack`. `linalg::paired` reads a right-hand side so that its rows pair with the matrix's rows by key, and `linalg::stored` lays a result out as the frame's axes, then the cell's. A right-hand side of one axis is one vector. Row positions read as index positions do (`structure::signed_positions`), so a float position must be a whole number. `py/src/linalg.rs` wraps each of `Stacked`'s methods: it converts the call's arguments, runs the method with the GIL released, and gives the results. Every binding follows that pattern through `convert::released`, which releases the GIL for a call's computation, and `convert::signals`, the module's interrupt check, which the module's init installs (`rustymath::interrupt::install`). It takes the GIL to check for Ctrl-C only on a thread running work that `released` runs. Rustymath's worker threads and calls that hold the GIL pass without taking it. Two bindings hold the GIL throughout: `optimize.rs`, which calls a Python objective on each step, and `Array.__setitem__`, whose mutable borrow would otherwise make another thread's conversion of the same array fail. A call reads each `rustymath.Array` argument as it was when converted, because the argument shares the array's storage and a write copies shared storage before changing it. A Python thread that writes to the items of a buffer argument, such as a NumPy array, during a call leaves the call's results undefined, as NumPy's are. @@ -283,7 +292,7 @@ Elementary real accuracy checks use a four-ulp bound with an additional allowanc ## CI and release -`Test` runs on pushes to main and pull requests. It uses `release` for native correctness tests and the installed `release` build for functional Python tests. CI runs no performance-budget suites. Syd1 is authoritative for x86 budgets; Jeremy's laptop is authoritative for macOS/ARM. `Release` runs separately on `v*` tags, builds dist wheels/sdist, publishes `rustymath` then `rustymath-buffer` to crates.io and the Python artifacts to PyPI, and attaches artifacts to the GitHub release. It does not wait for `Test` or run tests itself. +`Test` runs on pushes to main and pull requests. It uses `release` for native correctness tests and the installed `release` build for functional Python tests. CI runs no performance-budget suites. Syd1 is authoritative for x86 budgets; Jeremy's laptop is authoritative for macOS/ARM. `Release` runs separately on `v*` tags. It builds dist wheels/sdist, publishes the Python artifacts to PyPI and attaches artifacts to the GitHub release. `cargo publish --workspace --no-verify` publishes all publishable workspace members to crates.io in Cargo's dependency order. `Release` does not wait for `Test` or run tests itself. Before each release, run `bpl-tests prerelease` from basedpl's root. It runs rustymath's release checks and saves with basedpl's, as basedpl's DEV.md describes under "Release". Among them is `cargo test --profile checked`. That profile is the test profile with debug assertions on, which adds the standard library's checks in unsafe code. Those checks cost basedpl 12 to 55% per call, too much for the test profile that the budgets measure. The speed test marks itself ignored under debug assertions, so the run checks correctness only. diff --git a/core/Cargo.toml b/core/Cargo.toml new file mode 100644 index 0000000..342e437 --- /dev/null +++ b/core/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "rustymath-core" +version.workspace = true +edition.workspace = true +rust-version = "1.98" +license = "Apache-2.0" +description = "Numerical algorithms, formulas and compact array storage." +repository = "https://github.com/AnswerDotAI/rustymath" + +[lib] +test = false +doctest = false + +[features] +default = ["native"] +native = [] + +[dependencies] +half.workspace = true +hashbrown.workspace = true +foldhash.workspace = true +num-bigint.workspace = true +num-complex.workspace = true +num-integer.workspace = true +num-rational.workspace = true +num-traits.workspace = true +rand.workspace = true +strength_reduce.workspace = true + +# The browser executes its parallel pieces on the calling thread. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rayon.workspace = true + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +web-time.workspace = true + +[target.'cfg(unix)'.dependencies] +libc.workspace = true + diff --git a/build.rs b/core/build.rs similarity index 100% rename from build.rs rename to core/build.rs diff --git a/src/agreement.rs b/core/src/agreement.rs similarity index 99% rename from src/agreement.rs rename to core/src/agreement.rs index 909a998..b027c65 100644 --- a/src/agreement.rs +++ b/core/src/agreement.rs @@ -201,7 +201,7 @@ impl ReadPlan { let (source, start, stride) = self.run(&range)?; (stride == 1 || range.len() == 1).then(|| (source, start..start + range.len())) } - pub(crate) fn shape_hint(&self) -> Vec { match self.route() { ReadRoute::Map { map, .. } => map.shape(self.items), _ => vec![self.items] } } + pub fn shape_hint(&self) -> Vec { match self.route() { ReadRoute::Map { map, .. } => map.shape(self.items), _ => vec![self.items] } } /// The one source that a read over a result of `shape` reaches by an affine map, and that map, where every position stays within the source. `None` for a read that joins sources, fills, or maps otherwise. pub fn positional(&self, shape: &[usize]) -> Option<(usize, Mapping)> { if count(shape) != self.items { return None; } @@ -333,7 +333,7 @@ impl<'a> Binding<'a> { pub struct Read<'a> { pub sources: &'a [Binding<'a>], pub plan: &'a ReadPlan } /// A read by rows, from [`Read::rows`]: row `r` starts at `items[first + r × step]`, and reads a run of items from there where `run` is true, and else that one item for the whole row. -pub(crate) struct Rows<'a, T> { pub items: &'a [T], pub first: usize, pub step: isize, pub run: bool } +pub struct Rows<'a, T> { pub items: &'a [T], pub first: usize, pub step: isize, pub run: bool } impl<'s> Read<'s> { /// Checks each source that the plan reads: bound, of the length the plan reads, with its window inside that length. @@ -395,11 +395,11 @@ impl<'s> Read<'s> { Ok(Arg::Many(scratch)) } /// Whether `part` reads `range` in place, writing no scratch: one run of a source that holds `T`, or one item that every position reads. - pub(crate) fn in_place(&self, range: &Range) -> bool { + pub fn in_place(&self, range: &Range) -> bool { self.run_view(range).is_some_and(|(view, stride)| stride == 0 || view.dtype() == T::DTYPE) } /// This read as rows over a result of `shape`, which is rows then items, where every source is bound as the plan reads it, the plan maps one source that holds `T` affinely over `shape`, and each row reads one item or a run of items. `None` for any other read. - pub(crate) fn rows(&self, shape: [usize; 2]) -> Option> { + pub fn rows(&self, shape: [usize; 2]) -> Option> { self.bindings().ok()?; let (source, base, strides) = self.plan.affine(&shape)?; let (view, first) = self.sources.get(source)?.affine(base, &strides, &shape)?; diff --git a/src/arc.rs b/core/src/arc.rs similarity index 100% rename from src/arc.rs rename to core/src/arc.rs diff --git a/src/arith.rs b/core/src/arith.rs similarity index 90% rename from src/arith.rs rename to core/src/arith.rs index a20321a..3a9a12a 100644 --- a/src/arith.rs +++ b/core/src/arith.rs @@ -1,4 +1,4 @@ -//! Arithmetic as formula objects, which [`Kernel`](crate::operation::Kernel) runs through their [`Formula`](crate::facts::Formula) rows. Addition, subtraction, multiplication and negation take every float and complex type and the integer types of compact buffers, `u8`, `i16`, `i32` and `i64`. Division, reciprocals and multiples of π take every float and complex type, and squares take floats. Reciprocals also take the integer types, where they floor, as integer division of 1 does. Bitwise and logical functions take them and Booleans. A float or complex result never fails. An integer sum, difference, product or negation that doesn't fit its type fails, and holds the result wrapped to the type, as NumPy gives it. Any other integer result that the type can't hold fails. +//! Arithmetic as formula objects, which `rustymath::operation::Kernel` runs through their [`Formula`](crate::facts::Formula) rows. Addition, subtraction, multiplication and negation take every float and complex type and the integer types of compact buffers, `u8`, `i16`, `i32` and `i64`. Division, reciprocals and multiples of π take every float and complex type, and squares take floats. Reciprocals also take the integer types, where they floor, as integer division of 1 does. Bitwise and logical functions take them and Booleans. A float or complex result never fails. An integer sum, difference, product or negation that doesn't fit its type fails, and holds the result wrapped to the type, as NumPy gives it. Any other integer result that the type can't hold fails. use crate::float::sealed::Sealed; /// x + y. @@ -103,6 +103,7 @@ macro_rules! wide_product { macro_rules! signed { ($($t:ty: |$x:ident, $y:ident| $times:expr),+) => {$( impl Overflowing for $t { + #[inline] fn low(x: i64) -> Self { x as $t } #[inline(always)] fn limit(sign: i64) -> Self { match sign.signum() { 1 => <$t>::MAX, -1 => <$t>::MIN, _ => 0 } } @@ -125,6 +126,7 @@ signed!( macro_rules! unsigned { ($($t:ty: |$x:ident, $y:ident| $times:expr),+) => {$( impl Overflowing for $t { + #[inline] fn low(x: i64) -> Self { x as $t } #[inline(always)] fn limit(sign: i64) -> Self { match sign.signum() { 1 => <$t>::MAX, -1 => <$t>::MIN, _ => 0 } } diff --git a/core/src/array.rs b/core/src/array.rs new file mode 100644 index 0000000..a1b0dc2 --- /dev/null +++ b/core/src/array.rs @@ -0,0 +1,55 @@ +//! Borrowed array storage, layouts and positional reads. +use crate::agreement::{Binding, Mapping, Missing, Read, ReadPlan}; +use crate::buffer::{Buffer, DType, Element, Items, View}; +use crate::error::{Error, Kind}; +use crate::layout::Layout; +use std::borrow::Cow; + +/// Sources lent for one operation. The plan carries positions, never data or a materialization cache. +#[derive(Clone)] +pub struct ArrayView<'a> { layout: &'a Layout, sources: Vec>, plan: ReadPlan } + +impl<'a> ArrayView<'a> { + pub fn new(layout: &'a Layout, sources: Vec>, plan: ReadPlan) -> Result { + if layout.count() != plan.len() { return Err(Error::new(Kind::Length, "read plan doesn't fit its layout")); } + Ok(Self { layout, sources, plan }) + } + pub fn compact(layout: &'a Layout, storage: View<'a>) -> Self { + Self { layout, sources: vec![Binding::full(storage)], plan: ReadPlan::input(0, storage.len()) } + } + pub fn layout(&self) -> &'a Layout { self.layout } + pub fn dtype(&self) -> DType { self.sources[0].view.dtype() } + pub fn len(&self) -> usize { self.plan.len() } + pub fn is_empty(&self) -> bool { self.plan.is_empty() } + pub fn read(&self) -> Read<'_> { Read { sources: &self.sources, plan: &self.plan } } + pub fn plan(&self) -> &ReadPlan { &self.plan } + pub fn sources(&self) -> &[Binding<'a>] { &self.sources } + /// Changes metadata, retaining the logical sequence. + pub fn with_layout<'b>(&self, layout: &'b Layout) -> Result, Error> where 'a: 'b { + ArrayView::new(layout, self.sources.clone(), self.plan.clone()) + } + /// Composes a call's positional read. Missing positions read the prototype (`Missing::prototype`). + pub fn remapped(&self, mapping: Mapping, items: usize) -> Self { + let mut sources = self.sources.clone(); + let missing = Missing::prototype(self.dtype(), &mut sources); + Self { layout: self.layout, sources, plan: self.plan.remapped(mapping, items, missing) } + } + pub fn contiguous(&self) -> Option> { + if self.is_empty() { return Some(self.sources[0].view.slice(0..0)); } + let (source, range) = self.plan.contiguous(0..self.len())?; + self.sources.get(source)?.borrow(range) + } + pub fn affine(&self) -> Option<(View<'a>, usize, Vec)> { + let (source, base, strides) = self.plan.affine(self.layout.shape())?; + let (view, base) = self.sources.get(source)?.affine(base, &strides, self.layout.shape())?; + Some((view, base, strides)) + } + /// Borrows a contiguous read at its dtype, otherwise prepares exactly these logical items. + pub fn cast(&self, dtype: DType) -> Result, Error> { + if dtype == self.dtype() && let Some(view) = self.contiguous() { return Ok(Items::Borrowed(view)); } + let mut buffer = Buffer::with_capacity(dtype, self.len()); + self.read().read_into(0..self.len(), dtype, true, &mut buffer)?; + Ok(Items::Owned(buffer)) + } + pub fn items(&self) -> Result, Error> { Ok(self.cast(T::DTYPE)?.items()) } +} diff --git a/src/buffer.rs b/core/src/buffer.rs similarity index 99% rename from src/buffer.rs rename to core/src/buffer.rs index 95cc925..8f20c6e 100644 --- a/src/buffer.rs +++ b/core/src/buffer.rs @@ -300,6 +300,9 @@ impl DType { Self::Character => return None }) } + /// The float width that a Boolean or integer computes in under rustymath's own policy, NumPy's: the float that [`DType::inexact`] gives it under `f64`. `f64` for any other dtype. + pub fn numpy_float(self) -> FloatWidth { match self.inexact(FloatWidth::F64) { Some((DType::Float(w), _)) => w, _ => FloatWidth::F64 } } + /// The dtype that an argument of this dtype computes in, for a function with a kernel at each dtype where `has` holds. That's this dtype where it has a kernel. Otherwise it's the first with a kernel of these: for a Boolean or an integer, integers of `int` or of this dtype's width, whichever is wider, then floats of `float`; for a float or a complex number, the dtype that [`DType::inexact`] gives it; then each wider float, and each complex dtype whose parts are at least as wide; and last, for a float, integers of `int`, which compute it as the whole number it holds (`operation::whole_numbers`). `int` and `float` are the caller's policy for exact numbers. NumPy's is `i64` and the float that `inexact` gives under `f64`. `None` where no dtype on the way has a kernel. pub fn computes_in(self, int: Width, float: FloatWidth, has: impl Fn(DType) -> bool) -> Option { if has(self) { return Some(self); } diff --git a/src/characters.rs b/core/src/characters.rs similarity index 96% rename from src/characters.rs rename to core/src/characters.rs index c36d1f4..27f57f2 100644 --- a/src/characters.rs +++ b/core/src/characters.rs @@ -9,7 +9,7 @@ pub enum Rule { Identity, Ordered, Shift, Difference } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ByKinds { Truth(bool), Left, Right } impl ByKinds { - pub(crate) fn ordered([left, _]: [i32; 2], result: i32, boolean: bool) -> Self { + pub fn ordered([left, _]: [i32; 2], result: i32, boolean: bool) -> Self { if boolean { Self::Truth(result != 0) } else if result == left { Self::Left } else { Self::Right } } } diff --git a/src/complex.rs b/core/src/complex.rs similarity index 100% rename from src/complex.rs rename to core/src/complex.rs diff --git a/src/complex/bulk.rs b/core/src/complex/bulk.rs similarity index 82% rename from src/complex/bulk.rs rename to core/src/complex/bulk.rs index e4b815e..a642de3 100644 --- a/src/complex/bulk.rs +++ b/core/src/complex/bulk.rs @@ -1,6 +1,6 @@ //! Complex slice bodies split into real passes. Rare inputs use the same object's scalar body; the map owner checks only the final complex result. use super::*; -use crate::{Asinh, Atan2, Cos, Cosh, Exp, FUSED, Hypot, Ln, Log1p, Sin, Sinh, buffer::Cast, facts::Formula, recipe::NativeOps, slice::{self, Arg}}; +use crate::{Asinh, Atan2, Cosh, Exp, FUSED, Hypot, Ln, Log1p, Sinh, buffer::Cast, facts::Formula, recipe::NativeOps, slice::{self, Arg}}; use std::mem::MaybeUninit; mod trig; @@ -8,12 +8,34 @@ use trig::*; fn real, R: Native + Cast, const N: usize>(k: K, args: [&[R]; N], out: &mut Vec) { let n = args[0].len(); - slice::appended(out, n, |out| crate::operation::real_slice(&k, &NativeOps::DEFAULT, args.map(Arg::Many), n, out)); + unsafe { slice::appended(out, n, |out| crate::facts::real_slice(&k, &NativeOps::DEFAULT, args.map(Arg::Many), n, out)) }; } fn sin_cos(y: &[R], sin: &mut Vec, cos: &mut Vec) { - real(Sin, [y], sin); - real(Cos, [y], cos); + #[cfg(not(target_os = "macos"))] + { + use crate::{Cos, Sin}; + real(Sin, [y], sin); + real(Cos, [y], cos); + } + #[cfg(target_os = "macos")] + { + #[link(name = "Accelerate", kind = "framework")] + unsafe extern "C" { + fn vvsincos(sin: *mut f64, cos: *mut f64, y: *const f64, n: *const i32); + fn vvsincosf(sin: *mut f32, cos: *mut f32, y: *const f32, n: *const i32); + } + let n = i32::try_from(y.len()).expect("vForce takes at most `i32::MAX` items"); + // SAFETY: Native is sealed to f32/f64, WIDE selects that type, and appended provides disjoint slots for every result. + unsafe { + slice::appended(sin, y.len(), |sin| { + slice::appended(cos, y.len(), |cos| { + if R::WIDE { vvsincos(sin.as_mut_ptr().cast(), cos.as_mut_ptr().cast(), y.as_ptr().cast(), &n); } + else { vvsincosf(sin.as_mut_ptr().cast(), cos.as_mut_ptr().cast(), y.as_ptr().cast(), &n); } + }) + }); + } + } } fn split, R: Native + Cast, const N: usize>( @@ -35,7 +57,7 @@ fn split, R: Native + Cast, const N: usize>( fast(std::array::from_fn(|i| (&parts[i].0[..], &parts[i].1[..])), &mut re, &mut im); assert!(re.len() == n && im.len() == n); for ((slot, re), im) in out[..n].iter_mut().zip(re).zip(im) { slot.write(c(re, im)); } - let values = slice::written(&mut out[..n]); + let values = unsafe { slice::written(&mut out[..n]) }; for (i, value) in values.iter_mut().enumerate() { let x = std::array::from_fn(|j| args[j].nth(i)); if rare(x, *value) { *value = k.complex(ops, x).expect("a complex slice has a scalar body"); } diff --git a/src/complex/bulk/trig.rs b/core/src/complex/bulk/trig.rs similarity index 95% rename from src/complex/bulk/trig.rs rename to core/src/complex/bulk/trig.rs index 28961e3..d64c71e 100644 --- a/src/complex/bulk/trig.rs +++ b/core/src/complex/bulk/trig.rs @@ -25,15 +25,17 @@ pub(super) fn moduli(x: &[R], y: &[R], out: &mut Vec) { pub(super) fn sqrt_parts(x: &[R], y: &[R], re: &mut Vec, im: &mut Vec) { let mut h = Vec::new(); moduli(x, y, &mut h); - crate::slice::appended(re, x.len(), |re| { - crate::slice::appended(im, x.len(), |im| { - for ((((&x, &y), &h), re), im) in x.iter().zip(y).zip(&h).zip(re).zip(im) { - let (r, i) = root::(x, y, h); - re.write(r); - im.write(i); - } + unsafe { + crate::slice::appended(re, x.len(), |re| { + crate::slice::appended(im, x.len(), |im| { + for ((((&x, &y), &h), re), im) in x.iter().zip(y).zip(&h).zip(re).zip(im) { + let (r, i) = root::(x, y, h); + re.write(r); + im.write(i); + } + }) }) - }) + } } pub(super) fn rough_root(z: Complex) -> bool { diff --git a/src/convert.rs b/core/src/convert.rs similarity index 97% rename from src/convert.rs rename to core/src/convert.rs index e9fe817..ea0184c 100644 --- a/src/convert.rs +++ b/core/src/convert.rs @@ -130,12 +130,14 @@ pub fn count(out: &mut Vec, start: i64, down: bool, n: usize) { let step = if down { -1 } else { 1 }; let offsets: [f64; 8] = std::array::from_fn(|k| wide_float(step * k as i64)); let first = move |j: usize| wide_float(start + step * (8 * j) as i64); - crate::slice::appended(out, n, |slots| { - let (blocks, rest) = slots.as_chunks_mut::<8>(); - let whole = blocks.len(); - for (j, block) in blocks.iter_mut().enumerate() { let first = first(j); for k in 0..8 { block[k].write(F::narrow(first + offsets[k])); } } - for (slot, &offset) in rest.iter_mut().zip(&offsets) { slot.write(F::narrow(first(whole) + offset)); } - }); + unsafe { + crate::slice::appended(out, n, |slots| { + let (blocks, rest) = slots.as_chunks_mut::<8>(); + let whole = blocks.len(); + for (j, block) in blocks.iter_mut().enumerate() { let first = first(j); for k in 0..8 { block[k].write(F::narrow(first + offsets[k])); } } + for (slot, &offset) in rest.iter_mut().zip(&offsets) { slot.write(F::narrow(first(whole) + offset)); } + }) + }; } /// Whether the target converts between `f64` and 64-bit integers in vector form, as AArch64 and x86-64 with AVX-512 do. Elsewhere they convert through a float's bits. Both ways give the same results for whole floats. diff --git a/src/dd.rs b/core/src/dd.rs similarity index 100% rename from src/dd.rs rename to core/src/dd.rs diff --git a/src/derivative.rs b/core/src/derivative.rs similarity index 100% rename from src/derivative.rs rename to core/src/derivative.rs diff --git a/src/distribution.rs b/core/src/distribution.rs similarity index 99% rename from src/distribution.rs rename to core/src/distribution.rs index 5b4f778..6287a19 100644 --- a/src/distribution.rs +++ b/core/src/distribution.rs @@ -100,7 +100,7 @@ macro_rules! through_slices { fn one_item(many: impl FnOnce(&mut [MaybeUninit]) -> bool) -> F { let mut out = [MaybeUninit::uninit()]; many(&mut out); - written(&mut out)[0] + (unsafe { written(&mut out) })[0] } /// `f` of `x`, computed in `f64` and rounded to `x`'s type. @@ -163,7 +163,7 @@ fn composed( args.clear(); args.extend(x.iter().map(|&v| inner(v))); values(&mut args, &mut out[..x.len()]); - let results = written(&mut out[..x.len()]); + let results = unsafe { written(&mut out[..x.len()]) }; let mut clean = true; // The flag reads only the result, as `slice::each`'s does. x86-64 leaves this loop scalar when the flag also reads the argument. for (r, &v) in results.iter_mut().zip(x) { @@ -173,7 +173,7 @@ fn composed( ok &= clean || results.iter().zip(x).all(|(&r, &v)| kept(r, [v])); } if copies > 0 { - let r = written(&mut out[..1])[0]; + let r = unsafe { written(&mut out[..1]) }[0]; out[1..n].fill(MaybeUninit::new(r)); } ok @@ -705,7 +705,7 @@ impl Formulas for Weibull { let (degree, scale) = (F::narrow(self.shape), F::narrow(self.scale)); composed(p, n, out, |p| -p, move |a, v| { Log1p.slice([Arg::Many(a)], a.len(), v); - for (x, &l) in a.iter_mut().zip(written(v).iter()) { *x = -l; } + for (x, &l) in a.iter_mut().zip(unsafe { written(v) }.iter()) { *x = -l; } crate::pow::roots(degree, a, v); }, move |_, w| scale * w) } diff --git a/src/domain.rs b/core/src/domain.rs similarity index 100% rename from src/domain.rs rename to core/src/domain.rs diff --git a/src/error.rs b/core/src/error.rs similarity index 100% rename from src/error.rs rename to core/src/error.rs diff --git a/src/events.rs b/core/src/events.rs similarity index 100% rename from src/events.rs rename to core/src/events.rs diff --git a/src/exact.rs b/core/src/exact.rs similarity index 100% rename from src/exact.rs rename to core/src/exact.rs diff --git a/src/exp.rs b/core/src/exp.rs similarity index 99% rename from src/exp.rs rename to core/src/exp.rs index 44708a8..b576235 100644 --- a/src/exp.rs +++ b/core/src/exp.rs @@ -102,10 +102,12 @@ fn minus_one_with + ?Sized>(o: &O, r: o.multiply_add(o.multiply(r, h), scale, o.subtract(scale, o.number(F::one()))) } +#[inline] pub(crate) fn scaled_formula + ?Sized>(o: &O, x: O::V, scale: O::V) -> O::V { parts_with(o, x, o.number(-F::zero()), scale) } +#[inline] pub(crate) fn minus_one_formula + ?Sized>(o: &O, x: O::V, scale: O::V) -> O::V { let (low, top) = if o.width() == crate::buffer::FloatWidth::F64 { (-40.0, 1023.0) } else { (-18.0, 127.0) }; let low = o.number(F::narrow(low)); diff --git a/src/facts.rs b/core/src/facts.rs similarity index 95% rename from src/facts.rs rename to core/src/facts.rs index e49251f..a983e26 100644 --- a/src/facts.rs +++ b/core/src/facts.rs @@ -1,12 +1,12 @@ -//! Shared function definitions for numerical frontends. Each [`Formula`] row owns its kind and character policies, numerical body, validity and repair predicates, ranges, identities, partial derivatives and exact cases. [`Kernel`](crate::operation::Kernel) erases the arity for execution. [`facts!`](crate::facts) lists typed product pairs and Python names. -use crate::buffer::{DType, FloatWidth}; +//! Shared function definitions for numerical frontends. Each [`Formula`] row owns its kind and character policies, numerical body, validity and repair predicates, ranges, identities, partial derivatives and exact cases. `rustymath::operation::Kernel` erases the arity for execution. [`facts!`](crate::facts) lists typed product pairs and Python names. +use crate::buffer::{DType, FloatWidth, Source}; use crate::domain::{Range, Variant}; use crate::float::sealed::Core; use crate::pow::parity; use crate::recipe::{NativeOps, Ops}; use crate::reduce::{Maximum, Minimum, Product, Reduce, Reduction}; use crate::tolerant::{Residue, Tolerance, Tolerant, TolerantComplex, lcm_from_gcd}; -use crate::{Betainc, BetaincAt, Binary, FUSED, Flipped, Float, Native, Unary}; +use crate::{Arg, Betainc, BetaincAt, Binary, FUSED, Flipped, Float, Native, Unary}; use num_complex::Complex; use num_traits::{One, Signed, Zero}; use std::f64::consts::PI; @@ -65,9 +65,7 @@ pub trait Formula: Clone + Send + Sync + 'static { /// Whether kernels of this formula only map. Modified writes, folds, scans and walks of adjacent items build no code for it: each refuses it at compile time, as it refuses a formula of another arity. A derivative's slope (`derivative::Slope`) only maps. const MAP_ONLY: bool = false; fn result_dtype(&self, input: DType) -> DType { if Self::BOOLEAN { DType::Boolean } else { input } } - fn computing_dtype(&self, input: DType) -> Option { - input.computes_in(crate::buffer::Width::I64, crate::operation::numpy_float(input), |d| self.supports(d)) - } + fn computing_dtype(&self, input: DType) -> Option { input.computes_in(crate::buffer::Width::I64, input.numpy_float(), |d| self.supports(d)) } fn character_rule(&self) -> Option { None } fn ordered_kinds(&self, [x, y]: [i32; 2]) -> Option { let ops = crate::recipe::IntegerOps::::new(Variant::checked(DType::Integer(crate::buffer::Width::I32))); @@ -91,10 +89,6 @@ pub trait Formula: Clone + Send + Sync + 'static { /// Refines the real result interval when both operands denote the same value. fn same(&self, _arg: Range) -> Option { None } fn reads(&self) -> Reads { Reads::Items } - fn resolve(&self, variant: Variant, types: [DType; N]) -> Result { - Kernel::try_resolve(self.clone(), variant, types)? - .ok_or_else(|| crate::error::Error::new(crate::error::Kind::Domain, "function has no numerical body for these arguments")) - } fn supports(&self, _dtype: DType) -> bool { false } fn emit + ?Sized>(&self, _o: &O, _x: [O::V; N]) -> Option { None } fn emit_test + ?Sized>(&self, _o: &O, _x: [O::V; N]) -> Option { None } @@ -107,6 +101,7 @@ pub trait Formula: Clone + Send + Sync + 'static { fn emit_valid + ?Sized>(&self, _variant: Variant, o: &O, x: [O::V; N], result: O::V) -> Option<(crate::error::Kind, O::B)> { Self::FAILS_NAN.then(|| (crate::error::Kind::Domain, nan_valid(o, x, [result]))) } + #[inline] fn real(&self, ops: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [F; N]) -> Option { if const { Self::BOOLEAN } { self.emit_test(ops, x).map(|x| if x { F::one() } else { F::zero() }) } else { self.emit(ops, x) } } @@ -123,7 +118,7 @@ pub trait Formula: Clone + Send + Sync + 'static { } } -/// A use of a formula object by code generic over formulas, as [`Op::visit`] hands it over. `unary` and `binary` receive the one- and two-argument objects, through `visit` unless a visitor wraps them, as [`Kernel::slope`] does. +/// A use of a formula object by code generic over formulas, as [`Op::visit`] hands it over. `unary` and `binary` receive the one- and two-argument objects, through `visit` unless a visitor wraps them, as `rustymath::operation::Kernel::slope` does. pub trait Visit: Sized { type Out; fn visit, const N: usize>(self, object: K) -> Self::Out; @@ -134,7 +129,7 @@ pub trait Visit: Sized { /// Writes `Op` and `Op::visit` from one list: objects without fields by arity (`reduce`'s are binary), then objects that hold a value, with the arity and constructor of each. macro_rules! ops { (unary [$($u:ident)*] binary [$($b:ident)*] reduce [$($r:ident)*] ternary [$($t:ident)*] held [$($h:ident($ty:ty) => $how:ident $make:expr),* $(,)?]) => { - /// A formula object by name, for a front end that picks its formula at run time, as BPL's primitives and rustymath's Python functions do: [`Kernel::of`] resolves one, so each formula's kernel code compiles once, here, and [`Op::visit`] hands its object to code generic over formulas. + /// A formula object by name, for a front end that picks its formula at run time, as BPL's primitives and rustymath's Python functions do: `rustymath::operation::Kernel::of` resolves one, so each formula's kernel code compiles once in rustymath-kernels, and [`Op::visit`] hands its object to code generic over formulas. #[derive(Clone, Copy, Debug)] pub enum Op { $($u,)* $($b,)* $($r,)* $($t,)* $($h($ty),)* PiTimes, Bitwise(u8), BetaincAt(f64, f64), FlippedBinom } impl Op { @@ -198,6 +193,15 @@ pub struct NativeSlices { /// A slice form takes the item body's run context and writes all `n` results. Real and integer forms return their validity; complex forms leave validation to `complex_valid` at the map boundary. pub type Slice = fn(&K, &NativeOps<{ FUSED }>, [crate::Arg<'_, T>; N], usize, &mut [std::mem::MaybeUninit]) -> R; impl NativeSlices { + pub fn single(&self) -> Option> { self.single } + pub fn double(&self) -> Option> { self.double } + pub fn byte(&self) -> Option> { self.byte } + pub fn short(&self) -> Option> { self.short } + pub fn int(&self) -> Option> { self.int } + pub fn long(&self) -> Option> { self.long } + pub fn complex_single(&self) -> Option> { self.complex_single } + pub fn complex_double(&self) -> Option> { self.complex_double } + const NONE: Self = Self { single: None, double: None, byte: None, short: None, int: None, long: None, complex_single: None, complex_double: None }; fn with_complex(self, single: Slice, double: Slice) -> Self { Self { complex_single: Some(single), complex_double: Some(double), ..self } @@ -230,11 +234,24 @@ impl NativeSlices { } } +impl NativeSlices { + fn logarithm() -> Self { + Self { single: Some(|_, o, x, n, out| logarithm_slice::(o, x, n, out)), + double: Some(|_, o, x, n, out| logarithm_slice::(o, x, n, out)), ..Self::NONE } + } +} +impl NativeSlices { + fn mathish_logarithm() -> Self { + Self { single: Some(|_, o, x, n, out| logarithm_slice::(o, x, n, out)), + double: Some(|_, o, x, n, out| logarithm_slice::(o, x, n, out)), ..Self::NONE } + } +} + #[derive(Clone, Copy, Debug)] pub struct Repair { pub single: extern "C" fn(f32) -> f32, pub double: extern "C" fn(f64) -> f64 } #[inline(always)] -pub(crate) fn nan_valid + ?Sized>(o: &O, inputs: impl IntoIterator, results: impl IntoIterator) -> O::B { +pub fn nan_valid + ?Sized>(o: &O, inputs: impl IntoIterator, results: impl IntoIterator) -> O::B { let mut valid = o.equal(o.number(F::zero()), o.number(F::zero())); for result in results { valid = o.and(valid, o.equal(result, result)); } for input in inputs { valid = o.or(valid, o.not(o.equal(input, input))); } @@ -247,6 +264,14 @@ fn logarithm_valid + ?Sized>(o: &O, x: [O::V; 2], result: O: Some((crate::error::Kind::Domain, o.or(nonnegative, nan_valid(o, x, [result])))) } +fn logarithm_slice(ops: &NativeOps<{ FUSED }>, args: [Arg<'_, F>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { + crate::ln::logb_many::(args, n, out) || { + // SAFETY: logb_many initializes every output, even when it reports a NaN result. + let results = unsafe { crate::slice::written(&mut out[..n]) }; + crate::slice::judged(&args, results, |x, r| logarithm_valid(ops, x, r).unwrap().1) + } +} + impl Repair { pub fn apply(self, x: F) -> F { F::narrow(if F::Working::WIDE { (self.double)(x.wide()) } else { (self.single)(x.wide() as f32) as f64 }) } } @@ -257,8 +282,6 @@ pub enum Reads { Items, Whole } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Slot(pub u32); -pub use crate::operation::Kernel; - /// `k`'s scalar real result, including repairs, computed at the argument's working precision. #[inline(always)] pub fn formula_at, F: crate::Float, const N: usize>(k: &K, x: [F; N]) -> F { @@ -269,9 +292,19 @@ pub fn formula_at, F: crate::Float, const N: usize>(k: &K, x: [F; F::from_working(result) } +/// A real formula's slice body and rare-item repairs, without recording errors from intermediate values. +pub fn real_slice, F: Native + Source, O: Float, const N: usize>( + object: &K, ops: &NativeOps<{ FUSED }>, args: [Arg<'_, F>; N], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { + let body = |x| object.real(ops, x).expect("a resolved object has a real body"); + if let Some(repair) = object.repair_routine() { + let needs = |x| object.emit_repair(ops, x).unwrap_or(false); + crate::slice::two_pass(args, n, out, |x| (!needs(x)).then(|| body(x)), |x| if needs(x) { repair.apply(x[0]) } else { body(x) }) + } else { crate::slice::each(args, n, out, #[inline(always)] |x| Source::::read(body(x))) } +} + /// Bit `2x+y` of `table`, or bit `x` for one argument: a Boolean function's result by its truth table. #[inline(always)] -pub(crate) fn truth_bit(table: u8, x: [bool; N]) -> bool { (table >> x.iter().fold(0, |at, &b| 2 * at + u8::from(b))) & 1 != 0 } +pub fn truth_bit(table: u8, x: [bool; N]) -> bool { (table >> x.iter().fold(0, |at, &b| 2 * at + u8::from(b))) & 1 != 0 } /// Each object's facts, formula and partials have one definition. macro_rules! objects { @@ -570,8 +603,8 @@ objects!(2: crate::RealPow { range [x, _] => signs([x]); identity [None, Some(1.0)]; output real; formula |_, o, [x, y]| Some(crate::pow::real_formula(o, x, y)); shared real_power; native_complex |_, _, [x, y]| Some(crate::complex::pow::(x, y)); } // The yth root of x, x^(1/y). crate::Root { range [_, y] => signs([y]); domain _, [n, y] => root_domain(n, y); formula |_, o, [n, y]| crate::pow::root_formula(o, n, y); slopes |o, x, y, r| [o.negate(o.divide(o.multiply(r, o.ln(y)), o.multiply(x, x))), o.divide(r, o.multiply(x, y))]; native_complex |_, _, [n, y]| Some(crate::complex::nth_root::(n, y)); native_real |_, _, [n, y]| Some(crate::pow::root::(n, y)); slices root; exact |_, [n, y]| { if num_traits::Zero::is_zero(n) { return Err(crate::error::Error::new(crate::error::Kind::Domain, "root degree is zero")); } crate::exact::power(y, &n.recip()) }; checks nan; } - crate::Logb { domain _, [x, y] => inside([x, y], [Range::NONNEGATIVE; 2]); shared logb; native_complex |_, _, [b, x]| Some(crate::complex::divide(crate::complex::ln::(x), crate::complex::ln::(b))); native_real |_, _, [b, x]| Some(crate::ln::logb::(b, x)); valid |_, _, o, x, r| logarithm_valid(o, x, r); } - crate::MathishLogb { domain _, [x, y] => inside([x, y], [Range::NONNEGATIVE; 2]); shared logb; native_complex |_, _, [b, x]| { let one = num_complex::Complex::new(F::one(), F::zero()); Some(if b == one && x == one { num_complex::Complex::new(F::zero(), F::zero()) } else { crate::complex::divide(crate::complex::ln::(x), crate::complex::ln::(b)) }) }; native_real |_, _, [b, x]| { let r = crate::ln::logb::(b, x); Some(if (b == F::one()) & (x == F::one()) { F::zero() } else { r }) }; valid |_, _, o, x, r| logarithm_valid(o, x, r); } + crate::Logb { domain _, [x, y] => inside([x, y], [Range::NONNEGATIVE; 2]); shared logb; native_complex |_, _, [b, x]| Some(crate::complex::divide(crate::complex::ln::(x), crate::complex::ln::(b))); native_real |_, _, [b, x]| Some(crate::ln::logb::(b, x)); slices logarithm; valid |_, _, o, x, r| logarithm_valid(o, x, r); } + crate::MathishLogb { domain _, [x, y] => inside([x, y], [Range::NONNEGATIVE; 2]); shared logb; native_complex |_, _, [b, x]| { let one = num_complex::Complex::new(F::one(), F::zero()); Some(if b == one && x == one { num_complex::Complex::new(F::zero(), F::zero()) } else { crate::complex::divide(crate::complex::ln::(x), crate::complex::ln::(b)) }) }; native_real |_, _, [b, x]| Some(crate::ln::logb_mode::(b, x)); slices mathish_logarithm; valid |_, _, o, x, r| logarithm_valid(o, x, r); } // `y` modulo `x`. Its slope in `x` is minus the quotient's floor, (y - r) ÷ x from the result, and 0 where `x` is 0, where the residue is `y` itself. Integers take a positive power of two that every item shares as a mask (`masked`). Integer kernels run its slice forms (`integer.rs`). Where a whole block shares one divisor, those take a mask or a reciprocal in place of a division. crate::tolerant::Residue { identity [Some(0.0), None]; @@ -725,7 +758,7 @@ fn masked + ?Sized>(o: &O, x: O::V, y: fn native_tolerance(value: Option, fallback: Tolerance) -> Tolerance { value.and_then(Tolerance::new).unwrap_or(fallback) } /// The tolerance at which `o` compares at tolerance `t`: the one that `o` sets, otherwise `t` (`Ops::tolerance`). `None` where it isn't known when the recipe is built. -pub(crate) fn run_tolerance + ?Sized>(o: &O, t: crate::tolerant::Tolerance) -> Option { +pub fn run_tolerance + ?Sized>(o: &O, t: crate::tolerant::Tolerance) -> Option { crate::tolerant::Tolerance::new(o.constant(o.tolerance(F::narrow(t.value())))?.wide()) } /// Comparison `k` of `v` with the one number `c`, as `k`'s kernel compares items with a number: the band that `v` must lie in against the range of floats equal to `c` at tolerance `t`, or at the tolerance that `o` sets (`tolerant::against_steps`). `left` says whether `v` is the left argument. `None` where the tolerance isn't known when the recipe is built, or `c` is NaN. @@ -735,6 +768,7 @@ fn against(k: K, t: crate::tolerant::Tolerance, o: &O, x: O::V, y: O::V) -> O::B where K: crate::tolerant::Stepped + crate::tolerant::Banding + Copy, F: crate::Float, O: crate::recipe::Ops + ?Sized { let side = match (o.constant(x), o.constant(y)) { @@ -799,8 +833,11 @@ impl> Formula<2> for Flipped { fn supports(&self, dtype: DType) -> bool { self.0.supports(dtype) } const BOOLEAN: bool = K::BOOLEAN; const FAILS_NAN: bool = K::FAILS_NAN; + #[inline] fn integer(&self, o: &crate::recipe::IntegerOps, [x, y]: [I; 2]) -> Option { self.0.integer(o, [y, x]) } + #[inline] fn real(&self, o: &crate::recipe::NativeOps<{ crate::FUSED }>, [x, y]: [F; 2]) -> Option { self.0.real(o, [y, x]) } + #[inline] fn complex( &self, o: &crate::recipe::NativeOps<{ crate::FUSED }>, [x, y]: [num_complex::Complex; 2]) -> Option> { self.0.complex(o, [y, x]) diff --git a/src/field.rs b/core/src/field.rs similarity index 100% rename from src/field.rs rename to core/src/field.rs diff --git a/src/float.rs b/core/src/float.rs similarity index 100% rename from src/float.rs rename to core/src/float.rs diff --git a/src/gather.rs b/core/src/gather.rs similarity index 95% rename from src/gather.rs rename to core/src/gather.rs index a70366f..f7a8e08 100644 --- a/src/gather.rs +++ b/core/src/gather.rs @@ -28,61 +28,65 @@ const CYCLE_BLOCK: usize = 32 << 10; /// Appends `item(j)` for each `j` below `len`, as many times as its count. `counts` holds one count for all the items, or one for each item. The counts are nonnegative and sum to `total`. An item with a count of at most 4 writes four copies where its run starts. The items after it overwrite the copies beyond its count. A larger count fills its run. With a count for each item, a group of 4 items whose counts are all at most 4 writes its copies after one check for the whole group. With a check and a taken branch for each item, the loop's cost varied by about a third with where its code was placed. #[inline] pub fn repeat(out: &mut Vec, len: usize, counts: &[C], total: usize, item: impl Fn(usize) -> D) { - crate::slice::appended(out, total, |d| { - let mut k = 0; - let each = |d: &mut [MaybeUninit], k: &mut usize, x: D, n: usize| { - let x = MaybeUninit::new(x); - match d[*k..].first_chunk_mut::<4>() { Some(w) if n <= 4 => *w = [x; 4], _ => d[*k..*k + n].fill(x) } - *k += n; - }; - if let [n] = *counts { for j in 0..len { each(d, &mut k, item(j), n.key() as usize) } } else { - let count_groups = counts.as_chunks::<4>().0; - for (g, &c) in count_groups.iter().enumerate().take(len / 4) { - let j = 4 * g; - if k + 16 > total || c[0].key().max(c[1].key()).max(c[2].key()).max(c[3].key()) as u64 > 4 { - for i in 0..4 { each(d, &mut k, item(j + i), c[i].key() as usize) } - continue; - } - let w = &mut d[k..k + 16]; - let mut o = 0; - for i in 0..4 { - *w[o..].first_chunk_mut::<4>().expect("each count in the group is at most 4") = [MaybeUninit::new(item(j + i)); 4]; - o += c[i].key() as usize; + unsafe { + crate::slice::appended(out, total, |d| { + let mut k = 0; + let each = |d: &mut [MaybeUninit], k: &mut usize, x: D, n: usize| { + let x = MaybeUninit::new(x); + match d[*k..].first_chunk_mut::<4>() { Some(w) if n <= 4 => *w = [x; 4], _ => d[*k..*k + n].fill(x) } + *k += n; + }; + if let [n] = *counts { for j in 0..len { each(d, &mut k, item(j), n.key() as usize) } } else { + let count_groups = counts.as_chunks::<4>().0; + for (g, &c) in count_groups.iter().enumerate().take(len / 4) { + let j = 4 * g; + if k + 16 > total || c[0].key().max(c[1].key()).max(c[2].key()).max(c[3].key()) as u64 > 4 { + for i in 0..4 { each(d, &mut k, item(j + i), c[i].key() as usize) } + continue; + } + let w = &mut d[k..k + 16]; + let mut o = 0; + for i in 0..4 { + *w[o..].first_chunk_mut::<4>().expect("each count in the group is at most 4") = [MaybeUninit::new(item(j + i)); 4]; + o += c[i].key() as usize; + } + k += o; } - k += o; + for j in len / 4 * 4..len { each(d, &mut k, item(j), counts[j].key() as usize) } } - for j in len / 4 * 4..len { each(d, &mut k, item(j), counts[j].key() as usize) } - } - assert_eq!(k, total, "the counts must sum to `total`"); - }) + assert_eq!(k, total, "the counts must sum to `total`"); + }) + } } /// Appends `position(j)` for each position `j` whose count in `mask` is 1, in order. Every count is 0 or 1, and `total` of them are 1. Long runs cost little: a block of 64 counts that are all 0 is skipped, and one whose counts are all 1 is written in one loop. In a block that mixes both, `selected` computes only the selected items. Each count is read by its low bit, which writes every slot whatever the counts hold. #[inline] pub fn positions(out: &mut Vec, mask: &[M], total: usize, position: impl Fn(usize) -> D) { - crate::slice::appended(out, total, |d| { - // `k` slots are written. - let mut k = 0; - for (start, m) in (0..).step_by(64).zip(mask.chunks(64)) { - // A block's count is at most 64, which fits in a byte. Byte additions vectorize in the widest lanes. - let mut n = 0u8; - for x in m { n += x.key() as u8 & 1 } - let n = usize::from(n); - if n == m.len() { let run = &mut d[k..k + n]; for j in 0..n { run[j].write(position(start + j)); } } - else if n > 0 { - let (blocks, rest) = m.as_chunks::<8>(); - let mut at = k; - for (b, block) in blocks.iter().enumerate() { - let start = start + 8 * b; - at = selected(d, at, mask_bits(block), |i| position(start + usize::from(i))); + unsafe { + crate::slice::appended(out, total, |d| { + // `k` slots are written. + let mut k = 0; + for (start, m) in (0..).step_by(64).zip(mask.chunks(64)) { + // A block's count is at most 64, which fits in a byte. Byte additions vectorize in the widest lanes. + let mut n = 0u8; + for x in m { n += x.key() as u8 & 1 } + let n = usize::from(n); + if n == m.len() { let run = &mut d[k..k + n]; for j in 0..n { run[j].write(position(start + j)); } } + else if n > 0 { + let (blocks, rest) = m.as_chunks::<8>(); + let mut at = k; + for (b, block) in blocks.iter().enumerate() { + let start = start + 8 * b; + at = selected(d, at, mask_bits(block), |i| position(start + usize::from(i))); + } + let (start, last) = (start + 8 * blocks.len(), mask_bits(rest)); + selected(d, at, last, |i| position(start + usize::from(i))); } - let (start, last) = (start + 8 * blocks.len(), mask_bits(rest)); - selected(d, at, last, |i| position(start + usize::from(i))); + k += n; } - k += n; - } - assert_eq!(k, total, "`total` counts must be 1"); - }) + assert_eq!(k, total, "`total` counts must be 1"); + }) + } } /// For each mask of 8 bits, the positions of its set bits in ascending order, then zeros. @@ -903,7 +907,7 @@ impl Steps { } match axis { _ if base == Self::FILL => out.extend(std::iter::repeat_n(Self::FILL, axis.len())), - Self::Table(table) => crate::slice::appended(out, table.len(), |slots| table.write_offsets(base, slots)), + Self::Table(table) => unsafe { crate::slice::appended(out, table.len(), |slots| table.write_offsets(base, slots)) }, _ => { let len = axis.len(); for r0 in (0..len).step_by(TABLE_CHUNK) { @@ -1235,10 +1239,12 @@ pub trait Convert { fn one(&self, x: S) -> D; /// Appends the items of `s`, converted, in one contiguous pass. fn slice(&self, d: &mut Vec, s: &[S]) { - crate::slice::appended(d, s.len(), |slots| { - let slots = &mut slots[..s.len()]; - for i in 0..s.len() { slots[i].write(self.one(s[i])); } - }) + unsafe { + crate::slice::appended(d, s.len(), |slots| { + let slots = &mut slots[..s.len()]; + for i in 0..s.len() { slots[i].write(self.one(s[i])); } + }) + } } /// The items of `s`, converted, in one contiguous pass. fn all(&self, s: &[S]) -> Vec { @@ -1332,14 +1338,16 @@ pub fn interleave(out: &mut Vec, parts: &[(&[T], usize)], rows: usiz assert!(parts.iter().all(|&(items, n)| items.len() == rows * n), "each part holds one lane for every row"); let width = parts.iter().map(|&(_, n)| n).sum::(); let parts: Vec<(&[T::Unit], usize)> = parts.iter().map(|&(items, n)| (units(items), n)).collect(); - crate::slice::appended(out, rows * width, |slots| { - // SAFETY: each unit written below is a unit of an item, read through `units`. - let slots = unsafe { written_as::, MaybeUninit>(slots) }; - match *parts { - [(a, n), (b, m)] if n == m && (1..=RUN).contains(&n) => runs!(n, L => lane_pairs::(slots, [a, b])), - _ => lane_items(slots, &parts, width), - } - }) + unsafe { + crate::slice::appended(out, rows * width, |slots| { + // SAFETY: each unit written below is a unit of an item, read through `units`. + let slots = written_as::, MaybeUninit>(slots); + match *parts { + [(a, n), (b, m)] if n == m && (1..=RUN).contains(&n) => runs!(n, L => lane_pairs::(slots, [a, b])), + _ => lane_items(slots, &parts, width), + } + }) + } } /// [`interleave`]'s rows where two parts' lanes hold `L` units: each lane moves as one block, with both sources in registers. @@ -1509,10 +1517,10 @@ fn walk_axis(d: &mut Vec, s: &[T], fill: T, base: usize, axis: &Step let (width, start) = (cell.width(), d.len()); if let Steps::Stride { start: a, len: m, step } = *axis { // SAFETY: `windows` writes only units read from `s`, which hold items of `d`'s type. - runs!(cell.len, L => crate::slice::appended(d, m * width, |slots| windows::(unsafe { written_as(slots) }, units(s), base + a, step, &cell))); + runs!(cell.len, L => unsafe { crate::slice::appended(d, m * width, |slots| windows::(written_as(slots), units(s), base + a, step, &cell)) }); } else if !axis.fills() { // SAFETY, here and below: `cells_at` writes only units read from `s`, which hold items of `d`'s type. - runs!(cell.len, L => crate::slice::appended(d, axis.len() * width, |slots| cells_at::(unsafe { written_as(slots) }, units(s), base, axis, &cell))); + runs!(cell.len, L => unsafe { crate::slice::appended(d, axis.len() * width, |slots| cells_at::(written_as(slots), units(s), base, axis, &cell)) }); } else { d.resize(start + axis.len() * width, fill); runs!(cell.len, L => cells_at::(unsafe { written_as(&mut d[start..]) }, units(s), base, axis, &cell)); @@ -1534,19 +1542,21 @@ fn walk_axis(d: &mut Vec, s: &[T], fill: T, base: usize, axis: &Step Steps::Stride { start, len, step: 0 } => d.extend(std::iter::repeat_n(s[start], len)), Steps::Stride { start, len, step } => { checked_run(start, len, step, s.len()); - crate::slice::appended(d, len, |slots| { - let item = std::convert::identity( - #[inline(always)] - |i: usize, slot: &mut MaybeUninit| { - // SAFETY: both endpoints were checked; every i is within the run. - slot.write(unsafe { *s.get_unchecked(start.wrapping_add_signed(step * i as isize)) }); - }, - ); - // Fixed blocks expose independent strided loads to SLP. - let (blocks, tail) = slots.as_chunks_mut::<4>(); - for (i, block) in blocks.iter_mut().enumerate() { for (j, slot) in block.iter_mut().enumerate() { item(i * 4 + j, slot); } } - for (j, slot) in tail.iter_mut().enumerate() { item(blocks.len() * 4 + j, slot); } - }); + unsafe { + crate::slice::appended(d, len, |slots| { + let item = std::convert::identity( + #[inline(always)] + |i: usize, slot: &mut MaybeUninit| { + // SAFETY: both endpoints were checked; every i is within the run. + slot.write(*s.get_unchecked(start.wrapping_add_signed(step * i as isize))); + }, + ); + // Fixed blocks expose independent strided loads to SLP. + let (blocks, tail) = slots.as_chunks_mut::<4>(); + for (i, block) in blocks.iter_mut().enumerate() { for (j, slot) in block.iter_mut().enumerate() { item(i * 4 + j, slot); } } + for (j, slot) in tail.iter_mut().enumerate() { item(blocks.len() * 4 + j, slot); } + }) + }; } Steps::Rotated { shift, len, stride } => { walk_axis(d, s, fill, 0, &Steps::Stride { start: shift * stride, len: len - shift, step: stride as isize }, &[]); @@ -1566,11 +1576,13 @@ fn walk_axis(d: &mut Vec, s: &[T], fill: T, base: usize, axis: &Step } Steps::Table(ref t) => { if !t.take_into(d, s) { - crate::slice::appended(d, t.len(), |slots| { - t.chunks(|j0, run| { - for (slot, &o) in slots[j0..].iter_mut().zip(run) { slot.write(s.get(o).copied().unwrap_or(fill)); } + unsafe { + crate::slice::appended(d, t.len(), |slots| { + t.chunks(|j0, run| { + for (slot, &o) in slots[j0..].iter_mut().zip(run) { slot.write(s.get(o).copied().unwrap_or(fill)); } + }) }) - }) + } } } } @@ -1580,7 +1592,7 @@ fn walk_axis(d: &mut Vec, s: &[T], fill: T, base: usize, axis: &Step fn transposed(d: &mut Vec, s: &[T], m: usize, n: usize, stride: usize) { for i0 in (0..m).step_by(8) { let rows = 8.min(m - i0); - crate::slice::appended(d, rows * n, |slots| crate::linalg::transpose_into(&s[i0..], stride, slots, n, [n, rows], MaybeUninit::new)); + unsafe { crate::slice::appended(d, rows * n, |slots| crate::linalg::transpose_into(&s[i0..], stride, slots, n, [n, rows], MaybeUninit::new)) }; } } diff --git a/src/hashing.rs b/core/src/hashing.rs similarity index 76% rename from src/hashing.rs rename to core/src/hashing.rs index dcdd6a6..a73a9e5 100644 --- a/src/hashing.rs +++ b/core/src/hashing.rs @@ -1,8 +1,8 @@ -//! The hash maps and sets rustymath uses. `Hashing` is foldhash's fast `RandomState`, hashbrown's default hasher, with `hash_one` marked `#[inline(always)]`. hashbrown 0.17 hashes every key through `BuildHasher::hash_one`, and core's default body for it has no inline hint, so incremental builds can call it out of line on every lookup. +//! The hash maps and sets rustymath uses. `Hashing` is foldhash's fast hasher with its fixed seed (`FixedState`), with `hash_one` marked `#[inline(always)]`. With one seed in every process, maps hash and iterate alike between runs, and instruction counts repeat. hashbrown 0.17 hashes every key through `BuildHasher::hash_one`, and core's default body for it has no inline hint, so incremental builds can call it out of line on every lookup. use std::hash::{BuildHasher, Hash, Hasher}; #[derive(Clone, Default, Debug)] -pub struct Hashing(foldhash::fast::RandomState); +pub struct Hashing(foldhash::fast::FixedState); impl BuildHasher for Hashing { type Hasher = foldhash::fast::FoldHasher<'static>; diff --git a/src/hyperbolic.rs b/core/src/hyperbolic.rs similarity index 99% rename from src/hyperbolic.rs rename to core/src/hyperbolic.rs index 34f64a4..55bd26d 100644 --- a/src/hyperbolic.rs +++ b/core/src/hyperbolic.rs @@ -9,16 +9,19 @@ pub(crate) fn cosh(x: R) -> R { cosh_formula(&crat #[inline] pub(crate) fn tanh(x: R) -> R { tanh_formula(&crate::recipe::NativeOps::::DEFAULT, x) } +#[inline] pub(crate) fn sinh_formula + ?Sized>(o: &O, x: O::V) -> O::V { let one = o.number(F::one()); let h = crate::exp::minus_one_formula(o, o.magnitude(x), o.negate(one)); let result = o.multiply(h, o.add(one, o.divide(one, o.add(o.add(h, h), one)))); o.select(o.less(x, o.number(F::zero())), o.negate(result), result) } +#[inline] pub(crate) fn cosh_formula + ?Sized>(o: &O, x: O::V) -> O::V { let e = crate::exp::scaled_formula(o, o.magnitude(x), o.number(-F::one())); o.add(e, o.divide(o.number(F::narrow(0.25)), e)) } +#[inline] pub(crate) fn tanh_formula + ?Sized>(o: &O, x: O::V) -> O::V { let top = o.number(F::narrow(if o.width() == crate::buffer::FloatWidth::F64 { 20.0 } else { 9.0 })); let a = o.magnitude(x); diff --git a/src/integer.rs b/core/src/integer.rs similarity index 100% rename from src/integer.rs rename to core/src/integer.rs diff --git a/src/interrupt.rs b/core/src/interrupt.rs similarity index 100% rename from src/interrupt.rs rename to core/src/interrupt.rs diff --git a/src/lambert.rs b/core/src/lambert.rs similarity index 98% rename from src/lambert.rs rename to core/src/lambert.rs index a8a311f..a28ec9b 100644 --- a/src/lambert.rs +++ b/core/src/lambert.rs @@ -24,7 +24,7 @@ fn branch_offset(x: f64, epsilon: f64) -> f64 { } pub(crate) fn real_domain(dtype: DType, x: Range) -> Option<[Range; 1]> { - let width = crate::operation::numpy_float(dtype).max(FloatWidth::F32); + let width = dtype.numpy_float().max(FloatWidth::F32); (x.lo >= 0.0 || branch_offset::(x.lo, width.epsilon()) >= 0.0).then_some([x]) } /// W0(x), computed in f64. x below -1/e gives NaN. The float nearest -1/e lies below it at f32 and f64, so an x below -1/e by less than half an ulp of x gives -1. diff --git a/src/layout.rs b/core/src/layout.rs similarity index 100% rename from src/layout.rs rename to core/src/layout.rs diff --git a/core/src/lib.rs b/core/src/lib.rs new file mode 100644 index 0000000..8fe346f --- /dev/null +++ b/core/src/lib.rs @@ -0,0 +1,209 @@ +//! Maths in plain Rust. Its float functions take `f16`, `f32` and `f64`, each within 4 units in the last place (ulps) of the correctly rounded result. The README lists the functions with wider bounds. The crate also has APL's tolerant comparison, exact rationals, linear algebra, number theory and polynomials. Its Python functions evaluate eagerly and return Rustymath arrays or Python scalars. +#[macro_use] +pub mod slice; +mod arc; +mod arith; +#[macro_use] +pub mod buffer; +pub mod agreement; +pub mod array; +pub mod characters; +mod complex; +pub mod convert; +mod dd; +pub mod derivative; +pub mod distribution; +pub mod domain; +pub mod error; +pub mod events; +pub mod exact; +mod exp; +pub mod facts; +mod field; +mod float; +pub mod gather; +pub mod hashing; +mod hyperbolic; +pub mod integer; +pub mod interrupt; +mod lambert; +pub mod layout; +pub mod linalg; +mod ln; +pub mod memory; +pub mod number_theory; +pub mod optimize; +pub mod parallel; +mod parts; +pub mod polynomial; +mod pow; +pub mod random; +pub mod recipe; +pub mod reduce; +pub mod search; +pub mod signal; +mod special; +mod sqrts; +pub mod stats; +pub mod structure; +mod table; +pub mod tolerant; +mod trig; + +pub use arc::{RealAcos, RealAsin}; +pub use arith::{ + Add, BitAnd, BitOr, Bitwise, Divide, LeftShift, MathishDivide, Multiply, Nand, Negate, Nor, Not, PiTimes, Reciprocal, RightShift, Subtract, bitwise, +}; +pub use complex::{Cis, divide}; +pub use exp::{Exp, Expm1}; +pub use float::{FUSED, Float, Native}; +pub use hyperbolic::{RealAcosh, RealAtanh}; +pub use ln::{Ln, Log1p, Log2, Log10, Logb, MathishLogb, RealLn}; +pub use parts::{Abs, Angle, Choose, Clip, Conj, Imag, Isfinite, Isinf, Isnan, Real, Rint, Round, Scale, Sign, Trunc}; +pub use pow::{Exponent, Pow, RealPow, Root}; +pub use slice::{Arg, Binary, CHUNK, Flipped, KernelItem, Results, Ternary, Unary, Wrapped, pairs}; +pub use special::{BetaincAt, Erfc, Erfcinv, Erfinv}; +pub use sqrts::{RealSqrt, RealSqrt1mx2, RealSqrtm1mx2, RealSqrtx2m1, Sqrt, Sqrtm1mx2}; +pub use trig::{Cos, Sin, Tan}; +/// The crates whose types compact buffers hold, which the exported width macros name. +pub use {half, num_complex}; + +/// How many items a function handles between checks for an interrupt ([`interrupt::check`]), which keeps the check out of the loop over items. +pub const BETWEEN_CHECKS: usize = 1024; + +/// Declares each function at the crate root, where it multiplies and adds as the target's `FUSED` says, and in `fused` and `unfused`, which choose for themselves. Each entry names the module whose function of the same name implements it, generic over `R: Native` and `const FUSED: bool`. Every wrapper is `#[inline(always)]`, because LLVM leaves a large function such as `gamma` out of line in a caller's loop, and the call stops that loop vectorizing. An entry marked `as` also declares a function object of that name, for the facts table's formula. An object with forms, such as [`Pow`] or [`Atan`], is written with `forms!` beside its function. `function_doc!(ln)` gives an entry's doc by its name. The Python package's functions take their docs from it. +macro_rules! functions { + ($($(#[doc = $doc:literal])* $name:ident($($x:ident),+) in $module:ident $(as $object:ident)?;)+) => { + $( + $(#[doc = $doc])* + #[inline(always)] + pub fn $name($($x: F),+) -> F { functions!(@call FUSED, $module, $name, ($($x),+)) } + )+ + $(functions!(@object $($object)?; $name, $module, ($($x),+));)+ + /// Each function's doc by its name, as `function_doc!(ln)`, with its lines joined by newlines. + #[macro_export] + macro_rules! function_doc { + $(($name) => { concat!($($doc, "\n"),*) };)+ + } + /// rustymath's functions with `mla` fused, whatever the target's `FUSED`. Code compiled with FMA enabled for itself, such as an AVX2 version of a loop chosen when the program runs, calls these on a target whose baseline lacks FMA. + pub mod fused { + use crate::Float; + $( + #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), "), with `mla` fused.")] + #[inline(always)] + pub fn $name($($x: F),+) -> F { functions!(@call true, $module, $name, ($($x),+)) } + )+ + } + /// rustymath's functions with `mla` unfused, whatever the target's `FUSED`. + pub mod unfused { + use crate::Float; + $( + #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), "), with `mla` unfused.")] + #[inline(always)] + pub fn $name($($x: F),+) -> F { functions!(@call false, $module, $name, ($($x),+)) } + )+ + } + }; + (@object; $name:ident, $module:ident, ($($x:ident),+)) => {}; + (@object $object:ident; $name:ident, $module:ident, ($($x:ident),+)) => { + #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), ") as a function object.")] + #[derive(Clone, Copy, Debug)] + pub struct $object; + }; + (@call $fused:expr, $module:ident, $name:ident, ($($x:ident),+)) => { F::from_working(crate::$module::$name::<_, { $fused }>($($x.working()),+)) }; +} + +functions! { + /// e^x. `exp(∞)` is ∞, `exp(-∞)` is 0, and `exp(NaN)` is NaN. + exp(x) in exp; + /// e^x - 1, without the loss of digits near 0 that `exp(x) - 1` has. + expm1(x) in exp; + /// The natural logarithm of x. `ln(0)` is -∞, `ln(∞)` is ∞, and a negative real x or NaN gives NaN. + ln(x) in ln; + /// ln(1 + x), without the loss of digits near 0 that `ln(1 + x)` has. `log1p(-1)` is -∞. + log1p(x) in ln; + /// The logarithm of x to base 2. It is k exactly at 2^k. + log2(x) in ln; + /// The logarithm of x to base 10. It is k exactly at 10^k, and at the float nearest 10^k where 10^k isn't a float. + log10(x) in ln; + /// The logarithm of x to base b, ln x / ln b. Base 2 and base 10 take `log2` and `log10`, which are exact at powers of their base. + logb(b, x) in ln; + /// sin x. `sin(±0)` is ±0, and `sin(±∞)` and `sin(NaN)` are NaN. Beyond a magnitude of 2^23 at f64 or 2^20 at f32, x takes a slower, exact reduction. + sin(x) in trig; + /// cos x. `cos(±0)` is 1, and `cos(±∞)` and `cos(NaN)` are NaN. Beyond the magnitude that `sin` gives, x takes the same exact reduction. + cos(x) in trig; + /// tan x. `tan(±0)` is ±0, and `tan(±∞)` and `tan(NaN)` are NaN. Beyond the magnitude that `sin` gives, x takes the same exact reduction. + tan(x) in trig; + /// asin x. A real x gives a result from -π/2 to π/2 and NaN beyond a magnitude of 1. + asin(x) in arc as Asin; + /// acos x. A real x gives a result from 0 to π and NaN beyond a magnitude of 1. + acos(x) in arc as Acos; + /// atan x. A real x gives a result from -π/2 to π/2. + atan(x) in arc as Atan; + /// The angle of the point (x, y) from the positive x axis, from -π to π, with y first as in C's `atan2`. Zeros and infinities give C99's angles: atan2(±0, -0) is ±π, and atan2(±∞, ±∞) is an odd multiple of ±π/4. + atan2(y, x) in arc as Atan2; + /// sinh x. It stays finite up to where sinh itself overflows, beyond the point where e^x does. + sinh(x) in hyperbolic as Sinh; + /// cosh x. It stays finite up to where cosh itself overflows, beyond the point where e^x does. + cosh(x) in hyperbolic as Cosh; + /// tanh x. + tanh(x) in hyperbolic as Tanh; + /// asinh x. + asinh(x) in hyperbolic as Asinh; + /// acosh x. A real x below 1 gives NaN. + acosh(x) in hyperbolic as Acosh; + /// atanh x. ±1 gives ±∞, and a real x of magnitude beyond 1 gives NaN. + atanh(x) in hyperbolic as Atanh; + /// x^y. Special values follow IEEE 754's `pow`: x^0 and 1^y are 1, even for NaN, and a negative x has a power only for a whole y. + pow(x, y) in pow; + /// The real nth root of y, computed without rounding 1/n first. A negative y has one only for an odd whole n. n = 0, an infinite n and NaN give NaN. + root(n, y) in pow; + /// The cube root of x. + cbrt(x) in pow as Cbrt; + /// √(x² + y²), with no overflow or underflow in between. An infinite argument gives ∞, even beside NaN. + hypot(x, y) in pow as Hypot; + /// √(1 - x²), without the digits that squaring x first loses near |x| = 1. A real x of magnitude beyond 1 gives NaN. + sqrt1mx2(x) in sqrts as Sqrt1mx2; + /// √(1 + x²), with no overflow where x² would overflow. + sqrt1px2(x) in sqrts as Sqrt1px2; + /// √(x - 1) √(x + 1): √(x² - 1) with the sign of a real x, without the digits that squaring x first loses near |x| = 1. A real x of magnitude below 1 gives NaN. + sqrtx2m1(x) in sqrts as Sqrtx2m1; + /// Γ(x). Γ(±0) is ±∞, a negative whole x and -∞ give NaN, and Γ(∞) is ∞. + gamma(x) in special as Gamma; + /// Γ(x + 1), which is x! for a whole x from 0. + factorial(x) in special as Factorial; + /// ln |Γ(x)|. It is ∞ at 0, at the negative whole numbers and at ±∞. Near its zeros at 1 and 2 its error is within 4 ulps of 2, not of the result. + lgamma(x) in special as Lgamma; + /// ψ(x), the digamma function: the derivative of ln Γ(x), Γ′(x)/Γ(x). ψ(+0) is -∞, ψ(-0) is ∞, a negative whole x and -∞ give NaN, and ψ(∞) is ∞. Near its zeros its error is within 4 ulps of 1, not of the result, and below -20 within 4 ulps of ln(1 - x). + digamma(x) in special as Digamma; + /// ψ⁽ⁿ⁾(x), the polygamma function: the nth derivative of ψ, for a whole n from 0 to 170. For n from 1, x from 0 down gives NaN, as do a fractional or negative n and an n above 170. + polygamma(n, x) in special as Polygamma; + /// B(a, b) = Γ(a) Γ(b)/Γ(a + b), the beta function. A pole of Γ(a) or Γ(b) gives NaN, and a pole of Γ(a + b) alone gives 0. + beta(a, b) in special as Beta; + /// C(n, k), the binomial coefficient. Whole n and k from 0 multiply out exactly while the result times |n| + k is a whole number the type holds, and a negative whole n gives (-1)^k C(k - n - 1, k). Other n with a whole k from 0 to 64 multiply out n(n - 1)…(n - k + 1)/k!, within 2k ulps. Other real arguments take Γ(n + 1)/(Γ(k + 1) Γ(n - k + 1)) in double words, within 4 ulps in tests against mpmath, for large n and negative arguments too. + binom(n, k) in special as Binom; + /// ∂C(n, k)/∂n, the binomial coefficient's slope in n. For a whole k from 0 to 64 it is the slope of the polynomial n(n - 1)…(n - k + 1)/k! by the product rule, within 2k ulps. Other arguments take C(n, k)(ψ(n + 1) - ψ(n - k + 1)), with the ψ difference in double words where both arguments are from 10 up, with the polynomial's slope where a whole n below a whole k puts those ψ at a pole. Other poles of ψ give NaN. + binom_dn(n, k) in special as BinomDn; + /// ∂C(n, k)/∂k, the binomial coefficient's slope in k: C(n, k)(ψ(n - k + 1) - ψ(k + 1)). At a whole n from 0 below a whole k, it is the negative of `binom_dn`. Other poles of ψ, as at a negative whole k, give NaN. For a whole k from 0 to 64, its error is within 2k ulps of C(n, k) times ψ(n - k + 1) - ψ(k + 1), plus the error of that difference, so near n = 2k, where the two ψ cancel, its relative error has no bound. + binom_dk(n, k) in special as BinomDk; + /// The error function, 2/√π times the integral of e^-t² from 0 to x. + erf(x) in special as Erf; + /// 1 - erf(x), without the loss of digits for large x that `1 - erf(x)` has. + erfc(x) in special; + /// The x with erf(x) = p. ±1 gives ±∞, and a magnitude beyond 1 gives NaN. + erfinv(p) in special; + /// The x with erfc(x) = q. 0 gives ∞, 2 gives -∞, and q outside 0 to 2 gives NaN. + erfcinv(q) in special; + /// W0(x), the principal branch of the Lambert W function: the w of at least -1 with w e^w = x. A real x below -1/e gives NaN, except within half an ulp of -1/e, where W0 is -1. + lambertw(x) in lambert as LambertW; + /// The regularized lower incomplete gamma function P(a, x): the integral of t^(a-1) e^-t from 0 to x, divided by Γ(a). a must be positive and x nonnegative, or the result is NaN. + gammainc(a, x) in special as Gammainc; + /// The regularized upper incomplete gamma function Q(a, x) = 1 - P(a, x), without the loss of digits that `1 - gammainc(a, x)` has where Q is small. + gammaincc(a, x) in special as Gammaincc; + /// The x with P(a, x) = p, for p from 0 to 1. + gammaincinv(a, p) in special as Gammaincinv; + /// The regularized incomplete beta function I_x(a, b): the integral of t^(a-1) (1-t)^(b-1) from 0 to x, divided by B(a, b). a and b must be positive and x from 0 to 1, or the result is NaN. + betainc(a, b, x) in special as Betainc; + /// The x with I_x(a, b) = p, for p from 0 to 1. + betaincinv(a, b, p) in special as Betaincinv; +} diff --git a/core/src/linalg.rs b/core/src/linalg.rs new file mode 100644 index 0000000..f2fd060 --- /dev/null +++ b/core/src/linalg.rs @@ -0,0 +1,1444 @@ +//! Inner products of row-major matrices. The inner product `f.g` of an `m`×`k` matrix `a` and a `k`×`n` matrix `b` is the `m`×`n` matrix whose item at row `i` and column `j` folds `f` over `g(a[i][p], b[p][j])` for each `p`. `+.×` is the matrix product. The specialized pairs have associative `f`, and their loops combine its steps in whatever order suits them. A float result can therefore differ in its last bits from a fold taken in order. [`inner_with`] runs other native pairs through bounded term blocks and a caller's right fold, which preserves the association of non-associative functions. +//! +//! The loops follow GotoBLAS and BLIS. Packing copies a block of each argument into a contiguous buffer sized for a cache level. The blocking reuses each buffer while it is in cache. A micro-kernel computes a small tile of the result in registers, one step of `p` at a time. A micro-kernel in plain Rust vectorizes through LLVM's SLP vectorizer, which the builds of rustymath and basedpl turn on at opt-level 2. It vectorizes only tiles of up to 32 items, too few to keep the multiply-add units busy. On AArch64, in wasm and on x86-64 with AVX2 and FMA, the float pairs run micro-kernels on explicit vectors instead. Their maximum and minimum are AArch64's `fmaxnm`, wasm's `pmax` and x86-64's `maxpd`, with the running value as the argument each gives beside NaN. A tile starts at `-∞` or `∞`, so its running value is never NaN and a NaN term leaves it unchanged, as `Core::maximum` does. Targets without these vectors run the plain kernels. +//! +//! A product large enough splits its rows across the threads that [`crate::parallel::set_threads`] sets. On macOS, Accelerate computes real and complex plus-times products instead. `Tiled` also sends the decompositions' triangular solves, SVDs and Hermitian eigen decompositions to it, and the general eigen decompositions of real matrices from `accelerate::GEEV_FROM` rows. macOS builds with the `native` feature also compile rustymath's own kernels, and `RUSTYMATH_NATIVE=1` selects them for the whole process. +//! +//! The private module `factor` holds decompositions of real and complex matrices, whose blocked updates run on these loops. This module re-exports its public items, and [`Scalar`], the trait of the item types they take. [`transpose_into`] transposes a matrix through 8×8 tiles. +use crate::{ + Float, agreement::{Mapping, Walked, expanded_items}, buffer::{Element, Source, read_all}, convert, error::Error, float::{FUSED, sealed::Core}, + parallel, slice +}; +use num_complex::Complex; +use num_traits::{One, Zero}; +use std::{borrow::Cow, mem::MaybeUninit, sync::LazyLock}; + +#[cfg(accelerate)] +mod accelerate; +mod factor; +mod inner; +mod scalar; +// The platform's own BLAS and LAPACK, which every `Tiled` type implements: Accelerate on macOS, and none elsewhere. +#[cfg(accelerate)] +pub(crate) use accelerate::Accelerated as Vendor; +#[cfg(not(accelerate))] +pub(crate) trait Vendor {} +#[cfg(not(accelerate))] +impl Vendor for T {} + +/// Whether this process selects rustymath's own kernels where Accelerate is also compiled in: in macOS builds with the `native` feature, when `RUSTYMATH_NATIVE=1`. It's read once. +fn switched() -> bool { + static NATIVE: LazyLock = LazyLock::new(|| std::env::var_os("RUSTYMATH_NATIVE").is_some_and(|v| v == "1")); + cfg!(all(accelerate, native)) && *NATIVE +} + +/// Whether Accelerate serves this process's products, triangular solves and decompositions. +pub(crate) fn accelerated() -> bool { cfg!(accelerate) && !switched() } + +/// `$vendor` where Accelerate serves this process, and `$native` where rustymath's own kernels do. Each compiles only where its kernels do. +macro_rules! dispatch { + ($vendor:expr, $native:expr) => {{ + #[cfg(all(accelerate, native))] + let r = if accelerated() { $vendor } else { $native }; + #[cfg(all(accelerate, not(native)))] + let r = $vendor; + #[cfg(not(accelerate))] + let r = $native; + r + }}; +} +/// `$vendor` where Accelerate serves this process and the matrix's order `$n` is at least `$from`, and `$native` otherwise. Accelerate's LAPACK pays a fixed cost for each call, which makes the native solver faster for small matrices. Use it only in a function body: it returns `$vendor`'s result. +macro_rules! from_order { + ($n:expr, $from:expr, $vendor:expr, $native:expr) => {{ + #[cfg(accelerate)] + if $n >= $from && accelerated() { + return $vendor; + } + $native + }}; +} +pub mod exact; +pub use factor::*; +pub use inner::inner_with; +pub use scalar::Scalar; + +/// The `n`×`n` identity matrix. +pub(crate) fn identity(n: usize) -> Vec { + let mut x = Vec::new(); + identity_into(&mut x, n); + x +} + +/// [`identity`], written into `x` in place of its old items. +pub(crate) fn identity_into(x: &mut Vec, n: usize) { + x.clear(); + x.extend(std::iter::repeat_with(T::zero).take(n * n)); + for i in 0..n { x[i * (n + 1)] = T::one() } +} + +/// Writes the transpose of the `rows`×`cols` matrix in `src`, whose rows start `ls` items apart, into `dst`, whose rows start `ld` items apart. Each item passes through `f`: `dst[c * ld + r]` is `f(src[r * ls + c])`. Whole 8×8 tiles go through [`tiles`], and the items past the last whole tile one at a time. +#[inline] +pub fn transpose_into(src: &[S], ls: usize, dst: &mut [D], ld: usize, [rows, cols]: [usize; 2], f: impl Fn(S) -> D) { + if rows == 0 || cols == 0 { return; } + assert!(src.len() >= (rows - 1) * ls + cols && dst.len() >= (cols - 1) * ld + rows, "both matrices lie in their slices"); + let (whole_rows, whole_cols) = (rows / 8 * 8, cols / 8 * 8); + // Each write moves 16 bytes where the items allow, which is at most 8 items, as the registers that the compiler shuffles hold. + match 16 / size_of::().max(1) { + 8.. => tiles::<8, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), 4.. => tiles::<4, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), + 2.. => tiles::<2, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), _ => tiles::<1, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f) + } + if whole_cols < cols { for r in 0..whole_rows { for c in whole_cols..cols { dst[c * ld + r] = f(src[r * ls + c]) } } } + for r in whole_rows..rows { for c in 0..cols { dst[c * ld + r] = f(src[r * ls + c]) } } +} + +/// The whole 8×8 tiles of a transpose, the first `rows` rows and `cols` columns of each matrix, which its caller checked lie in their slices. Each tile reads `P` of its runs of 8 adjacent items at a time, and writes `P` adjacent items of each result run, one from each run read. The compiler shuffles the runs' lanes in registers, as `zip` instructions do for two runs of 8-byte items. +#[inline(always)] +fn tiles(src: &[S], ls: usize, dst: &mut [D], ld: usize, [rows, cols]: [usize; 2], f: &impl Fn(S) -> D) { + let (from, to) = (src.as_ptr(), dst.as_mut_ptr()); + for r0 in (0..rows).step_by(8) { + for c0 in (0..cols).step_by(8) { + for i in (0..8).step_by(P) { + // SAFETY: the tile's rows and result rows lie in `src` and `dst`, as the caller checked. + let read = |k: usize| unsafe { from.add((r0 + i + k) * ls + c0).cast::<[S; 8]>().read() }; + let mut runs = [read(0); P]; + for k in 1..P { runs[k] = read(k) } + for j in 0..8 { let out = unsafe { &mut *to.add((c0 + j) * ld + r0 + i).cast::<[D; P]>() }; for k in 0..P { out[k] = f(runs[k][j]) } } + } + } + } +} + +/// An inner product `f.g` at item type `T`. Each pair object implements it at the item types listed here. Its loops compile in rustymath, as slice forms do. +pub trait Inner { + /// The type of each result. + type Out: Copy + Default; + /// The `m`×`n` inner product of `a`, an `m`×`k` matrix, and `b`, a `k`×`n` matrix, both row-major, written row-major into `out`, which holds `m`×`n` items. `size` is `[m, k, n]`. Where `k` is 0, each result is the identity of `f`. An integer result that doesn't fit its type panics, where `checked_into` gives `false`. + fn inner_into(&self, a: &[T], b: &[T], size: [usize; 3], out: &mut [Self::Out]); + /// `inner_into`'s results, as a new row-major matrix. + fn inner(&self, a: &[T], b: &[T], size @ [m, _, n]: [usize; 3]) -> Vec { + let mut c = filled(m * n, Self::Out::default()); + self.inner_into(a, b, size, &mut c); + c + } + /// `inner_into`, or `false` where an integer result doesn't fit its type, for callers that keep integers exact or report overflow. Where it gives `true`, `out` holds the exact results. Floats always give `true`. + #[inline] + fn checked_into(&self, a: &[T], b: &[T], size: [usize; 3], out: &mut [Self::Out]) -> bool { self.inner_into(a, b, size, out); true } + /// `inner` of the matrices that `a` and `b` give in the orientations given, with `size` `[m, k, n]` for those matrices. A transposed `a` holds a `k`×`m` matrix, row-major, and gives its `m`×`k` transpose. This default copies a transposed operand into its orientation. Float and complex `PlusTimes` read it in place. + fn inner_ops(&self, a: &[T], b: &[T], [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3]) -> Vec where T: Copy { + self.inner(&oriented(a, oa, [m, k]), &oriented(b, ob, [k, n]), [m, k, n]) + } + /// `inner_mapped_into`'s results, as a new row-major matrix. + fn inner_mapped(&self, a: Mapped<'_, T>, b: Mapped<'_, T>, orientations: [Orientation; 2], size @ [m, _, n]: [usize; 3]) -> Vec where T: Element + { + let mut out = filled(m * n, Self::Out::default()); + self.inner_mapped_into(a, b, orientations, size, &mut out); + out + } + /// `inner_into` of operands whose items come through mappings, as [`Mapped`] describes. This default reads each operand whole and copies transposed operands into their orientations. Float and complex `PlusTimes` read an operand that isn't held in order one block of its stored rows at a time and multiply each block as it's read. Only one block of that operand then exists at once. + fn inner_mapped_into(&self, a: Mapped<'_, T>, b: Mapped<'_, T>, [oa, ob]: [Orientation; 2], size @ [m, k, n]: [usize; 3], out: &mut [Self::Out]) where + T: Element { + let (mut ab, mut bb) = (Vec::new(), Vec::new()); + self.inner_into(&oriented(a.read(0, m * k, &mut ab), oa, [m, k]), &oriented(b.read(0, k * n, &mut bb), ob, [k, n]), size, out) + } +} + +fn oriented(x: &[T], o: Orientation, [rows, cols]: [usize; 2]) -> std::borrow::Cow<'_, [T]> { + match o { + Orientation::Normal => std::borrow::Cow::Borrowed(x), Orientation::Transpose => (0..rows * cols).map(|ij| x[ij % cols * rows + ij / cols]).collect() + } +} + +/// `+.×`, the matrix product, at `i64`, `f32` and `f64`, and complex numbers with parts of either type. Native float steps use fused multiply-add where the target has one; Accelerate computes them on macOS. Nonfinite complex results need not match scalar arithmetic. Native complex products form each part from two real products of the parts, as scalar multiplication does. A small imaginary part beside a large real part keeps its own relative accuracy. +#[derive(Default)] +pub struct PlusTimes; +/// `⌈.+`, the max-plus product of longest-path problems, at `i64`, `f32` and `f64`. The module doc says how each target treats a NaN term. +#[derive(Default)] +pub struct MaxPlus; +/// `⌊.+`, the min-plus product of shortest-path problems, at `i64`, `f32` and `f64`. Its minimum treats NaN as `MaxPlus`'s maximum does. +#[derive(Default)] +pub struct MinPlus; +/// `∨.∧` on Booleans: whether some `p` has both `a[i][p]` and `b[p][j]`. +#[derive(Default)] +pub struct OrAnd; +/// `∧.∨` on Booleans: whether every `p` has `a[i][p]` or `b[p][j]`. +#[derive(Default)] +pub struct AndOr; +/// `+.=`, at `i16`, `i32`, `i64`, `f32` and `f64`: the number of `p` where `a[i][p]` equals `b[p][j]`. The count has the width of the items, as `i16`, `i32` or `i64`, so an `i16` count needs `k` below 2^15 and an `i32` count below 2^31. Floats compare exactly, so NaN equals nothing. +#[derive(Default)] +pub struct PlusEqual; +/// `+.≠`, at `i16`, `i32`, `i64`, `f32` and `f64`: the number of `p` where `a[i][p]` differs from `b[p][j]`, with `+.=`'s count type. +#[derive(Default)] +pub struct PlusNotEqual; + +/// A native inner-product pair. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Product { PlusTimes, MaxPlus, MinPlus, OrAnd, AndOr, PlusEqual, PlusNotEqual } + +impl Product { pub fn compares_exactly(self) -> bool { matches!(self, Self::PlusEqual | Self::PlusNotEqual) } } + +/// Layout and reads of an inner product. Contracted axes follow output axes in `iteration`. +#[derive(Clone, Debug)] +pub struct ProductPlan { pub layout: crate::layout::Layout, pub iteration: crate::layout::Layout, pub mappings: [Mapping; 2], pub contracted: usize } + +#[derive(Clone, Copy, Debug)] +pub struct MatrixProduct { pub frame_rank: usize, pub row_rank: usize, pub size: [usize; 3] } +impl MatrixProduct { + fn of(shape: &[usize], owners: &[u8]) -> Option { + let frame_rank = owners.iter().take_while(|&&owner| owner == 3).count(); + let row_rank = owners[frame_rank..].iter().take_while(|&&owner| owner == 1).count(); + if !owners[frame_rank + row_rank..].iter().all(|&owner| owner == 2) { return None; } + let sizes = [&shape[frame_rank..frame_rank + row_rank], &shape[owners.len()..], &shape[frame_rank + row_rank..owners.len()]] + .map(|shape| crate::layout::element_count(shape).ok()); + match sizes { [Some(m), Some(k), Some(n)] => Some(Self { frame_rank, row_rank, size: [m, k, n] }), _ => None } + } +} + +impl ProductPlan { + /// The number of pairings each result folds: the product of the contracted axes' lengths. + pub fn contracted_len(&self) -> usize { let shape = self.iteration.shape(); crate::layout::count(&shape[shape.len() - self.contracted..]) } + pub fn folded(iteration: crate::layout::Layout, mappings: [Mapping; 2], contracted: usize) -> Result { + let rank = iteration.shape().len(); + if contracted > rank { return Err(Error::new(crate::error::Kind::Rank, "too many contracted axes")); } + let layout = iteration.axes(0..rank - contracted); + Ok(Self { layout, iteration, mappings, contracted }) + } + + /// The matrix product that runs this plan, from the iteration axes each operand steps along: its read's steps where the read reaches its items positionally, and otherwise its `mappings` entry's, which says which items it reads. An axis that no read steps along, as a broadcast gives, takes the class of the operands whose free axis `mappings` makes it, and an axis of one item takes the class of the axis before it. `None` where the axes don't fall into frames, rows and columns, in that order. + pub fn matrix_for(&self, inputs: [crate::agreement::Read<'_>; 2]) -> Option { + let shape = self.iteration.shape(); + let mut owners = vec![0; shape.len() - self.contracted]; + let mut nominal = owners.clone(); + for (side, input) in inputs.iter().enumerate() { + let plan = input.plan.remapped(self.mappings[side].clone(), self.iteration.count(), crate::agreement::Missing::Invalid); + let map = + crate::agreement::Read { sources: input.sources, plan: &plan }.positional(shape).map_or_else(|| self.mappings[side].clone(), |(_, map)| map); + let (_, steps) = map.affine(shape)?; + for (axis, owner) in owners.iter_mut().enumerate() { if steps[axis] != 0 { *owner |= 1 << side; } } + if let Some((_, free)) = self.mappings[side].affine(shape) { + for (named, &step) in nominal.iter_mut().zip(&free) { if step != 0 { *named |= 1 << side; } } + } + } + // An axis of one item adds nothing to any size, so any class that keeps the order serves. + let mut last = 3; + for ((owner, named), &len) in owners.iter_mut().zip(nominal).zip(shape) { + if *owner == 0 { *owner = if len == 1 { last } else if named != 0 { named } else { 3 }; } + last = *owner; + } + MatrixProduct::of(shape, &owners) + } + pub fn dot([left, right]: [&crate::layout::Layout; 2]) -> Result { + let rank = left.shape().len(); + if rank == 0 || right.shape().is_empty() { return Err(Error::new(crate::error::Kind::Rank, "an inner product needs vectors or matrices")); } + Self::paired([left, right], &[(rank - 1, 0)], None) + } + pub fn matrices(inputs: [&crate::layout::Layout; 2], frame: usize) -> Result { + let (contracted, destinations) = Self::matrix_axes(inputs, frame)?; + Self::paired(inputs, &[contracted], Some(destinations)) + } + /// The contracted pair and the destinations of each input's remaining axes, in source order. + pub fn matrix_axes([left, right]: [&crate::layout::Layout; 2], frame: usize) -> Result<((usize, usize), [Vec; 2]), Error> { + use crate::agreement::Agreement; + let [lr, rr] = [left, right].map(|l| l.shape().len()); + if lr <= frame || rr <= frame { return Err(Error::new(crate::error::Kind::Rank, "a matrix product needs vector or matrix cells")); } + let [lc, rc] = [(lr - frame).min(2), (rr - frame).min(2)]; + let [lf, rf] = [left.axes(0..lr - lc), right.axes(0..rr - rc)]; + let [mut xa, mut ya] = Agreement::axis_pairs(&lf, &rf); + let first = xa.iter().chain(&ya).max().map_or(0, |a| a + 1); + if lc == 2 { xa.push(first); } + if rc == 2 { ya.push(first + usize::from(lc == 2)); } + Ok(((lr - 1, rr - rc), [xa, ya])) + } + /// Contracts `pairs`; optional destinations pair the remaining axes, in each input's axis order. Without destinations, remaining axes concatenate left then right. + pub fn paired(inputs @ [left, right]: [&crate::layout::Layout; 2], pairs: &[(usize, usize)], destinations: Option<[Vec; 2]>) -> Result { + use crate::{agreement::Agreement, error::Kind, layout::{element_count, generated_len}}; + let contracted: [Vec; 2] = [pairs.iter().map(|p| p.0).collect(), pairs.iter().map(|p| p.1).collect()]; + for (input, axes) in inputs.iter().zip(&contracted) { + if axes.iter().enumerate().any(|(i, a)| *a >= input.shape().len() || axes[..i].contains(a)) { + return Err(Error::new(Kind::Rank, "contracted axes must be distinct axes of their inputs")); + } + } + for &(a, b) in pairs { + let (x, y) = (left.shape()[a], right.shape()[b]); + if x != y { + return Err(Error::new(Kind::Length, format!("an inner product needs matching contracted lengths, not {x} and {y}"))); + } + Mapping::contract(left, a, right, b).map_err(|_| Error::new(Kind::Length, "an inner product needs the same keys on its contracted axes"))?; + } + let free: [Vec; 2] = std::array::from_fn(|i| (0..inputs[i].shape().len()).filter(|a| !contracted[i].contains(a)).collect()); + let free_layouts = std::array::from_fn::<_, 2, _>(|i| inputs[i].axes(free[i].iter().copied())); + let total = free[0].len() + free[1].len(); + let destinations = destinations.unwrap_or_else(|| [(0..free[0].len()).collect(), (free[0].len()..total).collect()]); + for (axes, positions) in free.iter().zip(&destinations) { + if axes.len() != positions.len() || positions.iter().enumerate().any(|(i, p)| *p >= total || positions[..i].contains(p)) { + return Err(Error::new(Kind::Rank, "free axes need distinct output destinations")); + } + } + let agreement = Agreement::mapped(&free_layouts[0], &free_layouts[1], &destinations[0], &destinations[1], false)?; + if agreement.left.fills() || agreement.right.fills() { return Err(Error::new(Kind::Length, "keyed frames need a cell that an argument lacks")); } + let layout = agreement.layout; + generated_len(layout.shape())?; + let first = layout.shape().len(); + let iteration = layout.concat(&left.axes(contracted[0].iter().copied())); + let items = element_count(iteration.shape())?; + let mappings = std::array::from_fn(|i| { + let mut axes = vec![0; inputs[i].shape().len()]; + for (&a, &p) in free[i].iter().zip(&destinations[i]) { axes[a] = p; } + for (p, &a) in contracted[i].iter().enumerate() { axes[a] = first + p; } + Mapping::new(inputs[i], &axes, &iteration, items) + }); + Ok(Self { layout, iteration, mappings, contracted: pairs.len() }) + } +} + +/// The scalar operations of a pair at item type `T`. +trait Pair { + type Out: Copy + Send; + /// The identity of `f`: the result of folding no steps. + const UNIT: Self::Out; + fn g(a: T, b: T) -> Self::Out; + fn f(x: Self::Out, y: Self::Out) -> Self::Out; + /// `f(acc, g(a, b))`, which `+.×` computes as one fused multiply-add. + #[inline(always)] + fn step(acc: Self::Out, a: T, b: T) -> Self::Out { Self::f(acc, Self::g(a, b)) } +} + +/// `Pair` impls, one for each entry: the object, the item type, the result type and `UNIT`, then `g`, `f` and an optional fused `step`. +macro_rules! pairs { + ($($obj:ty: $t:ty => $out:ty = $unit:expr, |$a:ident, $b:ident| $g:expr, |$x:ident, $y:ident| $f:expr $(, |$c:ident, $p:ident, $q:ident| $step:expr)?;)+) => {$( + impl Pair<$t> for $obj { + type Out = $out; + const UNIT: $out = $unit; + #[inline(always)] + fn g($a: $t, $b: $t) -> $out { $g } + #[inline(always)] + fn f($x: $out, $y: $out) -> $out { $f } + $(#[inline(always)] + fn step($c: $out, $p: $t, $q: $t) -> $out { $step })? + } + )+}; +} +macro_rules! floats { + ($($t:ty),+) => {$(pairs! { + PlusTimes: $t => $t = 0.0, |a, b| a * b, |x, y| x + y, |acc, a, b| a.mla::(b, acc); + MaxPlus: $t => $t = <$t>::NEG_INFINITY, |a, b| a + b, |acc, term| Core::maximum(term, acc); + MinPlus: $t => $t = <$t>::INFINITY, |a, b| a + b, |acc, term| Core::minimum(term, acc); + PlusTimes: Complex<$t> => Complex<$t> = Complex::new(0.0, 0.0), |a, b| a * b, |x, y| x + y, |acc, a, b| a.mul_acc(b, acc); + })+}; +} +floats!(f32, f64); + +pairs! { + OrAnd: bool => bool = false, |a, b| a & b, |x, y| x | y; + AndOr: bool => bool = true, |a, b| a | b, |x, y| x & y; +} + +impl Inner for OrAnd { + type Out = bool; + #[inline] + fn inner_into(&self, a: &[bool], b: &[bool], size: [usize; 3], out: &mut [bool]) { + if size[0] * size[1] * size[2] <= DIRECT_WORK { direct::(a, b, size, out) } else { packed::(a, b, size, out) } + } +} + +impl Inner for AndOr { + type Out = bool; + #[inline] + fn inner_into(&self, a: &[bool], b: &[bool], size: [usize; 3], out: &mut [bool]) { + if size[0] * size[1] * size[2] <= DIRECT_WORK { direct::(a, b, size, out) } else { packed::(a, b, size, out) } + } +} + +/// `∨.∧` of `a` and `b`, or with `NOT` set, `∧.∨`, which is `~(~a)∨.∧(~b)`, written into `c`. `NOT` flips each item of `a` and `b` as it is read, and each result as it is written. Each row of the result ORs together the rows of `b` where the row of `a` is true. The rows of `b` are packed 64 items to a word, in blocks of `WORDS` words, and each block of a row's results folds in a local array that stays in registers. An item of `a` becomes a mask of all ones or none, so the loop over it has no branch. +fn packed(a: &[bool], b: &[bool], [m, k, n]: [usize; 3], c: &mut [bool]) { + const WORDS: usize = 4; + assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); + if k == 0 || n == 0 { return c.fill(NOT); } + let blocks = n.div_ceil(64 * WORDS); + let mut packed = vec![[0u64; WORDS]; k * blocks]; + for (row, bits) in b.chunks_exact(n).zip(packed.chunks_exact_mut(blocks)) { + for (j, chunk) in row.chunks(64).enumerate() { + bits[j / WORDS][j % WORDS] = chunk.iter().enumerate().fold(0, |w, (t, &v)| w | u64::from(v ^ NOT) << t); + } + } + for (row, out) in a.chunks_exact(k).zip(c.chunks_exact_mut(n)) { + for (block, out) in out.chunks_mut(64 * WORDS).enumerate() { + let mut acc = [0u64; WORDS]; + for (&x, bits) in row.iter().zip(packed[block..].iter().step_by(blocks)) { + let mask = u64::from(x ^ NOT).wrapping_neg(); + for t in 0..WORDS { acc[t] |= bits[t] & mask } + } + for (j, o) in out.iter_mut().enumerate() { *o = (acc[j / 64] >> (j % 64) & 1 != 0) ^ NOT } + } + } +} +// The pairs at `i64` and `+.×`'s pair at `i32`. `tiered` runs the pairs at `i64` only where every value that a product forms fits `i64`. Their wrapping operations then never wrap. The pair at `i32` wraps, as the products of [`Wrapped`] do. The pairs at `i128`, which `checked_into` uses where a result could pass `i64`. Each step of `+.×` checks that its running total fits 128 bits. +pairs! { + PlusTimes: i64 => i64 = 0, |a, b| a.wrapping_mul(b), |x, y| x.wrapping_add(y); + PlusTimes: i32 => i32 = 0, |a, b| a.wrapping_mul(b), |x, y| x.wrapping_add(y); + MaxPlus: i64 => i64 = i64::MIN, |a, b| a.wrapping_add(b), |x, y| x.max(y); + MinPlus: i64 => i64 = i64::MAX, |a, b| a.wrapping_add(b), |x, y| x.min(y); + PlusTimes: i128 => Option = Some(0), |a, b| a.checked_mul(b), |x, y| x?.checked_add(y?); + MaxPlus: i128 => Option = Some(i128::MIN), |a, b| a.checked_add(b), |x, y| Some(x?.max(y?)); + MinPlus: i128 => Option = Some(i128::MAX), |a, b| a.checked_add(b), |x, y| Some(x?.min(y?)); +} +macro_rules! counts { + ($($t:ty => $out:ty),+) => {$(pairs! { PlusEqual: $t => $out = 0, |a, b| <$out>::from(a == b), |x, y| x + y; })+}; +} +counts!(i64 => i64, i32 => i32, i16 => i16, f64 => i64, f32 => i32); + +/// Steps of `p` in each packed block. +#[doc(hidden)] +pub const KC: usize = 256; +/// Rows of `a` in each packed block. +#[doc(hidden)] +pub const MC: usize = 128; +/// Columns of `b` in each packed block. +#[doc(hidden)] +pub const NC: usize = 2048; + +/// The tile shapes, `[MR, NR]`, and the bytes in a vector. A tile holds the rows and columns of the result that a micro-kernel keeps in registers. AArch64 has 32 vector registers of 16 bytes, and AVX2 has 16 registers of 32 bytes. wasm has 16-byte vectors, and V8 gives it 16 registers on x86-64. The extremes and the counts compare at each step, and on AVX2 they need registers for those comparisons, so their tiles have fewer rows. `TIMES32` serves both `f32` and `i32` products, whose items take 4 bytes. A plain kernel's tile holds at most 32 items. On the Mac, plain tiles of 48 or more floats compiled to scalar code and ran 4 to 5 times slower. +#[cfg(target_arch = "aarch64")] +mod tiles { + pub(super) const BYTES: usize = 16; + #[cfg(native)] + pub(super) const TIMES64: [usize; 2] = [8, 4]; + pub(super) const TIMES32: [usize; 2] = [8, 8]; + pub(super) const COMPARES64: [usize; 2] = [6, 8]; + pub(super) const COMPARES32: [usize; 2] = [6, 16]; + pub(super) const COMPARES16: [usize; 2] = [6, 32]; +} +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma"))] +mod tiles { + pub(super) const BYTES: usize = 32; + #[cfg(native)] + pub(super) const TIMES64: [usize; 2] = [6, 8]; + pub(super) const TIMES32: [usize; 2] = [6, 16]; + pub(super) const COMPARES64: [usize; 2] = [4, 8]; + pub(super) const COMPARES32: [usize; 2] = [4, 16]; + pub(super) const COMPARES16: [usize; 2] = [4, 32]; +} +#[cfg(not(any(target_arch = "aarch64", all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma"))))] +mod tiles { + pub(super) const BYTES: usize = 16; + pub(super) const TIMES64: [usize; 2] = [4, 4]; + pub(super) const TIMES32: [usize; 2] = [4, 8]; + pub(super) const COMPARES64: [usize; 2] = [4, 4]; + pub(super) const COMPARES32: [usize; 2] = [4, 8]; + pub(super) const COMPARES16: [usize; 2] = [4, 16]; +} + +/// The tile of the plain kernel on 64-bit integers, which only products too large for f64's 53 bits reach. +const PLAIN: [usize; 2] = [4, 8]; + +macro_rules! inner { + ($($object:ident at $t:ty => $out:ty: $tile:expr, $kernel:ident;)+) => {$( + impl Inner<$t> for $object { + type Out = $out; + fn inner_into(&self, a: &[$t], b: &[$t], size: [usize; 3], out: &mut [Self::Out]) { + const MR: usize = $tile[0]; + const NR: usize = $tile[1]; + tiled_into::<$t, $t, $object, MR, NR>(a, b, size, out, $kernel::() / tiles::BYTES }>) + } + } + )+}; +} +/// `n` copies of `x`. `vec!` of zeros allocates through calloc, whose `madvise` on a large block costs more than writing its items. +fn filled(n: usize, x: T) -> Vec { + let mut v = Vec::with_capacity(n); + v.resize(n, x); + v +} +/// `P`'s inner product of `a` and `b`, as `Inner::inner_into` gives it, written into `c`, which holds `m`×`n` items, through `kernel` on tiles of `MR`×`NR`. Each item reads as the kernel's type `T` through [`Source`] as it's packed, so an operand of narrower items is never copied whole. +#[inline(always)] +fn tiled_into + Copy + Sync, T: Copy + Default + Send + Sync, P: Pair, const MR: usize, const NR: usize>( + a: &[S], b: &[S], [m, k, n]: [usize; 3], c: &mut [P::Out], kernel: impl Fn(&[T], &[T], &mut [[P::Out; NR]; MR], bool) + Sync) { + assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); + if m * k * n <= DIRECT_WORK { return direct::(a, b, [m, k, n], c); } + let f = >::read; + product::( + c, n, |rows, p0, out| strip(a, k, false, rows, p0, out, f), |columns, p0, out| strip(b, n, true, columns, p0, out, f), [m, k, n], true, Part::Whole, + kernel); +} + +/// The most steps, `m`×`k`×`n`, of a product that [`direct`] computes, where packing the operands for the tiled kernels, or calling Accelerate, costs more than the steps. Measured on 22 October 2026 on an M5 Max, a stack of 4,096 products of 4×4 matrices cost 9.3 copies this way, against 25 on Accelerate and 38 on the packed kernels. A stack of 1,024 products of 8×8 matrices cost 10.1 copies this way, against 10.3 on Accelerate and 9.7 on the packed kernels, so the threshold stays below them. +const DIRECT_WORK: usize = 64; + +/// `P`'s inner product of `a` and `b`, written into `c`, which holds `m`×`n` items, by a plain loop. Each item reads as `T` through [`Source`]. Each result folds its row of `a` with its column of `b` in order, in a register, as the tiled kernels do on every target, NaN included. Four results of a row fold side by side, which vectorises, and the row's last columns fold one at a time. Nothing is packed or allocated. +// A stack of tiny products calls this once for each cell. Once the loop vectorizer ran before thin LTO's import step, this body grew too large to import into `product_at`, and the call cost a stack of 64 2×2 Boolean products about 600 instructions. +#[inline] +fn direct + Copy, T: Copy, P: Pair>(a: &[S], b: &[S], [m, k, n]: [usize; 3], c: &mut [P::Out]) { + let f = >::read; + const SIDE: usize = 4; + assert!(a.len() >= m * k && b.len() >= k * n && c.len() >= m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); + // SAFETY: every index below is `i * k + p`, `p * n + j` or `i * n + j` with `i < m`, `p < k` and `j < n`, which the assert bounds. Checking each one cost about half of `direct`'s instructions on a 4×4 product. + for i in 0..m { + let mut j = 0; + while j + SIDE <= n { + let mut acc = [P::UNIT; SIDE]; + for p in 0..k { + let x = f(unsafe { *a.get_unchecked(i * k + p) }); + for s in 0..SIDE { acc[s] = P::step(acc[s], x, f(unsafe { *b.get_unchecked(p * n + j + s) })) } + } + for s in 0..SIDE { unsafe { *c.get_unchecked_mut(i * n + j + s) = acc[s] } } + j += SIDE; + } + for j in j..n { + let mut acc = P::UNIT; + for p in 0..k { acc = P::step(acc, f(unsafe { *a.get_unchecked(i * k + p) }), f(unsafe { *b.get_unchecked(p * n + j) })) } + unsafe { *c.get_unchecked_mut(i * n + j) = acc } + } + } +} +inner! { + MaxPlus at f64 => f64: tiles::COMPARES64, max_plus64; + MaxPlus at f32 => f32: tiles::COMPARES32, max_plus32; + MinPlus at f64 => f64: tiles::COMPARES64, min_plus64; + MinPlus at f32 => f32: tiles::COMPARES32, min_plus32; + PlusEqual at i32 => i32: tiles::COMPARES32, equal_i32; + PlusEqual at i16 => i16: tiles::COMPARES16, equal_i16; + PlusEqual at f64 => i64: tiles::COMPARES64, equal64; + PlusEqual at f32 => i32: tiles::COMPARES32, equal32; +} + +/// `+.=` at `i64` counts at `i16` where `windowed` maps both arguments into it, and otherwise at `i32` where every item fits in 32 bits. A narrower count puts more items in each vector. +impl Inner for PlusEqual { + type Out = i64; + fn inner_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) { + if let Some(c) = windowed(a, b, size) { return out.iter_mut().zip(&c).for_each(|(o, &x)| *o = i64::from(x)); } + let narrow = |x: &[i64]| x.iter().map(|&v| i32::try_from(v).ok()).collect::>>(); + if let (Some(a), Some(b)) = (narrow(a), narrow(b)) { return out.iter_mut().zip(&self.inner(&a, &b, size)).for_each(|(o, &x)| *o = i64::from(x)); } + const MR: usize = tiles::COMPARES64[0]; + const NR: usize = tiles::COMPARES64[1]; + tiled_into::(a, b, size, out, equal_i64::() / tiles::BYTES }>) + } +} + +/// `+.=` of `a` and `b` counted at `i16`, where `k` is below 2^15 and every item of both lies in one window of 2^16 values. Each item becomes its distance from the window's start, read as an `i16`, so two items are equal exactly where they were before. +fn windowed(a: &[i64], b: &[i64], size: [usize; 3]) -> Option> { + if size[1] > i16::MAX as usize { return None; } + let ((la, ha), (lb, hb)) = (crate::reduce::extremes(a), crate::reduce::extremes(b)); + let (lo, hi) = (la.min(lb), ha.max(hb)); + if i128::from(hi) - i128::from(lo) > i128::from(u16::MAX) { return None; } + let shifted = |x: &[i64]| x.iter().map(|&v| v.wrapping_sub(lo) as u16 as i16).collect::>(); + Some(PlusEqual.inner(&shifted(a), &shifted(b), size)) +} + +/// `+.≠` as `k` less `+.=`. +macro_rules! differences { + ($($t:ty => $out:ty),+) => {$( + impl Inner<$t> for PlusNotEqual { + type Out = $out; + fn inner_into(&self, a: &[$t], b: &[$t], size: [usize; 3], out: &mut [$out]) { + PlusEqual.inner_into(a, b, size, out); + let k = size[1] as $out; + for x in out { *x = k - *x } + } + } + )+}; +} +differences!(i64 => i64, i32 => i32, i16 => i16, f64 => i64, f32 => i32); + +/// The largest magnitude among the items of `x`. +fn magnitude(x: &[T]) -> u128 where i64: From { u128::from(x.iter().fold(0, |m, &v| m.max(i64::from(v).unsigned_abs()))) } + +/// The integer inner products of the float pairs. Each computes at the narrowest item type that holds every value the product forms exactly: `f32` where none exceeds 2^24 in magnitude, `f64` where all lie below 2^51, `i64` where all fit it, and otherwise 128 bits through `wide`. `$bound` gives the largest magnitude that a product can form, from the largest magnitudes in `a` and `b` and from `k`. `checked_into` gives `false` where a result doesn't fit `i64`, and `inner` and `inner_into` panic there. +macro_rules! integers { + ($($object:ident: $bound:expr;)+) => {$( + impl Inner for $object { + type Out = i64; + fn inner_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) { + assert!(self.checked_into(a, b, size, out), "an integer inner product fits i64"); + } + fn checked_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) -> bool { + let bound = $bound(magnitude(a), magnitude(b), size[1] as u128); + if bound <= i64::MAX as u128 { tiered(self, a, b, size, bound, out); return true; } + wide::(a, b, size).map(|c| out.copy_from_slice(&c)).is_some() + } + } + )+}; +} +integers! { + PlusTimes: |a: u128, b: u128, k: u128| a.saturating_mul(b).saturating_mul(k); + MaxPlus: |a: u128, b: u128, _| a + b; + MinPlus: |a: u128, b: u128, _| a + b; +} + +/// `P`'s inner product of `a` and `b`, computed in 128 bits through `P`'s [`Pair`] at `i128`. `None` where a result doesn't fit `i64`, or where a running total doesn't fit 128 bits. +fn wide>>(a: &[i64], b: &[i64], [m, k, n]: [usize; 3]) -> Option> { + (0..m * n).map(|c| { + let (i, j) = (c / n, c % n); + let total = (0..k).fold(P::UNIT, |t, p| P::step(t, a[i * k + p].into(), b[p * n + j].into()))?; + i64::try_from(total).ok() + }).collect() +} + +/// `integers!`'s inner product of `pair`, whose values never exceed `bound` in magnitude, written into `out`. +fn tiered

(pair: &P, a: &[i64], b: &[i64], [m, k, n]: [usize; 3], bound: u128, out: &mut [i64]) where + P: Inner + Inner + Pair { + assert!(a.len() == m * k && b.len() == k * n, "`a` must hold `m`×`k` items and `b` `k`×`n`"); + if k == 0 { return out.fill(

>::UNIT); } + if exactly(pair, a, b, [m, k, n], bound, true, out, |x| x) { return; } + tiled_into::(a, b, [m, k, n], out, micro::) +} + +/// `pair`'s inner product of `a` and `b`, whose values never exceed `bound` in magnitude, in a float that holds each of them exactly: `f32` up to 2^24, and `f64` up to `WIDE_LIMIT` where `wide` holds. Each item reads through [`Source`], and each result, a whole number, converts from `i64` to the type of `out` through `whole`. `false`, with `out` untouched, where neither float holds every value. +fn exactly(pair: &P, a: &[S], b: &[S], size: [usize; 3], bound: u128, wide: bool, out: &mut [O], whole: impl Fn(i64) -> O) -> bool where + P: Inner + Inner, S: Source + Source { + fn at, S: Source, O>( + pair: &P, a: &[S], b: &[S], size: [usize; 3], out: &mut [O], whole: impl Fn(i64) -> O) -> bool { + for (o, x) in out.iter_mut().zip(pair.inner(&read_all(a), &read_all(b), size)) { *o = whole(convert::shifted(x.wide())) } + true + } + if bound <= 1 << 24 { at::(pair, a, b, size, out, whole) } + else { wide && bound <= convert::WIDE_LIMIT as u128 && at::(pair, a, b, size, out, whole) } +} + +/// `+.×` of `a` and `b` with wrapping `i32` steps, written into `out`, on the `i32` kernel. The kernel converts each item to `i32` through [`Source`] as it packs it. +fn times_i32_into>(a: &[S], b: &[S], size: [usize; 3], out: &mut [i32]) { + const MR: usize = tiles::TIMES32[0]; + const NR: usize = tiles::TIMES32[1]; + tiled_into::(a, b, size, out, times_i32::() / tiles::BYTES }>) +} + +/// [`Wrapped`] on `u8`, `i16` and `i32` items. Each result is its exact sum of products wrapped to 32 bits, as static kinds and fixed widths keep integer results. Nothing fails. [`Product::run_wrapped`] runs these products. `$largest` gives the largest magnitude that a product of two items can form: from the item type for `u8` and `i16`, and from the items' largest magnitudes for `i32`. Where `k` such products sum exactly in `f32`, the product computes in `f32` and wraps each whole result, as `integers!`'s products do. Where Accelerate serves the product, a sum that `f64` holds exactly computes in `f64` the same way. Every other product runs on the `i32` kernel, whose wrapping steps are exact to 32 bits whatever the items. On an M5 Max on 25 October 2026, against the `i32` kernel's time, a product of two 512×512 `u8` matrices took a fifth through Accelerate's `f64` product and 1.6 times as long through rustymath's own. A 512×256 by 256×512 product took 0.85 of the `i32` kernel's time through rustymath's own `f32` product. +macro_rules! wrapped_narrow { + ($($t:ty: $largest:expr;)+) => {$( + impl Inner<$t> for crate::Wrapped { + type Out = i32; + fn inner_into(&self, a: &[$t], b: &[$t], size @ [m, k, n]: [usize; 3], out: &mut [i32]) { + assert!(a.len() == m * k && b.len() == k * n && out.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `out` `m`×`n`"); + if m * k * n > DIRECT_WORK && exactly(&PlusTimes, a, b, size, $largest(a, b) * k as u128, accelerated(), out, |x| x as i32) { return } + times_i32_into(a, b, size, out) + } + } + )+}; +} +wrapped_narrow! { + u8: |_, _| 255 * 255; + i16: |_, _| 1 << 30; + i32: |a, b| magnitude(a) * magnitude(b); +} + +/// `+.×` of f16 matrices, summed and returned at f32. rustymath's own kernels convert each item to f32 as they pack it, so neither operand is copied whole. Accelerate has no f16 product, so where it serves the process, both operands convert to f32 copies for its f32 product. +impl Inner for PlusTimes { + type Out = f32; + fn inner_into(&self, a: &[half::f16], b: &[half::f16], size @ [m, k, n]: [usize; 3], c: &mut [f32]) { + assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); + if m * k * n <= DIRECT_WORK { return direct::(a, b, size, c); } + dispatch!(half_products_accelerated(a, b, size, c), half_products_native(a, b, size, c)) + } +} + +/// [`PlusTimes`] at f16 on Accelerate's f32 product, through f32 copies of both operands. +#[cfg(accelerate)] +fn half_products_accelerated(a: &[half::f16], b: &[half::f16], size @ [_, k, n]: [usize; 3], c: &mut [f32]) { + f32::gemm(c, n, Matrix::rows(&read_all(a), k), Matrix::rows(&read_all(b), n), size, 1.0, 0.0) +} + +/// [`PlusTimes`] at f16 on rustymath's own f32 kernels, which convert each item as they pack it. +#[cfg(native)] +fn half_products_native(a: &[half::f16], b: &[half::f16], size: [usize; 3], c: &mut [f32]) { + const MR: usize = tiles::TIMES32[0]; + const NR: usize = tiles::TIMES32[1]; + tiled_into::(a, b, size, c, times32::() / tiles::BYTES }>) +} + +/// A strided matrix borrowed in its stored, transposed, conjugated or adjoint orientation. +#[derive(Clone, Copy)] +pub(crate) struct Matrix<'a, T> { pub data: &'a [T], pub ld: usize, pub op: Op } +#[derive(Clone, Copy)] +pub(crate) enum Op { Normal, Transpose, Adjoint, Conjugate } +/// Whether an operand of [`Inner::inner_ops`] holds its matrix as stored or transposed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Orientation { Normal, Transpose } +/// A row-major matrix whose items `mapping` reads from `data`, as [`expanded_items`] lays an argument over a frame. [`Inner::inner_mapped`] takes its operands this way. A matrix held in order has `Mapping::Linear(1)`. +#[derive(Clone, Copy)] +pub struct Mapped<'a, T> { pub data: &'a [T], pub mapping: &'a Walked<'a> } + +impl<'a, T: Element> Mapped<'a, T> { + /// The matrix in orientation `o`, with no copy, where the mapping reads its stored matrix at positive strides, one of them 1: row by row, or column by column as a transpose. `None` for any other mapping. + fn matrix(&self, o: Orientation, [rows, cols]: [usize; 2]) -> Option> { + let [r, c] = if o == Orientation::Normal { [rows, cols] } else { [cols, rows] }; + let (first, strides) = self.mapping.affine(&[r, c])?; + // An axis of length 1 reads one item, so any stride serves it. A single row whose columns are strided reads as a transposed column. + let sc = if c == 1 { 1 } else { strides[1] }; + let sr = if r > 1 { strides[0] } else if sc == 1 { c as isize } else { 1 }; + let data = self.data.get(first..)?; + let stored = match (sr, sc) { + (s, 1) if s >= c as isize => Matrix::rows(data, s as usize), (1, s) if s >= r as isize => Matrix { data, ld: s as usize, op: Op::Transpose }, + _ => return None + }; + Some(if o == Orientation::Normal { stored } else { stored.transpose() }) + } + /// Whether [`Inner::inner_mapped`] reads this matrix where it lies, in orientation `o` with `size` rows and columns, rather than one block at a time. + pub fn in_place(&self, o: Orientation, size: [usize; 2]) -> bool { self.matrix(o, size).is_some() } + /// Items `start..start + n` of the matrix, as [`expanded_items`] reads them. + pub fn read<'b>(&'b self, start: usize, n: usize, buf: &'b mut Vec) -> &'b [T] { expanded_items(self.mapping, self.data, start, n, buf) } +} +#[derive(Clone, Copy)] +pub(crate) enum Side { Left, Right } +/// The stored triangle, before applying the matrix view's transpose or adjoint. +#[derive(Clone, Copy)] +pub(crate) enum Triangle { Lower, UnitUpper } +impl Op { + #[inline(always)] + fn transposed(self) -> bool { matches!(self, Self::Transpose | Self::Adjoint) } +} +impl<'a, T: Copy> Matrix<'a, T> { + pub(crate) fn rows(data: &'a [T], ld: usize) -> Self { Self { data, ld, op: Op::Normal } } + pub(crate) fn transpose(self) -> Self { + Self { op: match self.op { + Op::Normal => Op::Transpose, Op::Transpose => Op::Normal, Op::Adjoint => Op::Conjugate, Op::Conjugate => Op::Adjoint + }, ..self } + } + /// Starts at a row and column in the logical orientation. + pub(crate) fn offset(self, row: usize, col: usize) -> Self { + let at = if self.op.transposed() { col * self.ld + row } else { row * self.ld + col }; + Self { data: &self.data[at..], ..self } + } + /// The `rows`×`cols` matrix that an operand of [`Inner::inner_ops`] gives in orientation `o`. A transposed operand stores the `cols`×`rows` matrix, row-major. + pub(crate) fn oriented(data: &'a [T], o: Orientation, [rows, cols]: [usize; 2]) -> Self { + match o { Orientation::Normal => Self::rows(data, cols), Orientation::Transpose => Self { data, ld: rows, op: Op::Transpose } } + } + /// Checks the stored extent once, before any foreign call. + fn check(self, mut rows: usize, mut cols: usize) { + if self.op.transposed() { std::mem::swap(&mut rows, &mut cols) } + if rows == 0 || cols == 0 { return; } + let end = (rows - 1).checked_mul(self.ld).and_then(|n| n.checked_add(cols)); + assert!(self.ld >= cols && end.is_some_and(|n| n <= self.data.len()), "matrix view must hold its rows and columns"); + } +} +impl Matrix<'_, T> { + pub(crate) fn conjugate(self) -> Self { + // Conjugation is an identity for real scalars; don't make their BLAS views more restrictive. + if !T::COMPLEX { return self; } + Self { op: match self.op { + Op::Normal => Op::Conjugate, Op::Transpose => Op::Adjoint, Op::Adjoint => Op::Transpose, Op::Conjugate => Op::Normal + }, ..self } + } + pub(crate) fn adjoint(self) -> Self { self.transpose().conjugate() } + #[cfg(native)] + #[inline(always)] + pub(crate) fn at(self, i: usize, j: usize) -> T { + let x = self.data[if self.op.transposed() { j * self.ld + i } else { i * self.ld + j }]; + if matches!(self.op, Op::Adjoint | Op::Conjugate) { x.conj() } else { x } + } + /// Writes a packing strip of this view, times `scale`, as the free function `strip` does: item `[p][l]` is item `(r0 + l, p0 + p)`. A product packs strips of `a` from `a`, and of `b` from `b`'s transpose. + #[cfg(native)] + #[inline(always)] + fn strip(self, rows: [usize; 2], p0: usize, scale: T, out: &mut [[MaybeUninit; W]]) where T: Default { + let (transposed, one) = (self.op.transposed(), scale == T::one()); + if !T::COMPLEX || !matches!(self.op, Op::Adjoint | Op::Conjugate) { + if one { strip(self.data, self.ld, transposed, rows, p0, out, |x| x) } else { strip(self.data, self.ld, transposed, rows, p0, out, |x| scale * x) } + } else if one { strip(self.data, self.ld, transposed, rows, p0, out, |x: T| x.conj()) } + else { strip(self.data, self.ld, transposed, rows, p0, out, |x: T| scale * x.conj()) } + } +} +/// Writes all `W` items of each step of a packing strip, through `f`. Item `[p][l]` is item `(r0 + l, p0 + p)` of the matrix that `data` holds at `r * ld + p`, or at `p * ld + r` where `transposed`, for `l` below `lines`, and the default after that. Transposed data gives each step from one run, and other data from `lines` rows read side by side. Every step writes a fixed `W` items, which keeps short strips from calling `memcpy` or `memset` at each step. +#[inline(always)] +fn strip( + data: &[S], ld: usize, transposed: bool, [r0, lines]: [usize; 2], p0: usize, out: &mut [[MaybeUninit; W]], f: impl Fn(S) -> T) { + assert!(lines <= W); + let kc = out.len(); + let rows: [&[S]; W] = std::array::from_fn(|l| if transposed || l >= lines { &[] } else { &data[(r0 + l) * ld + p0..][..kc] }); + match (transposed, lines == W) { + (true, true) => { + for (p, step) in out.iter_mut().enumerate() { + let run: &[S; W] = data[(p0 + p) * ld + r0..].first_chunk().expect("a whole step lies in `data`"); + for l in 0..W { step[l] = MaybeUninit::new(f(run[l])) } + } + } + (true, false) => { + for (p, step) in out.iter_mut().enumerate() { + let run = &data[(p0 + p) * ld + r0..][..lines]; + for l in 0..W { step[l] = MaybeUninit::new(if l < lines { f(run[l]) } else { T::default() }) } + } + } + (false, true) => { for (p, step) in out.iter_mut().enumerate() { for l in 0..W { step[l] = MaybeUninit::new(f(rows[l][p])) } } } + (false, false) => { + for (p, step) in out.iter_mut().enumerate() { for l in 0..W { step[l] = MaybeUninit::new(if l < lines { f(rows[l][p]) } else { T::default() }) } } + } + } +} +/// Item types whose matrix products run on the tiled `+.×` kernels: `f32` and `f64`, and complex numbers with parts of either type. The decompositions in `factor` build on them. +pub(crate) trait Tiled: Scalar + Default + Vendor { + /// The row-major matrix product, allocated once through the same GEMM operation as updates. + #[inline] + fn multiply(a: Matrix<'_, Self>, b: Matrix<'_, Self>, size: [usize; 3]) -> Vec { + let mut c = filled(size[0] * size[2], Self::zero()); + Self::gemm(&mut c, size[2], a, b, size, Self::one(), Self::zero()); + c + } + /// `c = alpha a b + beta c`, with `size = [m, k, n]`. Dispatch precedes native packing and row splitting. + #[inline] + fn gemm(c: &mut [Self], ldc: usize, a: Matrix<'_, Self>, b: Matrix<'_, Self>, size: [usize; 3], alpha: Self, beta: Self) { + gemm(c, ldc, a, b, size, alpha, beta, Part::Whole) + } + /// The lower triangle of `c = alpha x xᴴ + beta c`, for `x` of size `[n, k]`. Native diagonal tiles may also change upper items. + #[inline] + fn gram(c: &mut [Self], ldc: usize, x: Matrix<'_, Self>, [n, k]: [usize; 2], alpha: Self::Real, beta: Self::Real) { + gemm(c, ldc, x, x.adjoint(), [n, k, n], Self::from_real(alpha), Self::from_real(beta), Part::Lower) + } + /// Solves `a x = b` or `x a = b` in place, with `triangle` describing the stored square matrix `a`. + fn triangular(x: &mut [Self], ldx: usize, a: Matrix<'_, Self>, [m, n]: [usize; 2], side: Side, triangle: Triangle) { + let order = match side { Side::Left => m, Side::Right => n }; + a.check(order, order); + Matrix::rows(x, ldx).check(m, n); + if m == 0 || n == 0 { return; } + dispatch!(accelerate::triangular(x, ldx, a, [m, n], side, triangle), factor::triangular_native(x, ldx, a, [m, n], side, triangle)) + } + /// `alpha a b` on rustymath's own packed kernels, written over `c` where `start` is true and added to it otherwise. Complex items run four real products through `complex_product`. + #[cfg(native)] + fn native_product(c: &mut [Self], ldc: usize, a: Matrix<'_, Self>, alpha: Self, b: Matrix<'_, Self>, size: [usize; 3], start: bool, part: Part); + /// The thin SVD of the finite, nonempty `m`×`n` matrix `a` as `(U, s, V)`, with its singular vectors only where `vectors` is true, or `None` where it doesn't converge. + fn svd(a: &[Self], size: [usize; 2], vectors: bool) -> Option<(Vec, Vec, Vec)>; + /// The eigen decomposition of the nonempty Hermitian `n`×`n` matrix whose finite lower triangle `a` holds, with its eigenvectors only where `vectors` is true, or `None` where it doesn't converge. + fn eigh(a: &[Self], n: usize, vectors: bool) -> Option>; + /// The eigen decomposition of the finite, nonempty `n`×`n` matrix `a`, with its eigenvectors only where `vectors` is true, or `None` where it doesn't converge. + fn eig(a: &[Self], n: usize, vectors: bool) -> Option>; +} + +fn gemm(c: &mut [T], ldc: usize, a: Matrix<'_, T>, b: Matrix<'_, T>, [m, k, n]: [usize; 3], alpha: T, beta: T, part: Part) { + a.check(m, k); + b.check(k, n); + Matrix::rows(c, ldc).check(m, n); + if m == 0 || n == 0 { return; } + if k == 0 || alpha == T::zero() { return scale(c, ldc, [m, n], beta, part); } + if by_vector(c, ldc, a, alpha, b, [m, k, n], beta, part) { return; } + dispatch!(accelerate::product(c, ldc, a, b, [m, k, n], alpha, beta, part), native_gemm(c, ldc, a, b, [m, k, n], alpha, beta, part)) +} + +/// The fewest items that [`blocked`] reads at a time from an operand that isn't held in order, rounded to whole stored rows. A block this size stays in cache while its product runs. +const BLOCK_ITEMS: usize = 1 << 18; +/// The most items of `b` for which [`blocked`] gives each thread its own piece of the result's rows, multiplied a block at a time. Each thread's product packs its own copy of `b`, and larger copies crowd the cache that the threads share. On one thread, no other copy of `b` shares the cache. A product there takes pieces whatever the size of `b`. With threads on an M5 Max, for a strided 4096×2048 view times a 2048×n matrix (28 October 2026), pieces took 3% to 14% less time than one copy and one product at 64 columns, 2^17 items, and up to 37% more from 128 columns on rustymath's own kernels. +const PIECE_MATRIX: usize = 1 << 17; +/// The most items of an operand that isn't held in order that [`blocked`] reads whole, where `b` is too large for [`PIECE_MATRIX`]. A larger operand is read in blocks of this many items, rounded up to whole rows, each multiplied in one threaded call. With threads on an M5 Max (28 October 2026), products of 2048 and 4096 rows of 2048 items took within 1% as long for each row at 512 columns, and 1024 rows took 4% longer on Accelerate and 8% longer on rustymath's own kernels. +const WHOLE_ITEMS: usize = 1 << 24; + +/// The stored rows of `cols` items in each block that [`blocks`] reads: at least `least` items and at least [`BLOCK_ITEMS`], in whole rows. +fn block_rows(cols: usize, least: usize) -> usize { least.max(BLOCK_ITEMS).div_ceil(cols) } + +/// Calls `f` on each block of an operand's stored rows `rows`, each row of `cols` items, with the block's first row, its number of rows and its items. Each block holds [`block_rows`] rows, apart from the last. +fn blocks(op: Mapped<'_, T>, rows: std::ops::Range, cols: usize, least: usize, mut f: impl FnMut(usize, usize, &[T])) { + let (step, mut block) = (block_rows(cols, least), Vec::new()); + for first in rows.clone().step_by(step) { + let len = step.min(rows.end - first); + f(first, len, op.read(first * cols, len * cols, &mut block)); + } +} + +/// Sets the result `c` to the sum of the products of each block of an operand's stored rows, `[rows, cols]` as stored, where those rows run along the contraction. `f(first, len, block, c, beta)` sets `c` to the product of the block of `len` rows from `first` plus `beta` times `c`. The block at row 0 replaces what `c` held, with `beta` 0. Every other block adds, with `beta` 1. A result of at most [`BLOCK_ITEMS`] items splits the rows into one piece of at least one block for each thread ([`parallel::summed`]). Each thread reads its rows in blocks of [`BLOCK_ITEMS`] items into its own copy of the result, and the copies then add into `c`. A larger result takes blocks of at least as many items as it holds, in turn. +fn contracted(op: Mapped<'_, T>, [rows, cols]: [usize; 2], c: &mut [T], f: &(dyn Fn(usize, usize, &[T], &mut [T], T) + Sync)) { + let beta = |first: usize| if first == 0 { T::zero() } else { T::one() }; + if c.len() > BLOCK_ITEMS { return blocks(op, 0..rows, cols, c.len(), |first, len, block| f(first, len, block, c, beta(first))); } + parallel::summed(rows, block_rows(cols, 0), c, &mut Vec::new(), &|range, sums| { + blocks(op, range, cols, 0, |first, len, block| f(first, len, block, sums, beta(first))) + }); +} + +/// The most rows of the result in one piece of [`by_columns`]'s split across threads. +const COLUMN_ROWS: usize = 4096; +/// The results that [`scaled_runs`] keeps in registers while every column adds into them. +const COLUMN_STRIP: usize = 32; +/// The results in each strip that [`scaled_runs`] takes after its strips of [`COLUMN_STRIP`], before it takes one result at a time. The windows of an image 256 items wide leave 30 results in each row after its strips of 32. +const TAIL_STRIP: usize = 8; +/// The fewest adjacent rows of `a` that [`by_columns`] reads at once. Shorter runs cost more in loop overhead than gathering saves. +const COLUMN_RUN: usize = 64; + +/// `+.×` of `a` and a vector `b`, where each row of `a` starts one item after the row before it in storage, as the windows of a vector or a single-channel image do. Each column of `a` is then a run of storage, which [`scaled_runs`] reads where it lies and scales by its item of `b`. `a` is never gathered. `false` where it doesn't split into entries for its rows and entries for its columns, or where its rows aren't adjacent in runs of at least [`COLUMN_RUN`]. The loop reads a fill as 0, the item type's own fill. +fn by_columns(a: Mapped<'_, T>, b: Mapped<'_, T>, oa: Orientation, [m, k]: [usize; 2], c: &mut [T]) -> bool { + use crate::gather::Steps; + let Mapping::Steps { base, steps } = &**a.mapping else { return false }; + if m * k == 0 { return false; } + // The entries of the stored matrix's leading axis come first, and give its first `lead` items. + let lead = if oa == Orientation::Normal { m } else { k }; + let (mut split, mut size) = (0, 1); + while size < lead && split < steps.len() { + size *= steps[split].len(); + split += 1; + } + if size != lead { return false; } + let (outer, inner) = steps.split_at(split); + let (rows, cols) = if oa == Orientation::Normal { (outer, inner) } else { (inner, outer) }; + let Some(&Steps::Stride { step: 1, len: width, .. }) = rows.iter().rev().find(|s| s.len() > 1) else { return false }; + if width < COLUMN_RUN { return false; } + let mut whole = Vec::new(); + let weights = b.read(0, k, &mut whole); + let columns: Vec<(T, usize)> = (0..k).filter_map(|p| Some((weights[p], Steps::reach(0, cols, p)?))).collect(); + parallel::split(c, COLUMN_ROWS, &|first, part| { + let mut r = first; + while r < first + part.len() { + let len = (width - r % width).min(COLUMN_ROWS).min(first + part.len() - r); + // A row whose outer entries give a fill reads only fills. + let out = &mut part[r - first..r - first + len]; + if let Some(at) = Steps::reach(*base, rows, r) { scaled_runs(out, a.data, at, &columns, true) } + else { out.fill(T::zero()) } + r += len; + } + true + }); + true +} + +/// For adjacent rows of a matrix whose first row starts at `data[at]` and each next row one item later, adds to each of `out`'s results the sum over `columns` of each column's weight times the row's item at the column's offset. Where `start` is true, it writes the sums in place of `out`'s results. Strips of [`COLUMN_STRIP`] results, then of [`TAIL_STRIP`], stay in registers while every column adds into them ([`scaled_strips`]). The rest take one result at a time. +fn scaled_runs(out: &mut [T], data: &[T], at: usize, columns: &[(T, usize)], start: bool) { + let wide = scaled_strips::(out, data, at, columns, start); + let done = wide + scaled_strips::(&mut out[wide..], data, at + wide, columns, start); + let at = at + done; + for (t, y) in out[done..].iter_mut().enumerate() { + *y = columns.iter().fold(if start { T::zero() } else { *y }, |acc, &(w, o)| w.mul_acc(data[at + o + t], acc)) + } +} + +/// [`scaled_runs`] for the whole strips of `N` results at the start of `out`, each kept in registers while every column adds into it. Gives how many results the strips hold. +fn scaled_strips(out: &mut [T], data: &[T], at: usize, columns: &[(T, usize)], start: bool) -> usize { + let (strips, _) = out.as_chunks_mut::(); + for (s, strip) in strips.iter_mut().enumerate() { + let (at, mut acc) = (at + s * N, if start { [T::zero(); N] } else { *strip }); + for &(w, o) in columns { let x = &data[at + o..][..N]; for t in 0..N { acc[t] = w.mul_acc(x[t], acc[t]) } } + *strip = acc; + } + strips.len() * N +} + +/// Writes `out`'s results for stored rows `first..` of a matrix whose rows start `ld` items apart in `data`: each the sum of products of its row with `x`. Four rows at a time share each read of `x`, each row in eight running sums, which vectorize and then add pairwise. +fn dots(out: &mut [T], data: &[T], ld: usize, first: usize, x: &[T]) { + let k = x.len(); + let (quads, tail) = out.as_chunks_mut::<4>(); + let (xc, xr) = x.as_chunks::<8>(); + let sum = |s: &[T; 8]| ((s[0] + s[4]) + (s[2] + s[6])) + ((s[1] + s[5]) + (s[3] + s[7])); + for (q, quad) in quads.iter_mut().enumerate() { + let rows: [&[T]; 4] = std::array::from_fn(|r| &data[(first + 4 * q + r) * ld..][..k]); + let mut s = [[T::zero(); 8]; 4]; + for (c, xs) in xc.iter().enumerate() { for r in 0..4 { let a = &rows[r][c * 8..][..8]; for l in 0..8 { s[r][l] = a[l].mul_acc(xs[l], s[r][l]) } } } + let at = xc.len() * 8; + for r in 0..4 { quad[r] = xr.iter().enumerate().fold(sum(&s[r]), |t, (p, &b)| rows[r][at + p].mul_acc(b, t)) } + } + let first = first + 4 * quads.len(); + for (i, y) in tail.iter_mut().enumerate() { *y = factor::dot(&data[(first + i) * ld..][..k], x) } +} + +/// The most items in the stored rows of a matrix that [`by_vector`] takes where Accelerate serves the process. Accelerate pays a fixed cost for each stored row. [`narrow_sums`] takes up to this many results, and [`narrow_dots`] stored rows of up to this many items. +const NARROW: usize = 8; + +/// [`dots`] for stored rows of `K` items, at most [`NARROW`], where `x` holds as many. Each row's products add into two running sums. Each pair of rows then adds its sums pairwise into both results in one vector step. +fn narrow_dots(out: &mut [T], data: &[T], ld: usize, first: usize, x: &[T]) { + let x: &[T; K] = x.try_into().expect("`x` must hold `K` items"); + let sums = |i: usize| { + let a = &data[i * ld..][..K]; + let mut s = [T::zero(); 2]; + for l in 0..K { s[l % 2] = a[l].mul_acc(x[l], s[l % 2]) } + s + }; + let last = first + out.len().saturating_sub(1); + let (pairs, odd) = out.as_chunks_mut::<2>(); + for (p, pair) in pairs.iter_mut().enumerate() { + let (s, t) = (sums(first + 2 * p), sums(first + 2 * p + 1)); + *pair = [s[0] + s[1], t[0] + t[1]]; + } + if let [y] = odd { let s = sums(last); *y = s[0] + s[1] } +} + +/// Writes each of `out`'s results, at most [`NARROW`], as the sum over the stored rows of `data`, which start `ld` items apart, of the row's item in `w` times its item at the result's position: a vector times the matrix whose columns those rows hold. Every sum stays in registers across all the rows, in four sets that each take every fourth row. A row's multiply-adds therefore don't wait on the row before. Each row adds [`NARROW`] items at once into its set, and the items past the results, which belong to the next row, reach no result. The last rows, whose [`NARROW`] items would pass the end of `data`, add one result at a time. +fn narrow_sums(out: &mut [T], data: &[T], ld: usize, w: &[T]) { + let n = out.len(); + let whole = if data.len() < NARROW { 0 } else { ((data.len() - NARROW) / ld.max(1) + 1).min(w.len()) }; + let mut acc = [[T::zero(); NARROW]; 4]; + let (quads, rest) = w[..whole].as_chunks::<4>(); + for (q, ws) in quads.iter().enumerate() { + for s in 0..4 { let a = &data[(4 * q + s) * ld..][..NARROW]; for t in 0..NARROW { acc[s][t] = a[t].mul_acc(ws[s], acc[s][t]) } } + } + for (p, &wp) in rest.iter().enumerate() { + let a = &data[(4 * quads.len() + p) * ld..][..NARROW]; + for t in 0..NARROW { acc[0][t] = a[t].mul_acc(wp, acc[0][t]) } + } + for (p, &wp) in w.iter().enumerate().skip(whole) { for t in 0..n { acc[0][t] = data[p * ld + t].mul_acc(wp, acc[0][t]) } } + for (t, y) in out.iter_mut().enumerate() { *y = (acc[0][t] + acc[2][t]) + (acc[1][t] + acc[3][t]) } +} + +/// `+.×` of mapped operands on the tiled kernels, for [`Inner::inner_mapped_into`]. An operand whose mapping reads it at positive strides, one of them 1, is used where it lies, row by row or as a transpose. Any other operand is read a block of its stored rows at a time, and each block's product goes straight into the result. Where `a`'s stored rows are the result's rows, and the product runs on one thread or `b` has at most [`PIECE_MATRIX`] items, the result's rows split into one piece for each thread. Each thread multiplies its piece's rows itself, reading them in blocks that hold at least [`BLOCK_ITEMS`] items and at least as many as `b`. Otherwise an operand of up to [`WHOLE_ITEMS`] items is read whole, and a larger one in blocks of at least that many. Each block is gathered across threads and multiplied in one threaded call. Rows along the contraction add into the whole result, as [`contracted`] reads them. Blocks of the result's columns hold at least as many items as the result. Where neither operand can be used where it lies, `b` is read whole. A vector `b` beside an `a` whose rows are adjacent in storage goes to [`by_columns`], which reads `a` where it lies. Reading an operand larger than the result through its map, by blocks or by columns, records the route `mapped product`. +fn blocked(a: Mapped<'_, T>, b: Mapped<'_, T>, [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3], c: &mut [T]) { + assert_eq!(c.len(), m * n); + if m == 0 || k == 0 || n == 0 { c.fill(T::zero()); return; } + let (ma, mb) = (a.matrix(oa, [m, k]), b.matrix(ob, [k, n])); + if let (Some(a), Some(b)) = (ma, mb) { return T::gemm(c, n, a, b, [m, k, n], T::one(), T::zero()); } + if ma.is_none() && n == 1 && by_columns(a, b, oa, [m, k], c) { crate::events::taken(crate::events::Route::MappedProduct, m * k); return; } + let read = if ma.is_none() { m * k } else { k * n }; + if read > m * n { crate::events::taken(crate::events::Route::MappedProduct, read); } + let (one, zero) = (T::one(), T::zero()); + let mut whole = Vec::new(); + match ma { + None => { + let b = match mb { Some(b) => b, None => Matrix::oriented(b.read(0, k * n, &mut whole), ob, [k, n]) }; + match oa { + Orientation::Normal => { + let (rows, least) = + if parallel::threads() == 1 || k * n <= PIECE_MATRIX { (m.div_ceil(parallel::threads()), k * n) } else { (m, WHOLE_ITEMS) }; + parallel::pieces_of(c, rows * n, usize::MAX, &|start, part| { + let from = start / n; + blocks(a, from..from + part.len() / n, k, least, |first, len, block| { + T::gemm(&mut part[(first - from) * n..], n, Matrix::rows(block, k), b, [len, k, n], one, zero) + }); + true + }); + } + Orientation::Transpose => contracted(a, [k, m], c, &|first, len, part, c, beta| { + T::gemm(c, n, Matrix::oriented(part, oa, [m, len]), b.offset(first, 0), [m, len, n], one, beta) + }), + } + } + Some(a) => match ob { + Orientation::Normal => { + contracted(b, [k, n], c, &|first, len, part, c, beta| T::gemm(c, n, a.offset(0, first), Matrix::rows(part, n), [m, len, n], one, beta)) + } + Orientation::Transpose => { + blocks(b, 0..n, k, m * n, |first, len, block| T::gemm(&mut c[first..], n, a, Matrix::oriented(block, ob, [k, len]), [m, k, len], one, zero)) + } + }, + } +} + +/// `c = beta c` for the items of `c` in `part`, where `c` is an `m`×`n` matrix whose rows start `ldc` items apart. A zero `beta` writes zeros, whatever `c` held. +fn scale(c: &mut [T], ldc: usize, [m, n]: [usize; 2], beta: T, part: Part) { + if beta == T::one() { return; } + for i in 0..m { + for j in part.columns(i, n) { + let x = &mut c[i * ldc + j]; + *x = if beta == T::zero() { T::zero() } else { beta * *x }; + } + } +} + +/// The multiply-adds in each unit of [`by_vector`]'s split across threads. `parallel::split` times the first unit to choose the pieces. +const VECTOR_WORK: usize = 1 << 15; +/// The results that [`by_vector`] keeps in the first-level cache while the stored rows of a transposed matrix add into them. +const VECTOR_STRIP: usize = 2048; +/// The stored rows of a transposed matrix that [`by_vector`] adds into its results together, read side by side in storage order. +const VECTOR_ROWS: usize = 8; +/// The fewest multiply-adds in each piece of [`by_vector`]'s split of stored rows across threads: twice what waking one more thread costs ([`THREAD_WORK`]), so that every split gains. At 4 threads on an M5 Max, a split cost about 10 to 12 µs whether it made two pieces or four, and pieces of 2^15 to 2^17 made a vector times a matrix of 1024 columns up to 76% slower than one piece where their work came to less than that. With pieces of 2^18, 128 and 256 rows ran as one piece, and 512 and 1024 rows ran 20 and 28% faster than before in two and four pieces. +const VECTOR_PIECE: usize = 2 * THREAD_WORK; + +/// `alpha a b + beta c`, where `m` or `n` is 1, for a nonzero `k` and `alpha`. The packed kernels would pack the whole matrix and pad the vector to a tile. As a matrix times a vector, `a b` or the transpose of `bᵀ aᵀ`, each result is the sum of products of a stored row with the vector where the matrix's stored rows run along the contraction: [`narrow_dots`] for stored rows of up to [`NARROW`] items, and [`dots`] for longer ones. Otherwise each result reads one item of every stored row. At most [`NARROW`] results keep their sums in registers across every stored row ([`narrow_sums`]). More results take strips of [`VECTOR_STRIP`]. [`scaled_runs`] adds [`VECTOR_ROWS`] stored rows at a time into each strip, in storage order. The results split across threads, or, where they fit one strip of a transposed matrix, the stored rows do, each piece adding into results of its own. The vector is read where it lies where its items are adjacent and need no conjugate, and the results are written straight into `c` where they need no scaling, conjugate or step. `false` for any other size and for `Part::Lower`. Accelerate multiplies longer stored rows faster: where it serves the process, `by_vector` gives `false` for stored rows of more than [`NARROW`] items. +fn by_vector(c: &mut [T], ldc: usize, a: Matrix<'_, T>, alpha: T, b: Matrix<'_, T>, [m, k, n]: [usize; 3], beta: T, part: Part) -> bool { + let (matrix, vector, rows, step) = match (m, n, part) { + (_, 1, Part::Whole) => (a, b.transpose(), m, ldc), (1, _, Part::Whole) => (b.transpose(), a, n, 1), _ => return false + }; + let narrow = if matrix.op.transposed() { rows } else { k } <= NARROW; + if accelerated() && !narrow { return false; } + // A conjugated matrix's results are the conjugates of its stored items' results with the conjugated vector. + let conj = matches!(matrix.op, Op::Adjoint | Op::Conjugate); + let flip = T::COMPLEX && conj != matches!(vector.op, Op::Adjoint | Op::Conjugate); + let stride = if vector.op.transposed() { vector.ld } else { 1 }; + let x: Cow<'_, [T]> = if stride == 1 && !flip { Cow::Borrowed(&vector.data[..k]) } + else { (0..k).map(|p| if flip { vector.data[p * stride].conj() } + else { vector.data[p * stride] }).collect() }; + let (data, ld, unit) = (matrix.data, matrix.ld, (VECTOR_WORK / k).max(1).next_multiple_of(4)); + let (one, start) = (alpha == T::one(), beta == T::zero()); + let direct = step == 1 && one && start && !conj; + let mut kept = if direct { Vec::new() } else { filled(rows, T::zero()) }; + let y = if direct { &mut c[..rows] } else { &mut kept[..] }; + if matrix.op.transposed() { + // Stored row `p` holds the matrix's column `p`, so the matrix's row `i` starts at `data[i]`. Results that fit one strip leave each thread a narrow part of every stored row, so the stored rows split instead. Each piece writes its rows' sums into results of its own, which add into `y` ([`parallel::summed`]). + let piece = (VECTOR_PIECE / rows).max(1); + if narrow { + parallel::summed(k, piece, y, &mut Vec::new(), &|range, part| narrow_sums(part, &data[range.start * ld..], ld, &x[range])); + } else { + let columns: Vec<(T, usize)> = x.iter().enumerate().map(|(p, &w)| (w, p * ld)).collect(); + if rows <= VECTOR_STRIP { + parallel::summed(columns.len(), piece, y, &mut Vec::new(), &|range, part| { + for (i, block) in columns[range].chunks(VECTOR_ROWS).enumerate() { scaled_runs(part, data, 0, block, i == 0) } + }); + } else { + parallel::split(y, unit, &|first, part| { + for (s, strip) in part.chunks_mut(VECTOR_STRIP).enumerate() { + for (i, block) in columns.chunks(VECTOR_ROWS).enumerate() { scaled_runs(strip, data, first + s * VECTOR_STRIP, block, i == 0) } + } + true + }); + } + } + } else { + let dots = match k { + 1 => narrow_dots::, 2 => narrow_dots::, 3 => narrow_dots::, 4 => narrow_dots::, 5 => narrow_dots::, + 6 => narrow_dots::, 7 => narrow_dots::, 8 => narrow_dots::, _ => dots:: + }; + parallel::split(y, unit, &|first, part| { dots(part, data, ld, first, &x); true }); + } + if direct { return true; } + // As the packed kernels do, `alpha` scales only where it isn't 1, which keeps a complex result's infinite part from making NaN. + for (i, &y) in kept.iter().enumerate() { + let y = if conj { y.conj() } else { y }; + let (y, z) = (if one { y } else { alpha * y }, &mut c[i * step]); + *z = if start { y } else { beta.mul_acc(*z, y) }; + } + true +} + +/// `gemm` on rustymath's own packed kernels, for a nonzero `k` and `alpha`. +#[cfg(native)] +fn native_gemm(c: &mut [T], ldc: usize, a: Matrix<'_, T>, b: Matrix<'_, T>, [m, k, n]: [usize; 3], alpha: T, beta: T, part: Part) { + let start = beta == T::zero(); + if !start { scale(c, ldc, [m, n], beta, part) } + T::native_product(c, ldc, a, alpha, b, [m, k, n], start, part) +} +/// The items of a product's result that it must fold: all of them, or those on and below its diagonal. +#[derive(Clone, Copy)] +pub(crate) enum Part { Whole, Lower } +impl Part { + /// The columns of row `i` that this part holds, in a row of `n` items. + #[inline(always)] + fn columns(self, i: usize, n: usize) -> std::ops::Range { match self { Part::Whole => 0..n, Part::Lower => 0..(i + 1).min(n) } } +} +macro_rules! tiled { + (@real $t:ty, $tile:ident, $kernel:ident; $c:ident, $ldc:ident, $a:ident, $alpha:ident, $b:ident, $size:ident, $start:ident, $part:ident) => {{ + const MR: usize = tiles::$tile[0]; + const NR: usize = tiles::$tile[1]; + let one = <$t>::one(); + product::<$t, PlusTimes, MR, NR>($c, $ldc, |rows, p0, out| $a.strip(rows, p0, $alpha, out), |columns, p0, out| $b.transpose().strip(columns, p0, one, out), $size, $start, $part, $kernel::() / tiles::BYTES }>) + }}; + (@complex $r:ty; $c:ident, $ldc:ident, $a:ident, $alpha:ident, $b:ident, $size:ident, $start:ident, $part:ident) => { + complex_product::<$r>($c, $ldc, $a, $alpha, $b, $size, $start, $part) + }; + (@eig real; $a:ident, $n:ident, $vectors:ident) => { from_order!($n, accelerate::GEEV_FROM, accelerate::eig($a, $n, $vectors), factor::general::native($a, $n, $vectors)) }; + (@eig complex; $a:ident, $n:ident, $vectors:ident) => { factor::general::native($a, $n, $vectors) }; + ($($t:ty: $kind:ident($($args:tt)*);)+) => {$( + impl Tiled for $t { + #[cfg(native)] + #[inline] + fn native_product(c: &mut [$t], ldc: usize, a: Matrix<'_, $t>, alpha: $t, b: Matrix<'_, $t>, size: [usize; 3], start: bool, part: Part) { + tiled!(@$kind $($args)*; c, ldc, a, alpha, b, size, start, part) + } + fn svd(a: &[$t], size: [usize; 2], vectors: bool) -> Option<(Vec<$t>, Vec<<$t as Scalar>::Real>, Vec<$t>)> { + from_order!(size[0].max(size[1]), accelerate::GESDD_FROM, accelerate::svd(a, size, vectors), factor::svd::native(a, size, vectors)) + } + fn eigh(a: &[$t], n: usize, vectors: bool) -> Option> { + from_order!(n, accelerate::HEEVD_FROM, accelerate::eigh(a, n, vectors), factor::eigen::native(a, n, vectors)) + } + fn eig(a: &[$t], n: usize, vectors: bool) -> Option::Real>> { + tiled!(@eig $kind; a, n, vectors) + } + } + impl Inner<$t> for PlusTimes { + type Out = $t; + fn inner_ops(&self, a: &[$t], b: &[$t], [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3]) -> Vec<$t> { + assert!(a.len() == m * k && b.len() == k * n, "`a` must hold `m`×`k` items and `b` `k`×`n`"); + <$t>::multiply(Matrix::oriented(a, oa, [m, k]), Matrix::oriented(b, ob, [k, n]), [m, k, n]) + } + #[inline] + fn inner_into(&self, a: &[$t], b: &[$t], [m, k, n]: [usize; 3], c: &mut [$t]) { + assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); + if m * k * n <= DIRECT_WORK { return direct::<$t, $t, PlusTimes>(a, b, [m, k, n], c) } + <$t>::gemm(c, n, Matrix::rows(a, k), Matrix::rows(b, n), [m, k, n], <$t>::one(), <$t>::zero()) + } + fn inner_mapped_into(&self, a: Mapped<'_, $t>, b: Mapped<'_, $t>, orientations: [Orientation; 2], size: [usize; 3], out: &mut [$t]) { blocked(a, b, orientations, size, out) } + } + )+}; +} +tiled! { + f32: real(f32, TIMES32, times32); + f64: real(f64, TIMES64, times64); + Complex: complex(f32); + Complex: complex(f64); +} +/// `alpha a b`, from four real products of the parts: real parts `aᵣbᵣ - aᵢbᵢ` and imaginary parts `aᵣbᵢ + aᵢbᵣ`, as scalar multiplication forms them. `parts` splits each operand once, before the real products. +#[cfg(native)] +fn complex_product( + c: &mut [Complex], ldc: usize, a: Matrix<'_, Complex>, alpha: Complex, b: Matrix<'_, Complex>, size: [usize; 3], start: bool, part: Part) { + let [m, k, n] = size; + if m == 0 || k == 0 || n == 0 { return; } + let ([ar, ai], [br, bi]) = (parts(a, [m, k], alpha), parts(b, [k, n], Complex::new(R::one(), R::zero()))); + let real = |c: &mut [R], x: &[R], y: &[R], scale: R, start: bool| R::native_product(c, n, Matrix::rows(x, k), scale, Matrix::rows(y, n), size, start, part); + let (mut re, mut im) = (filled(m * n, R::zero()), filled(m * n, R::zero())); + real(&mut re, &ar, &br, R::one(), true); + real(&mut re, &ai, &bi, -R::one(), false); + real(&mut im, &ar, &bi, R::one(), true); + real(&mut im, &ai, &br, R::one(), false); + for i in 0..m { + for j in part.columns(i, n) { + let (t, x) = (i * n + j, &mut c[i * ldc + j]); + *x = if start { Complex::new(re[t], im[t]) } else { Complex::new(x.re + re[t], x.im + im[t]) }; + } + } +} + +/// The real parts and the imaginary parts of `scale` times the `rows`×`cols` matrix that `x` views, each row by row. +#[cfg(native)] +fn parts(x: Matrix<'_, Complex>, [rows, cols]: [usize; 2], scale: Complex) -> [Vec; 2] { + let (one, conj) = (scale == Complex::new(R::one(), R::zero()), matches!(x.op, Op::Adjoint | Op::Conjugate)); + let value = |v: Complex| { let v = if conj { v.conj() } else { v }; if one { v } else { scale * v } }; + let mut z = filled(rows * cols, Complex::default()); + if x.op.transposed() { transpose_into(x.data, x.ld, &mut z, cols, [cols, rows], value) } else { + for (row, out) in z.chunks_exact_mut(cols).enumerate() { for (o, &v) in out.iter_mut().zip(&x.data[row * x.ld..][..cols]) { *o = value(v) } } + } + [z.iter().map(|v| v.re).collect(), z.iter().map(|v| v.im).collect()] +} + +/// The micro-kernels in plain Rust, for targets without the explicit ones. Each takes the number of vectors in a tile row, `V`, as the explicit ones do, and ignores it. +#[cfg(not(any( + target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") +)))] +mod plain { + use super::*; + macro_rules! plain { + ($($name:ident: $t:ty, $object:ident;)+) => {$( + #[inline(always)] + pub(super) fn $name( + a: &[$t], b: &[$t], t: &mut [[<$object as Pair<$t>>::Out; NR]; MR], first: bool, + ) { + micro::<$t, $object, MR, NR>(a, b, t, first) + } + )+}; + } + plain! { + times64: f64, PlusTimes; + times32: f32, PlusTimes; + times_i32: i32, PlusTimes; + max_plus64: f64, MaxPlus; + max_plus32: f32, MaxPlus; + min_plus64: f64, MinPlus; + min_plus32: f32, MinPlus; + equal64: f64, PlusEqual; + equal32: f32, PlusEqual; + equal_i64: i64, PlusEqual; + equal_i32: i32, PlusEqual; + equal_i16: i16, PlusEqual; + } +} +#[cfg(not(any( + target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") +)))] +use plain::*; + +/// Micro-kernels on explicit vectors: 16 bytes on AArch64 and in wasm, and 32 with AVX2. Each is `super::micro` on vectors, with each row of the tile `V` vectors, and its entry gives one step from the tile's running value `c`, `a` and `b`. A tile starts where its entry says, at the pair's unit for a tile of items, as `micro`'s does, so the first step drops a NaN term as the later steps do. +#[cfg(any( + target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") +))] +mod lanes { + #[cfg(target_arch = "aarch64")] + use core::arch::aarch64::*; + #[cfg(target_arch = "wasm32")] + use core::arch::wasm32::*; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::*; + macro_rules! kernels { + ($($name:ident: $t:ty => $out:ty, $lanes:literal, $splat:ident, $load:ident, $start:expr, $load_out:ident, $store_out:ident, |$a:ident, $b:ident, $c:ident| $step:expr;)+) => {$( + #[inline(always)] + pub(super) fn $name(a: &[$t], b: &[$t], t: &mut [[$out; NR]; MR], first: bool) { + assert!(NR == $lanes * V && a.len() / MR == b.len() / NR); + let steps = a.len() / MR; + // SAFETY: this module compiles only for targets with these instructions. Each load and store moves `$lanes` items inside `a`, `b` or a row of `t`. + unsafe { + let (ap, bp) = (a.as_ptr(), b.as_ptr()); + let mut acc = [[$start; V]; MR]; + if !first { for r in 0..MR { for v in 0..V { acc[r][v] = $load_out(t[r].as_ptr().add($lanes * v).cast()) } } } + for p in 0..steps { + let y: [_; V] = core::array::from_fn(|v| $load(bp.add(p * NR + $lanes * v).cast())); + for r in 0..MR { + let $a = $splat(*ap.add(p * MR + r) as _); + for v in 0..V { acc[r][v] = { let ($c, $b) = (acc[r][v], y[v]); $step } } + } + } + for r in 0..MR { for v in 0..V { $store_out(t[r].as_mut_ptr().add($lanes * v).cast(), acc[r][v]) } } + } + } + )+}; + } + /// `kernels!` for tiles of items, which start at the entry's pair's unit. + macro_rules! lanes { + ($($name:ident: $t:ty, $pair:ident, $lanes:literal, $splat:ident, $load:ident, $store:ident, |$a:ident, $b:ident, $c:ident| $step:expr;)+) => { + kernels! { $($name: $t => $t, $lanes, $splat, $load, $splat(>::UNIT), $load, $store, |$a, $b, $c| $step;)+ } + }; + } + // rustymath's own plus-times products, which macOS compiles only with the `native` feature. + #[cfg(all(target_arch = "aarch64", native))] + lanes! { + times64: f64, PlusTimes, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vfmaq_f64(c, a, b); + times32: f32, PlusTimes, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vfmaq_f32(c, a, b); + } + #[cfg(target_arch = "aarch64")] + lanes! { + // Integer plus-times products have no Accelerate form, so every target compiles them. + times_i32: i32, PlusTimes, 4, vdupq_n_s32, vld1q_s32, vst1q_s32, |a, b, c| vmlaq_s32(c, a, b); + max_plus64: f64, MaxPlus, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vmaxnmq_f64(c, vaddq_f64(a, b)); + max_plus32: f32, MaxPlus, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vmaxnmq_f32(c, vaddq_f32(a, b)); + min_plus64: f64, MinPlus, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vminnmq_f64(c, vaddq_f64(a, b)); + min_plus32: f32, MinPlus, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vminnmq_f32(c, vaddq_f32(a, b)); + } + // Baseline wasm has no fused multiply-add. + #[cfg(target_arch = "wasm32")] + lanes! { + times64: f64, PlusTimes, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_add(c, f64x2_mul(a, b)); + times32: f32, PlusTimes, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_add(c, f32x4_mul(a, b)); + times_i32: i32, PlusTimes, 4, i32x4_splat, v128_load, v128_store, |a, b, c| i32x4_add(c, i32x4_mul(a, b)); + max_plus64: f64, MaxPlus, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_pmax(c, f64x2_add(a, b)); + max_plus32: f32, MaxPlus, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_pmax(c, f32x4_add(a, b)); + min_plus64: f64, MinPlus, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_pmin(c, f64x2_add(a, b)); + min_plus32: f32, MinPlus, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_pmin(c, f32x4_add(a, b)); + } + // rustymath's own plus-times products, which macOS compiles only with the `native` feature. + #[cfg(all(target_arch = "x86_64", native))] + lanes! { + times64: f64, PlusTimes, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_fmadd_pd(a, b, c); + times32: f32, PlusTimes, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_fmadd_ps(a, b, c); + } + #[cfg(target_arch = "x86_64")] + lanes! { + // Integer plus-times products have no Accelerate form, so every target compiles them. AVX2 has no multiply-add on 32-bit integers. + times_i32: i32, PlusTimes, 8, _mm256_set1_epi32, _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| _mm256_add_epi32(c, _mm256_mullo_epi32(a, b)); + // AVX2's `maxpd` and `minpd` give their second argument beside NaN: the running value `c`. + max_plus64: f64, MaxPlus, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_max_pd(_mm256_add_pd(a, b), c); + max_plus32: f32, MaxPlus, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_max_ps(_mm256_add_ps(a, b), c); + min_plus64: f64, MinPlus, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_min_pd(_mm256_add_pd(a, b), c); + min_plus32: f32, MinPlus, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_min_ps(_mm256_add_ps(a, b), c); + } + // `PlusEqual`'s tiles count equal pairs. Where items are equal, a comparison gives a lane of all ones, which is -1 as an integer, so subtracting it adds 1 to the count. + #[cfg(target_arch = "aarch64")] + kernels! { + equal64: f64 => i64, 2, vdupq_n_f64, vld1q_f64, vdupq_n_s64(0), vld1q_s64, vst1q_s64, |a, b, c| vsubq_s64(c, vreinterpretq_s64_u64(vceqq_f64(a, b))); + equal32: f32 => i32, 4, vdupq_n_f32, vld1q_f32, vdupq_n_s32(0), vld1q_s32, vst1q_s32, |a, b, c| vsubq_s32(c, vreinterpretq_s32_u32(vceqq_f32(a, b))); + equal_i64: i64 => i64, 2, vdupq_n_s64, vld1q_s64, vdupq_n_s64(0), vld1q_s64, vst1q_s64, |a, b, c| vsubq_s64(c, vreinterpretq_s64_u64(vceqq_s64(a, b))); + equal_i32: i32 => i32, 4, vdupq_n_s32, vld1q_s32, vdupq_n_s32(0), vld1q_s32, vst1q_s32, |a, b, c| vsubq_s32(c, vreinterpretq_s32_u32(vceqq_s32(a, b))); + equal_i16: i16 => i16, 8, vdupq_n_s16, vld1q_s16, vdupq_n_s16(0), vld1q_s16, vst1q_s16, |a, b, c| vsubq_s16(c, vreinterpretq_s16_u16(vceqq_s16(a, b))); + } + #[cfg(target_arch = "wasm32")] + kernels! { + equal64: f64 => i64, 2, f64x2_splat, v128_load, i64x2_splat(0), v128_load, v128_store, |a, b, c| i64x2_sub(c, f64x2_eq(a, b)); + equal32: f32 => i32, 4, f32x4_splat, v128_load, i32x4_splat(0), v128_load, v128_store, |a, b, c| i32x4_sub(c, f32x4_eq(a, b)); + equal_i64: i64 => i64, 2, i64x2_splat, v128_load, i64x2_splat(0), v128_load, v128_store, |a, b, c| i64x2_sub(c, i64x2_eq(a, b)); + equal_i32: i32 => i32, 4, i32x4_splat, v128_load, i32x4_splat(0), v128_load, v128_store, |a, b, c| i32x4_sub(c, i32x4_eq(a, b)); + equal_i16: i16 => i16, 8, i16x8_splat, v128_load, i16x8_splat(0), v128_load, v128_store, |a, b, c| i16x8_sub(c, i16x8_eq(a, b)); + } + #[cfg(target_arch = "x86_64")] + kernels! { + equal64: f64 => i64, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { + _mm256_sub_epi64(c, _mm256_castpd_si256(_mm256_cmp_pd::<_CMP_EQ_OQ>(a, b))) + }; + equal32: f32 => i32, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { + _mm256_sub_epi32(c, _mm256_castps_si256(_mm256_cmp_ps::<_CMP_EQ_OQ>(a, b))) + }; + equal_i64: i64 => i64, 4, _mm256_set1_epi64x, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { + _mm256_sub_epi64(c, _mm256_cmpeq_epi64(a, b)) + }; + equal_i32: i32 => i32, 8, _mm256_set1_epi32, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { + _mm256_sub_epi32(c, _mm256_cmpeq_epi32(a, b)) + }; + equal_i16: i16 => i16, 16, _mm256_set1_epi16, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { + _mm256_sub_epi16(c, _mm256_cmpeq_epi16(a, b)) + }; + } +} +#[cfg(any( + target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") +))] +use lanes::*; + +/// The multiply-adds in each band of rows that `product` hands to a thread. On an M5 Max, 2^19 made 512×512 QR and inverses 5 to 13% faster on 4 threads than 2^20 did. 2^18 made `+.×`, Cholesky and QR 10 to 40% slower on 18 threads than 2^19. +const BAND_WORK: usize = 1 << 19; + +/// The multiply-adds that waking one more thread costs a product. A panel of `w` multiply-adds splits across at most √(w / `THREAD_WORK`) threads, the count that minimises w / t + t × `THREAD_WORK`. On an M5 Max, `+.×` of two 256×256 matrices took 21.6 copies on 8 threads and 28.8 on 18, and larger products were fastest on all 18. +const THREAD_WORK: usize = 1 << 17; + +/// The most items of `b` that `product` packs into one panel. It splits each panel's bands across threads in one call, however many steps of `KC` the panel holds. +const PANEL: usize = 1 << 22; +/// The most bytes of `b` that `product` packs into one panel when this thread's calls run serially, as on a pool thread. Larger panels outgrow the cache: on an M5 Max, a one-thread Gram product of a 16384×256 f64 matrix took 3% longer with 8 MB panels and 10% longer with 32 MB ones, and a threaded tall QR whose threads each ran one such product was 2% slower with 4 MB panels than with 2 MB ones. +const SERIAL_PANEL: usize = 2 << 20; +/// The steps of `b` that `product` packs across all of a panel's strips before packing the next steps. Packing a strip reads `NR` items from each row of a row-major `b`. Unless those items fill whole cache lines, the next strip reads the rest of each line. When `b`'s rows were a power of two apart, packing one strip at a time had already evicted those lines: on an M5 Max, a 64×256×16384 f64 product took 15 to 28% longer than with 16 steps at a time. +const ROWS: usize = 16; +/// The columns of the result that `product` fills one row of tiles at a time, before the next row of tiles. Filling each row of tiles in turn writes each row of `c` in order, whatever `ldc` is. Sweeping the whole width instead wrote the rows of a column of tiles together, and a power-of-two `ldc` maps those rows to the same cache sets: on an M5 Max, a 64×32×16384 f64 product took 2.3 times as long as a 64×32×16000 one. 128 columns of a packed `KC`-step f64 block of `b` hold 256 KB, which stays in a Zen 4 core's 1 MB second-level cache while each row of tiles rereads it. +const SWEEP: usize = 128; +/// Folds the inner product of `P` into `c`, an `m`×`n` matrix whose rows start `ldc` items apart, with `kernel` folding each `MR`×`NR` tile as `micro` does. `a(rows, p0, steps)` writes a strip of the `m`×`k` matrix `a`, and `b(columns, p0, steps)` a strip of the transpose of the `k`×`n` matrix `b`, as `Matrix::strip` does. With `start`, the product replaces `c`, which must then hold `f`'s identity where `k` is 0. Otherwise each result folds into `c`'s item with `f`. Only tiles that hold items of `part` fold. Items outside `part` in those tiles fold too. Each block of `NC` columns packs `b` in panels of at most [`PANEL`] items, or [`SERIAL_PANEL`] bytes on one thread, [`ROWS`] steps at a time across the panel's strips, across threads as [`parallel::split`] chooses. Then it runs bands of rows across at most √(w / [`THREAD_WORK`]) of this thread's [`parallel::threads`], for a panel of `w` multiply-adds: about one band for each [`BAND_WORK`] multiply-adds, up to four bands for each thread, each at most `MC` rows. Each band runs the panel's steps of `KC` in turn, packing its own block of `a` for each. Within a block of `a`, it fills one row of tiles at a time across [`SWEEP`] columns of the result. +#[inline(always)] +fn product, const MR: usize, const NR: usize>( + c: &mut [P::Out], ldc: usize, a: impl Fn([usize; 2], usize, &mut [[MaybeUninit; MR]]) + Sync, + b: impl Fn([usize; 2], usize, &mut [[MaybeUninit; NR]]) + Sync, [m, k, n]: [usize; 3], start: bool, part: Part, + kernel: impl Fn(&[T], &[T], &mut [[P::Out; NR]; MR], bool) + Sync) { + if m == 0 || k == 0 || n == 0 { return; } + assert!(ldc >= n && c.len() >= (m - 1) * ldc + n, "`c` must hold `m` rows of `n` items, `ldc` apart"); + let c = &mut c[..(m - 1) * ldc + n]; + let width = NC.min(n).next_multiple_of(NR); + let panel = if parallel::threads() == 1 { SERIAL_PANEL / size_of::() } else { PANEL }; + let depth = (panel / (width * KC)).clamp(1, k.div_ceil(KC)) * KC; + let mut bp = Vec::with_capacity(width * depth.min(k)); + for jc in (0..n).step_by(NC) { + let nc = NC.min(n - jc); + for pp in (0..k).step_by(depth) { + let kp = depth.min(k - pp); + let unit = NR * kp; + let work = m * kp * nc; + let threads = if work < 2 * BAND_WORK { 1 } else { parallel::threads().min((work / THREAD_WORK).isqrt()) }; + let band_rows = if threads == 1 { m } else { m.div_ceil((work / BAND_WORK).min(4 * threads)).next_multiple_of(MR).min(MC) }; + let pack_b = |at: usize, units: &mut [MaybeUninit]| { + for p0 in (0..kp).step_by(ROWS) { + let rows = ROWS.min(kp - p0); + for (u, strip) in units.chunks_exact_mut(unit).enumerate() { + let s = at / unit + u; + let (j, lines) = (jc + s * NR, NR.min(nc - s * NR)); + pack::<_, NR>(&mut strip[p0 * NR..][..rows * NR], rows, lines, |first, ln, steps| b([j + first, ln], pp + p0, steps)); + } + } + true + }; + bp.clear(); + unsafe { + slice::appended(&mut bp, nc.div_ceil(NR) * unit, |slots| if threads == 1 { pack_b(0, slots) } else { parallel::split(slots, unit, &pack_b) }) + }; + let (a, bp, kernel) = (&a, &bp[..], &kernel); + let lower = matches!(part, Part::Lower); + parallel::pieces_of(c, band_rows * ldc, threads, &move |first, band| { + let (i0, i1) = (first / ldc, (first / ldc + band_rows).min(m)); + let mut slots = Box::<[T]>::new_uninit_slice(MC.min(i1 - i0).next_multiple_of(MR) * KC.min(kp)); + for step in 0..kp.div_ceil(KC) { + let pc = pp + step * KC; + let (kc, load) = (KC.min(k - pc), pc > 0 || !start); + for ic in (i0..i1).step_by(MC) { + let mc = MC.min(i1 - ic); + if lower && ic + mc <= jc { continue; } + let ap = pack::<_, MR>(&mut slots, kc, mc, |first, lines, steps| a([ic + first, lines], pc, steps)); + for jb in (0..nc).step_by(SWEEP) { + for ir in (0..mc).step_by(MR) { + for jr in (jb..nc.min(jb + SWEEP)).step_by(NR) { + let bj = &bp[jr / NR * unit + step * NR * KC..][..NR * kc]; + if lower && ic + ir + MR.min(mc - ir) <= jc + jr { continue; } + let (rows, cols) = (MR.min(mc - ir), NR.min(nc - jr)); + let at = |r: usize| (ic - i0 + ir + r) * ldc + jc + jr; + let mut t = [[P::UNIT; NR]; MR]; + // A whole row moves as one run of `NR` items, which compiles to vector moves. Item by item, each move compiles to a scalar load, a store and a bounds check. + if load { + for r in 0..rows { + if cols == NR { t[r].copy_from_slice(&band[at(r)..][..NR]) } else { for s in 0..cols { t[r][s] = band[at(r) + s] } } + } + } + kernel(&ap[ir * kc..(ir + MR) * kc], bj, &mut t, !load); + for r in 0..rows { + if cols == NR { band[at(r)..][..NR].copy_from_slice(&t[r]) } else { for s in 0..cols { band[at(r) + s] = t[r][s] } } + } + } + } + } + } + } + true + }); + } + } +} + +/// Packs `len` lines of `kc` steps into strips of `W` lines at the start of `buffer`, and gives the items it wrote. Each strip holds its `W` items for step 0, then those for step 1, and so on. `strip(first, lines, steps)` writes all `W` items of each of the strip's `kc` steps: `lines` items for lines `first` on, then the default. The results computed from the defaults are never stored. +fn pack( + buffer: &mut [MaybeUninit], kc: usize, len: usize, strip: impl Fn(usize, usize, &mut [[MaybeUninit; W]])) -> &[T] { + let strips = len.div_ceil(W); + for (s, chunk) in buffer.chunks_exact_mut(W * kc).take(strips).enumerate() { strip(s * W, W.min(len - s * W), chunk.as_chunks_mut::().0); } + unsafe { slice::written(&mut buffer[..strips * W * kc]) } +} + +/// Folds the steps of the strips `a` and `b`, which `pack` wrote, into the tile `t`. With `first`, the tile starts at the pair's unit, as `direct`'s sums do. +#[inline(always)] +fn micro, const MR: usize, const NR: usize>(a: &[T], b: &[T], t: &mut [[P::Out; NR]; MR], first: bool) { + let mut acc = if first { [[P::UNIT; NR]; MR] } else { *t }; + for (a, b) in a.as_chunks::().0.iter().zip(b.as_chunks::().0) { for r in 0..MR { for s in 0..NR { acc[r][s] = P::step(acc[r][s], a[r], b[s]) } } } + *t = acc; +} diff --git a/src/linalg/accelerate.rs b/core/src/linalg/accelerate.rs similarity index 100% rename from src/linalg/accelerate.rs rename to core/src/linalg/accelerate.rs diff --git a/src/linalg/exact.rs b/core/src/linalg/exact.rs similarity index 100% rename from src/linalg/exact.rs rename to core/src/linalg/exact.rs diff --git a/src/linalg/factor.rs b/core/src/linalg/factor.rs similarity index 99% rename from src/linalg/factor.rs rename to core/src/linalg/factor.rs index de17634..177fe4f 100644 --- a/src/linalg/factor.rs +++ b/core/src/linalg/factor.rs @@ -518,13 +518,15 @@ pub(super) fn transpose_by_into( a: &[T], [ls, lt]: [usize; 2], [rows, cols]: [usize; 2], f: impl Fn(T) -> T + Sync, t: &mut Vec) { t.clear(); if rows * cols == 0 { return; } - appended(t, lt * cols, |slots| { - parallel::split(slots, lt, &|start, part| { - transpose_into(&a[start / lt..], ls, part, lt, [rows, part.len() / lt], |x| MaybeUninit::new(f(x))); - for row in part.chunks_exact_mut(lt) { row[rows..].fill(MaybeUninit::new(T::zero())) } - true + unsafe { + appended(t, lt * cols, |slots| { + parallel::split(slots, lt, &|start, part| { + transpose_into(&a[start / lt..], ls, part, lt, [rows, part.len() / lt], |x| MaybeUninit::new(f(x))); + for row in part.chunks_exact_mut(lt) { row[rows..].fill(MaybeUninit::new(T::zero())) } + true + }) }) - }); + }; } /// The transpose of the `m`×`n` matrix `a`. diff --git a/src/linalg/factor/eigen.rs b/core/src/linalg/factor/eigen.rs similarity index 100% rename from src/linalg/factor/eigen.rs rename to core/src/linalg/factor/eigen.rs diff --git a/src/linalg/factor/general.rs b/core/src/linalg/factor/general.rs similarity index 100% rename from src/linalg/factor/general.rs rename to core/src/linalg/factor/general.rs diff --git a/src/linalg/factor/secular.rs b/core/src/linalg/factor/secular.rs similarity index 100% rename from src/linalg/factor/secular.rs rename to core/src/linalg/factor/secular.rs diff --git a/src/linalg/factor/svd.rs b/core/src/linalg/factor/svd.rs similarity index 100% rename from src/linalg/factor/svd.rs rename to core/src/linalg/factor/svd.rs diff --git a/src/linalg/factor/truncated.rs b/core/src/linalg/factor/truncated.rs similarity index 100% rename from src/linalg/factor/truncated.rs rename to core/src/linalg/factor/truncated.rs diff --git a/src/linalg/inner.rs b/core/src/linalg/inner.rs similarity index 100% rename from src/linalg/inner.rs rename to core/src/linalg/inner.rs diff --git a/src/linalg/scalar.rs b/core/src/linalg/scalar.rs similarity index 100% rename from src/linalg/scalar.rs rename to core/src/linalg/scalar.rs diff --git a/src/ln.rs b/core/src/ln.rs similarity index 83% rename from src/ln.rs rename to core/src/ln.rs index 8629f8b..f86f3d8 100644 --- a/src/ln.rs +++ b/core/src/ln.rs @@ -1,6 +1,7 @@ //! Logarithms. They share SLEEF's reduction from `xlog` and `xlogf`: x = m × 2^e with m from 0.75 to 1.5, and ln m = ln((1+s)/(1-s)) with s = (m-1)/(m+1), from SLEEF's f64 polynomial and minimax ones for narrower results. See `THIRD-PARTY.md`. +use crate::slice::{Arg, each_usual, none, written}; use crate::{dd::Double, float::{Native, horner, ln2, sealed::Core, split_positive}}; -use std::f64::consts::{LN_2, LOG2_E, LOG10_E}; +use std::{f64::consts::{LN_2, LOG2_E, LOG10_E}, mem::MaybeUninit}; /// The coefficients of (ln((1+s)/(1-s)) - 2s)/s³ as a polynomial in s², highest degree first, for |s| up to 0.2. const POLY_64: [f64; 7] = [ @@ -109,6 +110,42 @@ pub struct Logb; #[derive(Clone, Copy, Debug)] pub struct MathishLogb; +#[inline(always)] +pub(crate) fn logb_mode(b: R, x: R) -> R { + let y = logb::<_, { crate::FUSED }>(b, x); + if MATHISH && b == R::one() && x == R::one() { R::zero() } else { y } +} + +/// Hoists the constant operand's logarithm and retains base 2/10's exact powers. +pub(crate) fn logb_many([b, x]: [Arg<'_, R>; 2], n: usize, out: &mut [MaybeUninit]) -> bool { + let out = &mut out[..n]; + match (b, x) { + (Arg::One(c), _) => { + // Bases 2 and 10 divide by 1, which their loop skips, and take the logarithms' fast form for usual items. Any other base divides by ln b, which can be 0 or NaN, so its loop judges every result. + let (to, d) = for_base::<_, { crate::FUSED }>(c); + if MATHISH && c == R::one() { + each!([b, x], n, out, |[_, x]| if x == R::one() { R::zero() } else { log::<_, { crate::FUSED }>(x, to) / d }) + } else if d == R::one() { + each_usual( + x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), to),), none(), #[inline(always)] + |x| log::<_, { crate::FUSED }>(x, to)) + } else { each!([b, x], n, out, |[_, x]| log::<_, { crate::FUSED }>(x, to) / d) } + } + (Arg::Many(bs), Arg::One(c)) => { + // A base of exactly 2 or 10 takes `log2` or `log10` of c. A second pass writes those items, because a select in the main loop stops it vectorizing. At those bases either result is NaN only where c is negative or NaN. The main loop's outcome holds. + let l = ln::<_, { crate::FUSED }>(c); + let ok = if MATHISH && c == R::one() { + each!([b, x], n, out, |[b, _]| if b == R::one() { R::zero() } else { l / ln::<_, { crate::FUSED }>(b) }) + } else { each!([b, x], n, out, |[b, _]| l / ln::<_, { crate::FUSED }>(b)) }; + let exact = [log2::<_, { crate::FUSED }>(c), log10::<_, { crate::FUSED }>(c)]; + // SAFETY: each! writes every output, including when it reports an invalid result. + for (r, &b) in unsafe { written(out) }.iter_mut().zip(bs) { *r = by_base(b, [exact[0], exact[1], *r]); } + ok + } + _ => each!([b, x], n, out, |[b, x]| logb_mode::<_, MATHISH>(b, x)), + } +} + /// ln u + c/u, for u = 1 + x split as `(m, e)` and c = x - (u - 1), the part of 1 + x that rounding u lost. It gives none of `log1p`'s special values. #[inline(always)] fn log1p_of(x: R, u: R, split: (R, R)) -> R { log_of::(split, natural()) + (x - (u - R::one())) / u } @@ -195,6 +232,7 @@ pub(crate) fn formula + ?Sized>(o: &O, log_with(o, x, (crate::recipe::ln2(o), o.number(F::one()))) } +#[inline] pub(crate) fn log1p_formula + ?Sized>(o: &O, x: O::V) -> O::V { let (zero, one) = (o.number(F::zero()), o.number(F::one())); let u = o.add(x, one); diff --git a/src/memory.rs b/core/src/memory.rs similarity index 100% rename from src/memory.rs rename to core/src/memory.rs diff --git a/src/number_theory.rs b/core/src/number_theory.rs similarity index 100% rename from src/number_theory.rs rename to core/src/number_theory.rs diff --git a/src/optimize.rs b/core/src/optimize.rs similarity index 100% rename from src/optimize.rs rename to core/src/optimize.rs diff --git a/src/parallel.rs b/core/src/parallel.rs similarity index 92% rename from src/parallel.rs rename to core/src/parallel.rs index 58ce04a..05db062 100644 --- a/src/parallel.rs +++ b/core/src/parallel.rs @@ -1,7 +1,12 @@ //! Splitting work across threads. [`pieces`] cuts a call's items into pieces of the sizes its caller gives, and [`ranges`] into as many pieces as the time of its first unit justifies. The pieces run on a Rayon pool with Rayon's default number of threads: `RAYON_NUM_THREADS` where it's set, and otherwise one for each core. [`set_threads`] changes that number for the whole process. A call made on one of a pool's threads, such as a function that another call's piece runs, doesn't split again. In the browser build, which has no threads, every call runs its pieces in turn on the calling thread. use crate::slice::{Arg, CHUNK}; -use crate::{error::Error, interrupt::check, timing::Instant}; +use crate::{error::Error, interrupt::check}; +/// The clock that `parallel`'s rounds and the speed tests' timings read: the browser's in a browser build, where `std::time::Instant` panics. +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +pub use std::time::Instant; use std::{mem::MaybeUninit, ops::Range, sync::atomic::{AtomicUsize, Ordering::Relaxed}, time::Duration}; +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] +pub use web_time::Instant; /// The number of threads that calls split across, for the whole process, or 0 for the default pool's number. static THREADS: AtomicUsize = AtomicUsize::new(0); @@ -129,10 +134,14 @@ fn each_cell(count: usize, write: impl Fn(usize, &mut S) -> bool + S for first in (timed..count).step_by(round) { check()?; let cells = round.min(count - first); - ok &= pieces(cells, cells.div_ceil(threads * PIECES).max(least).max(1), usize::MAX, &|range| { - let mut scratch = S::default(); - range.map(|c| first + c).fold(true, |ok, i| write(i, &mut scratch) & ok) - }); + if threads == 1 { + ok &= (first..first + cells).fold(true, |ok, i| write(i, &mut scratch) & ok); + } else { + ok &= pieces(cells, cells.div_ceil(threads * PIECES).max(least).max(1), usize::MAX, &|range| { + let mut scratch = S::default(); + range.map(|c| first + c).fold(true, |ok, i| write(i, &mut scratch) & ok) + }); + } } Ok(ok) } @@ -199,12 +208,14 @@ pub fn each(count: usize, f: impl Fn(usize, &mut S) -> R + /// Appends `n` items to `out`, and gives whether every call of `write` said `true`. `write(range, slots)` fills `slots` with the items in `range`, in order. The calls split across threads as [`split`] splits a call, in units of `unit` items. A call that leaves any of its slots unfilled panics, and `out` keeps its length. pub fn extended(out: &mut Vec, n: usize, unit: usize, write: &(dyn Fn(Range, &mut Slots<'_, B>) -> bool + Sync)) -> bool { - crate::slice::appended(out, n, |slots| { - split(slots, unit, &|start, slots| { - let end = start + slots.len(); - filled(slots, |slots| write(start..end, slots)) + unsafe { + crate::slice::appended(out, n, |slots| { + split(slots, unit, &|start, slots| { + let end = start + slots.len(); + filled(slots, |slots| write(start..end, slots)) + }) }) - }) + } } /// `write`'s result for `slots`, which it fills in order through one [`Slots`]. Panics where it leaves any unfilled. @@ -221,18 +232,26 @@ pub struct Slots<'a, B> { slots: &'a mut [MaybeUninit], written: usize } impl Slots<'_, B> { /// Appends `n` items to `out` on this thread, and gives `write`'s result. `write` fills the slots in order. Where it leaves any unfilled, `append` panics and `out` keeps its length. pub fn append(out: &mut Vec, n: usize, write: impl FnOnce(&mut Slots<'_, B>) -> T) -> T { - crate::slice::appended(out, n, |slots| filled(slots, write)) + unsafe { crate::slice::appended(out, n, |slots| filled(slots, write)) } } /// Writes the `n` results of [`pairs`](crate::pairs) after the items already written, on this thread, and gives whether every result succeeded. Panics where they don't fit. pub fn pairs(&mut self, x: Arg<'_, X>, y: Arg<'_, Y>, n: usize, f: impl Fn(X, Y) -> (B, bool)) -> bool { - self.fill(n, |slots| crate::slice::pairs_into(x, y, n, slots, f)) + unsafe { self.fill(n, |slots| crate::slice::pairs_into(x, y, n, slots, f)) } } /// Writes `items` after the items already written, on this thread. Panics where they don't fit. pub fn copy(&mut self, items: &[B]) where B: Copy { - self.fill(items.len(), |slots| { for (slot, &item) in slots.iter_mut().zip(items) { slot.write(item); } true }); + unsafe { + self.fill(items.len(), |slots| { + for (slot, &item) in slots.iter_mut().zip(items) { slot.write(item); } + true + }) + }; } /// `write`'s result for the next `n` slots, which it must fill. They then count as written. Panics where they don't fit. - pub fn fill(&mut self, n: usize, write: impl FnOnce(&mut [MaybeUninit]) -> bool) -> bool { + /// + /// # Safety + /// `write` must initialize every slot it receives before returning, whether it returns `true` or `false`. + pub unsafe fn fill(&mut self, n: usize, write: impl FnOnce(&mut [MaybeUninit]) -> bool) -> bool { let ok = write(&mut self.slots[self.written..self.written + n]); self.written += n; ok diff --git a/src/parts.rs b/core/src/parts.rs similarity index 100% rename from src/parts.rs rename to core/src/parts.rs diff --git a/src/polynomial.rs b/core/src/polynomial.rs similarity index 97% rename from src/polynomial.rs rename to core/src/polynomial.rs index fa934f6..42caad6 100644 --- a/src/polynomial.rs +++ b/core/src/polynomial.rs @@ -9,9 +9,11 @@ pub fn trimmed(c: &[T]) -> &[T] { &c[..c.iter().rposition(|k| !k.is_zer /// Appends to `out` the value of the polynomial `c` at each item of `x`, by Horner's rule. An f16 polynomial computes in f32 and rounds each result once. pub fn evaluate(c: &[F], x: &[F], out: &mut Vec) { let Some((&top, rest)) = trimmed(c).split_last() else { return out.resize(out.len() + x.len(), F::zero()) }; - crate::slice::appended(out, x.len(), |out| { - if crate::slice::running(size_of::()) == 16 { blocks::(top, rest, x, out) } else { blocks::(top, rest, x, out) } - }) + unsafe { + crate::slice::appended(out, x.len(), |out| { + if crate::slice::running(size_of::()) == 16 { blocks::(top, rest, x, out) } else { blocks::(top, rest, x, out) } + }) + } } /// `evaluate` through `horner` on blocks of `L` items, as many as `slice::running` gives. diff --git a/src/pow.rs b/core/src/pow.rs similarity index 98% rename from src/pow.rs rename to core/src/pow.rs index 6fa985f..992a3c6 100644 --- a/src/pow.rs +++ b/core/src/pow.rs @@ -206,7 +206,7 @@ pub(crate) fn power_slice(_: &Pow, [x, y]: [crate::Arg< let e = Exponent::of(k.wide(), crate::FUSED); powers(e, k, &xs[..n], &mut out[..n]); return !matches!(e, Exponent::Halves { .. } | Exponent::General) || k.is_nan() - || xs[..n].iter().zip(crate::slice::written(&mut out[..n]).iter()).all(|(x, r)| !r.is_nan() || x.is_nan()); + || xs[..n].iter().zip(unsafe { crate::slice::written(&mut out[..n]) }.iter()).all(|(x, r)| !r.is_nan() || x.is_nan()); } general_slice(x, y, n, out, false) } @@ -227,11 +227,11 @@ pub(crate) fn root_slice(_: &Root, [n, y]: [crate::Arg< if let (crate::Arg::One(k), crate::Arg::Many(ys)) = (n, y) { roots(k, &ys[..len], &mut out[..len]); return matches!(Degree::of(k.wide()), Degree::One | Degree::Three) || k.is_nan() - || ys[..len].iter().zip(crate::slice::written(&mut out[..len]).iter()).all(|(y, r)| !r.is_nan() || y.is_nan()); + || ys[..len].iter().zip(unsafe { crate::slice::written(&mut out[..len]) }.iter()).all(|(y, r)| !r.is_nan() || y.is_nan()); } let o = crate::recipe::NativeOps::<{ crate::FUSED }>::DEFAULT; general_slice(y, n, len, out, true); - let results = crate::slice::written(&mut out[..len]); + let results = unsafe { crate::slice::written(&mut out[..len]) }; for (i, r) in results.iter_mut().enumerate() { if let Some(exact) = root_formula(&o, n.nth(i), y.nth(i)) { *r = exact; } } results.iter().enumerate().all(|(i, r)| !r.is_nan() || n.nth(i).is_nan() || y.nth(i).is_nan()) } @@ -258,6 +258,7 @@ const CBRT_MAGIC: [f64; 2] = [1499661097332631.0, 2773273.0]; #[inline(always)] pub(crate) fn cbrt(x: R) -> R { cbrt_formula(&crate::recipe::NativeOps::::DEFAULT, x) } +#[inline] pub(crate) fn cbrt_formula + ?Sized>(o: &O, x: O::V) -> O::V { use crate::{buffer::FloatWidth, recipe::Integer}; let c = |x| o.number(F::narrow(x)); diff --git a/src/random.rs b/core/src/random.rs similarity index 100% rename from src/random.rs rename to core/src/random.rs diff --git a/src/recipe.rs b/core/src/recipe.rs similarity index 97% rename from src/recipe.rs rename to core/src/recipe.rs index e80f7b6..dbd5466 100644 --- a/src/recipe.rs +++ b/core/src/recipe.rs @@ -132,12 +132,12 @@ pub(crate) fn integer_compare(op: Cmp, x: u64, y: u64) -> bool { #[inline(always)] fn shifted(x: u64, count: i64) -> u64 { if count >= 64 { 0 } else if count >= 0 { x << count } else { ((x as i64) >> count.unsigned_abs().min(63)) as u64 } } -/// The named methods of `Ops`, each the operation of its id. +/// The named methods of `Ops`, each the operation of its id. Inline metadata keeps each constant operation id visible across crate boundaries, so native item loops can specialize and vectorize. macro_rules! named { (unary: $($u:ident $uid:ident),+; binary: $($b:ident $bid:ident),+; compare: $($c:ident $cid:ident),+) => { - $( fn $u(&self, x: Self::V) -> Self::V { self.unary(Unary::$uid, x) } )+ - $( fn $b(&self, x: Self::V, y: Self::V) -> Self::V { self.binary(Binary::$bid, x, y) } )+ - $( fn $c(&self, x: Self::V, y: Self::V) -> Self::B { self.compare(Cmp::$cid, x, y) } )+ + $( #[inline] fn $u(&self, x: Self::V) -> Self::V { self.unary(Unary::$uid, x) } )+ + $( #[inline] fn $b(&self, x: Self::V, y: Self::V) -> Self::V { self.binary(Binary::$bid, x, y) } )+ + $( #[inline] fn $c(&self, x: Self::V, y: Self::V) -> Self::B { self.compare(Cmp::$cid, x, y) } )+ }; } @@ -179,22 +179,31 @@ pub trait Ops { /// `x` where `m` holds, otherwise `y`. `∂`'s recorder runs a recipe on whole values, where `m` and each array case must have one layout: build them from the same arguments. fn select(&self, m: Self::B, x: Self::V, y: Self::V) -> Self::V; /// 1 where `m` holds, otherwise 0. + #[inline] fn one_where(&self, m: Self::B) -> Self::V { self.select(m, self.number(F::one()), self.number(F::zero())) } /// `v` where it's one number that every item shares, known before any item is read, such as the 3 of `x^3`. A recipe can pick a cheaper formula by it. `None` where `v` varies by item. + #[inline] fn constant(&self, _v: Self::V) -> Option { None } /// Whether values are integers, as a compiler's values at an integer type are. A recipe for an integer object asks before taking an integer-only step (`Binary::BitAnd`). Floats never are. + #[inline] fn integral(&self) -> bool { false } /// Captured tolerance; recording backends return its runtime setting slot. + #[inline] fn tolerance(&self, fallback: F) -> Self::V { self.number(fallback) } /// The sign of `x`, as [`sign`] gives it. + #[inline] fn sign(&self, x: Self::V) -> Self::V { sign(self, x) } /// `1 ÷ x`, as [`reciprocal`] gives it. + #[inline] fn reciprocal(&self, x: Self::V) -> Self::V { reciprocal(self, x) } /// `x` raised to `k`, a constant that `Exponent::of` routes off the general route, as [`power`] gives it. + #[inline] fn power(&self, k: F, x: Self::V) -> Self::V { power(self, Exponent::of(k.wide(), self.fused()), x).expect("an exponent off the general route") } /// The real part of `x^y` through the existing power formula. + #[inline] fn real_power(&self, x: Self::V, y: Self::V) -> Self::V { crate::pow::real_formula(self, x, y) } /// The partial in `x` of `x^y = r`; `halves` permits constant half-integer powers. + #[inline] fn power_slope(&self, x: Self::V, y: Self::V, r: Self::V, halves: bool) -> Self::V { crate::derivative::power_slope(self, x, y, r, halves) } named! { unary: negate Negate, magnitude Magnitude, sqrt Sqrt, floor Floor, ln Ln, sin Sin, cos Cos, sinh Sinh, cosh Cosh, sqrt1mx2 Sqrt1mx2, sqrt1px2 Sqrt1px2, digamma Digamma, exp Exp, lgamma Lgamma, complement Not; @@ -204,7 +213,7 @@ pub trait Ops { } /// Conservative range interpretation of the shared predicates. -pub(crate) struct RangeOps(pub crate::buffer::DType); +pub struct RangeOps(pub crate::buffer::DType); impl RangeOps { fn compare(op: Cmp, x: crate::domain::Range, y: crate::domain::Range) -> Option { if x.is_empty() || y.is_empty() { return None; } @@ -223,7 +232,7 @@ impl Ops for RangeOps { type B = Option; type I = crate::domain::Range; type W = crate::domain::Range; - fn width(&self) -> crate::buffer::FloatWidth { crate::operation::numpy_float(self.0) } + fn width(&self) -> crate::buffer::FloatWidth { self.0.numpy_float() } fn fused(&self) -> bool { crate::FUSED } fn integral(&self) -> bool { self.0.is_integral() } fn widen(&self, _: Self::V) -> Self::W { crate::domain::Range::ANY } diff --git a/src/recipe/integer.rs b/core/src/recipe/integer.rs similarity index 100% rename from src/recipe/integer.rs rename to core/src/recipe/integer.rs diff --git a/src/reduce.rs b/core/src/reduce.rs similarity index 89% rename from src/reduce.rs rename to core/src/reduce.rs index 12cf295..6af71f9 100644 --- a/src/reduce.rs +++ b/core/src/reduce.rs @@ -2,14 +2,48 @@ use crate::error::{Error, Kind}; pub use crate::slice::running; use crate::{ - Arg, Float, Native, agreement::{Mapping, Read, Walked, expanded_items}, arith::Overflowing, buffer::{Buffer, Cast, Element, Source, View}, - float::sealed::{Core, Folds}, gather::Steps, layout::Lanes, linalg::Scalar, operation::{Piece as InputPiece, Pieces, Seed}, parallel, - slice::{CHUNK, appended, written} + Arg, Float, Native, agreement::{Mapping, Read, Walked, expanded_items}, arith::Overflowing, buffer::{Buffer, Cast, DType, Element, Source, View, Width}, + float::sealed::{Core, Folds}, gather::Steps, layout::Lanes, linalg::Scalar, parallel, slice::{CHUNK, appended, written} }; use half::f16; use num_complex::Complex; use std::ops::Range; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Direction { Left, Right } + +#[derive(Clone, Copy)] +pub enum Seed<'a> { Identity, Initial(crate::agreement::Read<'a>), Tail(crate::agreement::Read<'a>) } +impl<'a> Seed<'a> { + /// The seed's items, or `None` for the identity. + pub fn read(self) -> Option> { match self { Self::Initial(read) | Self::Tail(read) => Some(read), Self::Identity => None } } +} + +/// A compiled fold's total over `items` items, as its 8-byte slot holds it: a float as an f64's bits, an integer extended to 64 bits, and a Boolean as 0 or 1. +#[derive(Clone, Copy, Debug)] +pub struct Total { pub items: usize, pub bits: u64 } +impl Total { + pub fn value(self) -> Result { + use crate::buffer::{Floats, Ints}; + match T::DTYPE { + DType::Boolean => T::cast(View::Booleans(&[self.bits != 0]), 0), + DType::Integer(width) => { + let value = match width { + Width::U8 => self.bits as u8 as i64, Width::I16 => self.bits as i16 as i64, Width::I32 => self.bits as i32 as i64, + Width::I64 => self.bits as i64 + }; + T::cast(View::Integers(Ints::I64(&[value])), 0) + } + DType::Float(_) => T::cast(View::Floats(Floats::F64(&[f64::from_bits(self.bits)])), 0), + _ => Err(Error::new(Kind::Domain, "compiled totals have a real machine dtype")), + } + } +} + +pub enum Piece<'a> { Items(View<'a>), Total(Total) } + +pub type Pieces<'a> = dyn Fn(std::ops::Range, &mut dyn FnMut(usize, Piece<'_>)) -> Result<(), Error> + Sync + 'a; + /// The sum of `x`, 0 for an empty slice, and whether it fits. An integer sum that doesn't fit wraps. #[inline] pub fn sum(x: &[T]) -> (>::Total, bool) where Sum: Reduction { Sum::slice(x) } @@ -103,12 +137,12 @@ pub trait Reduction { pub trait Totals: Copy { /// The buffer that holds `totals`. fn buffer_of(totals: Vec) -> crate::buffer::Buffer; - fn from_partial(_: crate::operation::Total) -> Result { Err(Error::new(Kind::Domain, "this reduction cannot read compiled partial totals")) } + fn from_partial(_: Total) -> Result { Err(Error::new(Kind::Domain, "this reduction cannot read compiled partial totals")) } } impl Totals for T { #[inline] fn buffer_of(totals: Vec) -> crate::buffer::Buffer { T::buffer(totals) } - fn from_partial(total: crate::operation::Total) -> Result { total.value() } + fn from_partial(total: Total) -> Result { total.value() } } impl Totals for (S, bool) { #[inline] @@ -200,44 +234,46 @@ const SCAN_BLOCK: usize = 64; pub(crate) fn grouped( x: &[A], start: Option, out: &mut Vec, from: impl Fn(A) -> S, f: impl Fn(S, S) -> S, to: impl Fn(S) -> O, reorders: bool) { let Some((&first, rest)) = x.split_first() else { return }; - appended(out, x.len(), |slots| { - let first = from(first); - let mut total = start.map_or(first, |s| f(s, first)); - slots[0].write(to(total)); - let ((groups, tail), (outs, last)) = (rest.as_chunks::<4>(), slots[1..].as_chunks_mut::<4>()); - let scanned = |[a, b, c, d]: [A; 4]| { - let (a, b, c, d) = (from(a), from(b), from(c), from(d)); - let ab = f(a, b); - [a, ab, f(ab, c), f(ab, f(c, d))] - }; - let (mut sums, mut ends) = ([std::mem::MaybeUninit::<[S; 4]>::uninit(); SCAN_BLOCK], [std::mem::MaybeUninit::::uninit(); SCAN_BLOCK]); - // Blocks by index: a loop over `chunks(SCAN_BLOCK)` zipped with `chunks_mut` cost `+\b` 90% more instructions. - let mut i0 = 0; - while i0 < groups.len() { - let n = (groups.len() - i0).min(SCAN_BLOCK); - let (groups, outs) = (&groups[i0..i0 + n], &mut outs[i0..i0 + n]); - if reorders { + unsafe { + appended(out, x.len(), |slots| { + let first = from(first); + let mut total = start.map_or(first, |s| f(s, first)); + slots[0].write(to(total)); + let ((groups, tail), (outs, last)) = (rest.as_chunks::<4>(), slots[1..].as_chunks_mut::<4>()); + let scanned = |[a, b, c, d]: [A; 4]| { + let (a, b, c, d) = (from(a), from(b), from(c), from(d)); + let ab = f(a, b); + [a, ab, f(ab, c), f(ab, f(c, d))] + }; + let (mut sums, mut ends) = ([std::mem::MaybeUninit::<[S; 4]>::uninit(); SCAN_BLOCK], [std::mem::MaybeUninit::::uninit(); SCAN_BLOCK]); + // Blocks by index: a loop over `chunks(SCAN_BLOCK)` zipped with `chunks_mut` cost `+\b` 90% more instructions. + let mut i0 = 0; + while i0 < groups.len() { + let n = (groups.len() - i0).min(SCAN_BLOCK); + let (groups, outs) = (&groups[i0..i0 + n], &mut outs[i0..i0 + n]); + if reorders { + for g in 0..n { + let s = scanned(groups[g]); + sums[g].write(s); + ends[g].write(s[3]); + } + } for g in 0..n { - let s = scanned(groups[g]); - sums[g].write(s); - ends[g].write(s[3]); + // SAFETY: with `reorders`, the first pass wrote entries 0 to `n`. + let sums = if reorders { sums[g].assume_init() } else { scanned(groups[g]) }; + // Four writes, not `array::map`, which cost `+\b` 53% more instructions. + for k in 0..4 { outs[g][k].write(to(f(total, sums[k]))); } + // The group's sum from its own buffer, which the compiler can't see is the last of `sums`. Taken from `sums`, the total would wait on the vector the outputs use. + total = f(total, if reorders { ends[g].assume_init() } else { sums[3] }); } + i0 += n; } - for g in 0..n { - // SAFETY: with `reorders`, the first pass wrote entries 0 to `n`. - let sums = if reorders { unsafe { sums[g].assume_init() } } else { scanned(groups[g]) }; - // Four writes, not `array::map`, which cost `+\b` 53% more instructions. - for k in 0..4 { outs[g][k].write(to(f(total, sums[k]))); } - // The group's sum from its own buffer, which the compiler can't see is the last of `sums`. Taken from `sums`, the total would wait on the vector the outputs use. - total = f(total, if reorders { unsafe { ends[g].assume_init() } } else { sums[3] }); + for (o, &v) in last.iter_mut().zip(tail) { + total = f(total, from(v)); + o.write(to(total)); } - i0 += n; - } - for (o, &v) in last.iter_mut().zip(tail) { - total = f(total, from(v)); - o.write(to(total)); - } - }) + }) + } } /// The position of the first item of `x` that `test` picks, or its length where it picks none. Each block of 64 items is tested with no branch, so the test vectorizes. @@ -480,36 +516,38 @@ mod lanes { /// The running sums of `x` after `start`, 16 items at a time. Each vector of four scans within itself in two shifted additions. Each vector then adds the total of everything before it: the running total, plus the sums of the vectors before it in its block. Only one addition in each block waits for the block before. fn sum_scan(x: &[f32], start: Option, out: &mut Vec) { use quad::*; - appended(out, x.len(), |slots| { - let ((chunks, rest), (outs, tail)) = (x.as_chunks::<16>(), slots.as_chunks_mut::<16>()); - let (fill, mut c) = (splat(-0.0), splat(start.unwrap_or(-0.0))); - for (o, x) in outs.iter_mut().zip(chunks) { - let scanned = |k: usize| { - // SAFETY: the load reads four items of one chunk. - let v = unsafe { load(x.as_ptr().add(4 * k)) }; - let v = add(v, up1(fill, v)); - add(v, up2(fill, v)) - }; - let (p0, p1, p2, p3) = (scanned(0), scanned(1), scanned(2), scanned(3)); - let s0 = last(p0); - let s01 = add(s0, last(p1)); - let s012 = add(s01, last(p2)); - let o = o.as_mut_ptr().cast::(); - // SAFETY: each store writes four slots of one chunk. - unsafe { - store(o, add(c, p0)); - store(o.add(4), add(add(c, s0), p1)); - store(o.add(8), add(add(c, s01), p2)); - store(o.add(12), add(add(c, s012), p3)); + unsafe { + appended(out, x.len(), |slots| { + let ((chunks, rest), (outs, tail)) = (x.as_chunks::<16>(), slots.as_chunks_mut::<16>()); + let (fill, mut c) = (splat(-0.0), splat(start.unwrap_or(-0.0))); + for (o, x) in outs.iter_mut().zip(chunks) { + let scanned = |k: usize| { + // SAFETY: the load reads four items of one chunk. + let v = unsafe { load(x.as_ptr().add(4 * k)) }; + let v = add(v, up1(fill, v)); + add(v, up2(fill, v)) + }; + let (p0, p1, p2, p3) = (scanned(0), scanned(1), scanned(2), scanned(3)); + let s0 = last(p0); + let s01 = add(s0, last(p1)); + let s012 = add(s01, last(p2)); + let o = o.as_mut_ptr().cast::(); + // SAFETY: each store writes four slots of one chunk. + unsafe { + store(o, add(c, p0)); + store(o.add(4), add(add(c, s0), p1)); + store(o.add(8), add(add(c, s01), p2)); + store(o.add(12), add(add(c, s012), p3)); + } + c = add(c, add(s012, last(p3))); } - c = add(c, add(s012, last(p3))); - } - let mut total = first(c); - for (o, &v) in tail.iter_mut().zip(rest) { - total += v; - o.write(total); - } - }) + let mut total = first(c); + for (o, &v) in tail.iter_mut().zip(rest) { + total += v; + o.write(total); + } + }) + } } #[cfg(target_arch = "wasm32")] lanes! { @@ -572,31 +610,35 @@ fn wide_sum(x: &[i64]) -> (i64, bool) { /// The running sums of `x` from `start` at `i64`, wrapped, and whether every one fits. A sum overflows where its sign differs from both its arguments' signs, as `Overflowing::plus` tests it. The loop gathers those sign bits in one word and tests it once at the end. A bool for each step, as `Overflowing::plus` gives, measured 7% slower. #[inline(always)] fn signed_cumsum>(x: &[T], start: i64, out: &mut Vec) -> bool { - appended(out, x.len(), |slots| { - let (mut total, mut overflow) = (start, 0i64); - for (o, &v) in slots.iter_mut().zip(x) { - let v: i64 = v.into(); - let sum = total.wrapping_add(v); - overflow |= (total ^ sum) & (v ^ sum); - total = sum; - o.write(sum); - } - overflow >= 0 - }) + unsafe { + appended(out, x.len(), |slots| { + let (mut total, mut overflow) = (start, 0i64); + for (o, &v) in slots.iter_mut().zip(x) { + let v: i64 = v.into(); + let sum = total.wrapping_add(v); + overflow |= (total ^ sum) & (v ^ sum); + total = sum; + o.write(sum); + } + overflow >= 0 + }) + } } /// Appends the running totals of `x` from `start` by `op`, which gives each total wrapped and whether it fits, and gives whether every total fits. Each step waits only for the wrapped total before it. #[inline(always)] fn checked_scan, S: Copy>(x: &[T], start: S, out: &mut Vec, op: impl Fn(S, S) -> (S, bool)) -> bool { - appended(out, x.len(), |slots| { - let (mut total, mut ok) = (start, true); - for (o, &v) in slots.iter_mut().zip(x) { - let (next, fits) = op(total, v.into()); - (total, ok) = (next, ok & fits); - o.write(next); - } - ok - }) + unsafe { + appended(out, x.len(), |slots| { + let (mut total, mut ok) = (start, true); + for (o, &v) in slots.iter_mut().zip(x) { + let (next, fits) = op(total, v.into()); + (total, ok) = (next, ok & fits); + o.write(next); + } + ok + }) + } } /// [`Sum`] and [`Product`] for each integer type of compact buffers, at `i64`. `$sum` sums a slice. Each step and product wraps through [`Overflowing`], which also tests whether it fits. @@ -1225,51 +1267,55 @@ macro_rules! with_reduce { }; } +// Shared with rustymath-kernels, whose crate prelude imports the width-dispatch macros. +#[doc(hidden)] +#[macro_export] macro_rules! native_reduction { (@inverse $kind:expr, $dtype:expr, |$R:ident, $T:ident| $body:expr) => {{ - use crate::buffer::{DType, FloatWidth}; + use $crate::buffer::{DType, FloatWidth}; + use $crate::reduce::{Alternated, Product, Reduce, ScaledProduct, Sum}; match ($dtype, $kind) { - (DType::Integer(w), Reduce::Sum) => with_width!(w, I => native_reduction!(@at Alternated, I, |$R, $T| $body)), - (DType::Float(FloatWidth::F16 | FloatWidth::F32), Reduce::Sum) => native_reduction!(@at Alternated, f32, |$R, $T| $body), - (DType::Float(FloatWidth::F64), Reduce::Sum) => native_reduction!(@at Alternated, f64, |$R, $T| $body), - (DType::Float(FloatWidth::F16 | FloatWidth::F32), Reduce::Product) => native_reduction!(@at Alternated, f32, |$R, $T| $body), - (DType::Float(FloatWidth::F64), Reduce::Product) => native_reduction!(@at Alternated, f64, |$R, $T| $body), - (DType::Complex(w), Reduce::Sum) => with_complex_width!(w, C => native_reduction!(@at Alternated, C, |$R, $T| $body)), - (DType::Complex(w), Reduce::Product) => with_complex_width!(w, C => native_reduction!(@at Alternated, C, |$R, $T| $body)), + (DType::Integer(w), Reduce::Sum) => with_width!(w, I => $crate::native_reduction!(@at Alternated, I, |$R, $T| $body)), + (DType::Float(FloatWidth::F16 | FloatWidth::F32), Reduce::Sum) => $crate::native_reduction!(@at Alternated, f32, |$R, $T| $body), + (DType::Float(FloatWidth::F64), Reduce::Sum) => $crate::native_reduction!(@at Alternated, f64, |$R, $T| $body), + (DType::Float(FloatWidth::F16 | FloatWidth::F32), Reduce::Product) => $crate::native_reduction!(@at Alternated, f32, |$R, $T| $body), + (DType::Float(FloatWidth::F64), Reduce::Product) => $crate::native_reduction!(@at Alternated, f64, |$R, $T| $body), + (DType::Complex(w), Reduce::Sum) => with_complex_width!(w, C => $crate::native_reduction!(@at Alternated, C, |$R, $T| $body)), + (DType::Complex(w), Reduce::Product) => with_complex_width!(w, C => $crate::native_reduction!(@at Alternated, C, |$R, $T| $body)), _ => None, } }}; (@at $r:ty, $t:ty, |$R:ident, $T:ident| $body:expr) => {{ type $R = $r; type $T = $t; $body }}; ($kind:expr, $dtype:expr, $checked:expr, |$R:ident, $T:ident| $body:expr) => {{ - use crate::buffer::DType; - use crate::reduce::Reduce; + use $crate::buffer::DType; + use $crate::reduce::Reduce; match ($dtype, $kind) { - (DType::Boolean, Reduce::Sum) => native_reduction!(@at crate::reduce::SumAs, bool, |$R, $T| $body), - (DType::Boolean, Reduce::Product | Reduce::And | Reduce::Min) => native_reduction!(@at crate::reduce::And, bool, |$R, $T| $body), - (DType::Boolean, Reduce::Or | Reduce::Max) => native_reduction!(@at crate::reduce::Or, bool, |$R, $T| $body), - (DType::Boolean, Reduce::Xor) => native_reduction!(@at crate::reduce::Xor, bool, |$R, $T| $body), - (DType::Boolean, Reduce::Xnor) => native_reduction!(@at crate::reduce::Xnor, bool, |$R, $T| $body), - (DType::Integer(w), Reduce::Sum) if !$checked => with_width!(w, I => native_reduction!(@at crate::Wrapped, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Sum) => with_width!(w, I => native_reduction!(@at crate::reduce::Sum, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Product) if !$checked => with_width!(w, I => native_reduction!(@at crate::Wrapped, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Product) => with_width!(w, I => native_reduction!(@at crate::reduce::Product, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Max) => with_width!(w, I => native_reduction!(@at crate::reduce::Max, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Min) => with_width!(w, I => native_reduction!(@at crate::reduce::Min, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::And) => with_width!(w, I => native_reduction!(@at crate::reduce::And, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Or) => with_width!(w, I => native_reduction!(@at crate::reduce::Or, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Xor) => with_width!(w, I => native_reduction!(@at crate::reduce::Xor, I, |$R, $T| $body)), - (DType::Integer(w), Reduce::Xnor) => with_width!(w, I => native_reduction!(@at crate::reduce::Xnor, I, |$R, $T| $body)), - (DType::Float(w), Reduce::Sum) => with_float_width!(w, F => native_reduction!(@at crate::reduce::Sum, F, |$R, $T| $body)), - (DType::Float(w), Reduce::Product) => with_float_width!(w, F => native_reduction!(@at crate::reduce::Product, F, |$R, $T| $body)), - (DType::Float(w), Reduce::Max) => with_float_width!(w, F => native_reduction!(@at crate::reduce::Max, F, |$R, $T| $body)), - (DType::Float(w), Reduce::Min) => with_float_width!(w, F => native_reduction!(@at crate::reduce::Min, F, |$R, $T| $body)), - (DType::Complex(w), Reduce::Sum) => with_complex_width!(w, C => native_reduction!(@at crate::reduce::Sum, C, |$R, $T| $body)), - (DType::Complex(w), Reduce::Product) => with_complex_width!(w, C => native_reduction!(@at crate::reduce::Product, C, |$R, $T| $body)), + (DType::Boolean, Reduce::Sum) => $crate::native_reduction!(@at $crate::reduce::SumAs, bool, |$R, $T| $body), + (DType::Boolean, Reduce::Product | Reduce::And | Reduce::Min) => $crate::native_reduction!(@at $crate::reduce::And, bool, |$R, $T| $body), + (DType::Boolean, Reduce::Or | Reduce::Max) => $crate::native_reduction!(@at $crate::reduce::Or, bool, |$R, $T| $body), + (DType::Boolean, Reduce::Xor) => $crate::native_reduction!(@at $crate::reduce::Xor, bool, |$R, $T| $body), + (DType::Boolean, Reduce::Xnor) => $crate::native_reduction!(@at $crate::reduce::Xnor, bool, |$R, $T| $body), + (DType::Integer(w), Reduce::Sum) if !$checked => with_width!(w, I => $crate::native_reduction!(@at $crate::Wrapped<$crate::reduce::Sum>, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Sum) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Sum, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Product) if !$checked => with_width!(w, I => $crate::native_reduction!(@at $crate::Wrapped<$crate::reduce::Product>, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Product) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Product, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Max) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Max, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Min) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Min, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::And) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::And, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Or) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Or, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Xor) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Xor, I, |$R, $T| $body)), + (DType::Integer(w), Reduce::Xnor) => with_width!(w, I => $crate::native_reduction!(@at $crate::reduce::Xnor, I, |$R, $T| $body)), + (DType::Float(w), Reduce::Sum) => with_float_width!(w, F => $crate::native_reduction!(@at $crate::reduce::Sum, F, |$R, $T| $body)), + (DType::Float(w), Reduce::Product) => with_float_width!(w, F => $crate::native_reduction!(@at $crate::reduce::Product, F, |$R, $T| $body)), + (DType::Float(w), Reduce::Max) => with_float_width!(w, F => $crate::native_reduction!(@at $crate::reduce::Max, F, |$R, $T| $body)), + (DType::Float(w), Reduce::Min) => with_float_width!(w, F => $crate::native_reduction!(@at $crate::reduce::Min, F, |$R, $T| $body)), + (DType::Complex(w), Reduce::Sum) => with_complex_width!(w, C => $crate::native_reduction!(@at $crate::reduce::Sum, C, |$R, $T| $body)), + (DType::Complex(w), Reduce::Product) => with_complex_width!(w, C => $crate::native_reduction!(@at $crate::reduce::Product, C, |$R, $T| $body)), _ => None, } }}; } -pub(crate) use native_reduction; +pub use crate::native_reduction; macro_rules! wrapped_products { ($($t:ty),+) => {$( diff --git a/src/reduce/along.rs b/core/src/reduce/along.rs similarity index 100% rename from src/reduce/along.rs rename to core/src/reduce/along.rs diff --git a/src/reduce/pieces.rs b/core/src/reduce/pieces.rs similarity index 100% rename from src/reduce/pieces.rs rename to core/src/reduce/pieces.rs diff --git a/src/reduce/run.rs b/core/src/reduce/run.rs similarity index 84% rename from src/reduce/run.rs rename to core/src/reduce/run.rs index 9e90a72..89d45f8 100644 --- a/src/reduce/run.rs +++ b/core/src/reduce/run.rs @@ -74,12 +74,12 @@ impl Reduce { )))).unwrap_or_else(|| Err(Error::new(Kind::Domain, "reduction does not support this dtype"))) } #[inline(never)] - pub(crate) fn inverse_pieces(self, variant: crate::domain::Variant, dims: [usize; 3], pieces: &Pieces<'_>) -> Result, Error> { + pub fn inverse_pieces(self, variant: crate::domain::Variant, dims: [usize; 3], pieces: &Pieces<'_>) -> Result, Error> { inverse_result( native_reduction!(@inverse self, variant.dtype, |R, T| Some(native_pieces::(dims, variant.checked, pieces))), dims.into_iter().product()) } #[inline(never)] - pub(crate) fn inverse_read(self, variant: crate::domain::Variant, shape: &[usize], axis: usize, input: Read<'_>) -> Result, Error> { + pub fn inverse_read(self, variant: crate::domain::Variant, shape: &[usize], axis: usize, input: Read<'_>) -> Result, Error> { inverse_result( native_reduction!(@inverse self, variant.dtype, |R, T| Some(native_read::(shape, Some(axis), variant.checked, input).map(|(buffer, fits, order)| (buffer.ordered(order), fits)))), crate::layout::count(shape)) @@ -130,11 +130,11 @@ fn native_read(shape: &[usize], axis: Option, checked: bool, input: }); } } - let pieces = |range: Range, emit: &mut dyn FnMut(usize, InputPiece<'_>)| { + let pieces = |range: Range, emit: &mut dyn FnMut(usize, Piece<'_>)| { let mut scratch = Buffer::with_capacity(T::DTYPE, 0); let start = range.start; let view = input.part_view(range, T::DTYPE, checked, &mut scratch)?; - emit(start, InputPiece::Items(view)); + emit(start, Piece::Items(view)); Ok(()) }; native_pieces::(dims, checked, &pieces).map(|(buffer, fits)| (buffer, fits, None)) @@ -151,14 +151,14 @@ fn native_pieces(dims: [usize; 3], checked: bool, pieces: &Pieces<'_>) -> crate::error::checked(|| { let fits = fold_pieces::(&mut totals, dims, &|range, emit| { let result = pieces(range, &mut |start, piece| match piece { - InputPiece::Items(view) => match T::of(view) { + Piece::Items(view) => match T::of(view) { Some(items) => emit(start, Part::Items(items)), None => match Buffer::cast_results(view, T::DTYPE, checked) { Ok(values) => emit(start, Part::Items(T::of(values.view()).expect("converted fold piece has its requested dtype"))), Err(error) => crate::error::record(error), }, }, - InputPiece::Total(total) => match R::Total::from_partial(total) { + Piece::Total(total) => match R::Total::from_partial(total) { Ok(value) => emit(start, Part::Total(total.items, value)), Err(error) => crate::error::record(error) }, }); @@ -196,8 +196,8 @@ fn totals>((totals, fits): (Vec, bool), checked: bo impl Reduce { pub fn run_scan( - self, variant: crate::domain::Variant, dims: [usize; 3], seed: crate::operation::Seed<'_>, direction: crate::operation::Direction, - input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer) -> Result<(), Error> { + self, variant: crate::domain::Variant, dims: [usize; 3], seed: Seed<'_>, direction: Direction, input: crate::agreement::Read<'_>, + output: &mut crate::buffer::Buffer) -> Result<(), Error> { if self == Reduce::Sum && let Some(done) = bounded_scan(variant, dims, seed, direction, false, input, output) { return done; } native_reduction!(self, variant.dtype, variant.fails_on_overflow(), |R, T| Some(native_scan::( dims, seed, direction, variant.checked, input, output @@ -206,8 +206,8 @@ impl Reduce { } fn native_scan( - dims: [usize; 3], seed: crate::operation::Seed<'_>, direction: crate::operation::Direction, checked: bool, input: crate::agreement::Read<'_>, - output: &mut crate::buffer::Buffer) -> Result<(), Error> where T: crate::buffer::Cast, R: Reduction, R::Total: crate::buffer::Cast { + dims: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer +) -> Result<(), Error> where T: crate::buffer::Cast, R: Reduction, R::Total: crate::buffer::Cast { scan_read::(dims, seed, direction, checked, input, output, &|items, dims, seed, out| { let fits = scan_along::(items, dims, seed, out); if checked && !fits { Err(Error::new(Kind::Overflow, "integer scan overflowed")) } else { Ok(()) } @@ -215,14 +215,14 @@ fn native_scan( } /// A left scan by `-`, run as `+`'s scan (`scan_along`) of negated items (`negated_cells`). Floats and complex numbers, 16-bit floats in 32 bits as the ordered scan computes them, and integers and Booleans whose caller asks for totals in `i16` or `i32` (`bounded_scan`). `Ok(false)`, with `output` unchanged, for other dtypes. -pub(crate) fn negated_scan( - variant: crate::domain::Variant, dims: [usize; 3], seed: crate::operation::Seed<'_>, input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer +pub fn negated_scan( + variant: crate::domain::Variant, dims: [usize; 3], seed: Seed<'_>, input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer ) -> Result { use crate::buffer::{DType, FloatWidth}; macro_rules! negated { ($t:ty) => { - scan_read::<$t, $t>(dims, seed, crate::operation::Direction::Left, variant.checked, input, output, &|items, dims, seed, out| { - negated_cells::<$t, Sum>(items, dims, seed, out, |x| -x); + scan_read::<$t, $t>(dims, seed, Direction::Left, variant.checked, input, output, &|items, dims, seed, out| { + super::scans::negated_cells::<$t, Sum>(items, dims, seed, out, |x| -x); Ok(()) }) }; @@ -232,28 +232,17 @@ pub(crate) fn negated_scan( DType::Float(FloatWidth::F64) => negated!(f64)?, DType::Complex(w) => with_complex_width!(w, C => negated!(C))?, DType::Integer(_) | DType::Boolean => { - return bounded_scan(variant, dims, seed, crate::operation::Direction::Left, true, input, output).map_or(Ok(false), |done| done.map(|()| true)); + return bounded_scan(variant, dims, seed, Direction::Left, true, input, output).map_or(Ok(false), |done| done.map(|()| true)); } _ => return Ok(false), } Ok(true) } -/// Appends the left scan by `-` of each cell of `items`, `[_, n, inner]` with `n` rows, to `out`, as `R`'s scan of the items that `neg` negates: each lane's first row, or `seed` less it, and then each later row subtracted from the total before it. -fn negated_cells>(items: &[T], [_, n, inner]: [usize; 3], seed: Option<&[T]>, out: &mut Vec, neg: impl Fn(T) -> T) { - let mut negated = Vec::with_capacity(n * inner); - for cell in items.chunks_exact(n * inner) { - let (start, rest) = match seed { Some(s) => (s, cell), None => { out.extend_from_slice(&cell[..inner]); (&cell[..inner], &cell[inner..]) } }; - negated.clear(); - negated.extend(rest.iter().map(|&x| neg(x))); - scan_along::(&negated, [1, rest.len() / inner, inner], Some(start), out); - } -} - /// A sum scan of integers or Booleans, or with `negate` a left scan by `-` of them, whose caller asks for totals in `output`'s dtype, `i16` or `i32`, wide enough for every stored item that the scan reads, whatever width the kernel computes at. The caller asks only where its bound on every running total fits that type. Each lane totals in it through `SumAs`, wrapping. `None` for any other request. -pub(crate) fn bounded_scan( - variant: crate::domain::Variant, dims: [usize; 3], seed: crate::operation::Seed<'_>, direction: crate::operation::Direction, negate: bool, - input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer) -> Option> { +pub fn bounded_scan( + variant: crate::domain::Variant, dims: [usize; 3], seed: Seed<'_>, direction: Direction, negate: bool, input: crate::agreement::Read<'_>, + output: &mut crate::buffer::Buffer) -> Option> { use crate::buffer::{DType, Width}; let DType::Integer(total @ (Width::I16 | Width::I32)) = output.dtype() else { return None }; let fits = |d: DType| d == DType::Boolean || matches!(d, DType::Integer(w) if total.join(w) == total); @@ -266,12 +255,12 @@ pub(crate) fn bounded_scan( /// `bounded_scan`'s scan in `C`, reading the items as `C`. Where `checked`, each total is then tested from the total before it and its item: a sum overflows where its sign differs from both its arguments', and a difference where its arguments' signs differ and its sign differs from the first's. Every item fits `C`, which makes the tests exact. An overflow fails the scan. fn totalled( - dims: [usize; 3], seed: crate::operation::Seed<'_>, direction: crate::operation::Direction, checked: bool, negate: bool, input: crate::agreement::Read<'_>, - output: &mut crate::buffer::Buffer) -> Result<(), Error> where C: crate::buffer::Cast + Tally + Term + Into + Ord + Default - + std::ops::BitXor + std::ops::BitAnd + std::ops::BitOr { + dims: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, negate: bool, input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer +) -> Result<(), Error> where C: crate::buffer::Cast + Tally + Term + Into + Ord + Default + std::ops::BitXor + std::ops::BitAnd + + std::ops::BitOr { scan_read::(dims, seed, direction, checked, input, output, &|items, [g, n, inner], seed, out| { let from = out.len(); - if negate { negated_cells::>(items, [g, n, inner], seed, out, |x| C::low(-x.into())) } + if negate { super::scans::negated_cells::>(items, [g, n, inner], seed, out, |x| C::low(-x.into())) } else { scan_along::>(items, [g, n, inner], seed, out); } if !checked { return Ok(()); } let test = |p: C, x: C, c: C| if negate { (p ^ x) & (p ^ c) } else { (p ^ c) & (x ^ c) }; @@ -286,17 +275,30 @@ fn totalled( } /// Cell `cell`'s `inner` seed items from `read`: its one item in every column where it holds one, else the cell's own row. -pub(crate) fn seed_row( +pub fn seed_row( read: &crate::agreement::Read<'_>, cell: usize, inner: usize, checked: bool, scratch: &mut Vec) -> Result, Error> { let one = read.plan.len() == 1; let arg = read.part::(if one { 0..1 } else { cell * inner..(cell + 1) * inner }, checked, scratch)?; Ok((0..inner).map(|i| arg.nth(if one { 0 } else { i })).collect()) } -pub(crate) fn scan_read( - dims: [usize; 3], seed: crate::operation::Seed<'_>, direction: crate::operation::Direction, checked: bool, input: crate::agreement::Read<'_>, - output: &mut crate::buffer::Buffer, run: &dyn Fn(&[T], [usize; 3], Option<&[S]>, &mut Vec) -> Result<(), Error>) -> Result<(), Error> { - use crate::{agreement::{Mapping, Missing, Read}, operation::Direction}; +pub fn scan_read( + dims: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, input: Read<'_>, output: &mut Buffer, + run: &dyn Fn(&[T], [usize; 3], Option<&[S]>, &mut Vec) -> Result<(), Error>) -> Result<(), Error> { + scan_with_carry::(dims, seed, direction, checked, input, output, run) +} + +/// A Boolean-result scan with its raw seed and chunk carries read at the input's numeric type. +pub fn boolean_scan_read( + dims: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, input: Read<'_>, output: &mut Buffer, + run: &dyn Fn(&[T], [usize; 3], Option<&[T]>, &mut Vec) -> Result<(), Error>) -> Result<(), Error> where bool: Source { + scan_with_carry::(dims, seed, direction, checked, input, output, run) +} + +fn scan_with_carry>( + dims: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, input: Read<'_>, output: &mut Buffer, + run: &dyn Fn(&[T], [usize; 3], Option<&[C]>, &mut Vec) -> Result<(), Error>) -> Result<(), Error> { + use crate::agreement::{Mapping, Missing, Read}; let [outer, len, inner] = dims; let reversed = matches!(direction, Direction::Right).then(|| { input.plan.remapped(Mapping::from(vec![ @@ -326,7 +328,8 @@ pub(crate) fn scan_read( } let carry = carry.get_or_insert_with(Vec::new); carry.clear(); - carry.extend_from_slice(&values[values.len() - inner..]); + let tail = &values[values.len() - inner..]; + if let Some(items) = S::borrowed(tail) { carry.extend_from_slice(items); } else { carry.extend(tail.iter().copied().map(Source::::read)); } } } if reversed.is_some() { reverse_rows(&mut values, dims); } @@ -335,13 +338,16 @@ pub(crate) fn scan_read( } /// Each item of `input`, an array of shape `dims`, combined with the item one row before it along the middle axis, and each lane's first row with `seed`'s item in its column, or kept as it is without a seed, as an inverse scan runs it. `block` combines the items of two equal slices, the earlier items first unless `flip`, and gives the kind of any combination that failed: a formula's loop over a block. A failure is an error where the read is checked. -pub(crate) fn adjacent_read( +/// +/// # Safety +/// `block` must initialize every output slot before returning, including when it returns `Err`. +pub unsafe fn adjacent_read( [outer, len, inner]: [usize; 3], seed: Option>, checked: bool, flip: bool, input: crate::agreement::Read<'_>, output: &mut crate::buffer::Buffer, block: &dyn Fn(&[T], &[T], &mut [std::mem::MaybeUninit]) -> Result<(), Kind>) -> Result<(), Error> { let (mut values, mut scratch, mut seed_scratch) = (Vec::with_capacity(outer * len * inner), Vec::new(), Vec::new()); let mut failure = None; let mut pair = |earlier: &[T], items: &[T], values: &mut Vec| { - failure = failure.or(appended(values, items.len(), |out| if flip { block(items, earlier, out) } else { block(earlier, items, out) }).err()); + failure = failure.or(unsafe { appended(values, items.len(), |out| if flip { block(items, earlier, out) } else { block(earlier, items, out) }) }.err()); }; for cell in (0..outer).filter(|_| len > 0) { let start = seed.map(|read| seed_row(&read, cell, inner, checked, &mut seed_scratch)).transpose()?; @@ -356,7 +362,7 @@ pub(crate) fn adjacent_read( Ok(()) } -pub(crate) fn reverse_rows(values: &mut [T], [outer, len, inner]: [usize; 3]) { +pub fn reverse_rows(values: &mut [T], [outer, len, inner]: [usize; 3]) { if len < 2 || inner == 0 { return; } for cell in values[..outer * len * inner].chunks_exact_mut(len * inner) { for row in 0..len / 2 { for column in 0..inner { cell.swap(row * inner + column, (len - row - 1) * inner + column); } } diff --git a/src/reduce/scans.rs b/core/src/reduce/scans.rs similarity index 64% rename from src/reduce/scans.rs rename to core/src/reduce/scans.rs index 539dcf1..a5c84aa 100644 --- a/src/reduce/scans.rs +++ b/core/src/reduce/scans.rs @@ -8,29 +8,33 @@ pub fn scan_along>(data: &[T], dims: [usize; 3], seed: } /// Appends the ordered left scan of each lane of `[outer, len, inner]` in `data` to `out`, in the same layout: along the last axis by `lane`, one call for every lane, stepping from `seed`'s item; along any other axis by `block`, one call for each row, stepping the row before it, or `seed`'s row, by the row's items. Without a seed, a lane's first item or row is kept as it is. Gives the kind of the first step that failed. -pub(crate) fn ordered_scan( +/// +/// # Safety +/// `block` must initialize every output slot before returning, including when it returns `Err`. +pub unsafe fn ordered_scan( data: &[T], [_, len, inner]: [usize; 3], seed: Option<&[T]>, out: &mut Vec, lane: &dyn Fn(Option, &[T], usize, &mut Vec) -> Option, block: &dyn Fn(&[T], &[T], &mut [std::mem::MaybeUninit]) -> Result<(), Kind>) -> Option { if inner == 1 { return lane(seed.map(|s| s[0]), data, len, out); } - appended(out, data.len(), |slots| { - let mut failure = None; - for (items, outs) in data.chunks(len * inner).zip(slots.chunks_mut(len * inner)) { - match seed { - Some(s) => failure = failure.or(block(s, &items[..inner], &mut outs[..inner]).err()), - None => { for (slot, &x) in outs.iter_mut().zip(&items[..inner]) { slot.write(x); } } - } - for r in 1..len { - let (done, next) = outs.split_at_mut(r * inner); - failure = failure.or(block(written(&mut done[(r - 1) * inner..]), &items[r * inner..(r + 1) * inner], &mut next[..inner]).err()); + unsafe { + appended(out, data.len(), |slots| { + let mut failure = None; + for (items, outs) in data.chunks(len * inner).zip(slots.chunks_mut(len * inner)) { + match seed { + Some(s) => failure = failure.or(block(s, &items[..inner], &mut outs[..inner]).err()), + None => { for (slot, &x) in outs.iter_mut().zip(&items[..inner]) { slot.write(x); } } + } + for r in 1..len { + let (done, next) = outs.split_at_mut(r * inner); + failure = failure.or(block(written(&mut done[(r - 1) * inner..]), &items[r * inner..(r + 1) * inner], &mut next[..inner]).err()); + } } - } - failure - }) + failure + }) + } } /// Appends the running results along each lane of `len` consecutive items in `data` to `out`, and gives whether every one fits: `first` gives each lane's first result, and `step` each later one from the one before. Lanes shorter than [`SHORT_LANE`] run [`LANES`] side by side (`rows_scan`). -pub(crate) fn lanes_scan( - data: &[T], len: usize, out: &mut Vec, first: impl Fn(T) -> (T, bool) + Copy, step: impl Fn(T, T) -> (T, bool) + Copy) -> bool { +pub fn lanes_scan(data: &[T], len: usize, out: &mut Vec, first: impl Fn(T) -> (T, bool) + Copy, step: impl Fn(T, T) -> (T, bool) + Copy) -> bool { rows_scan(data, [0, len, 1], out, |lane, out| { let (total, ok) = first(lane[0]); out.push(total); @@ -46,43 +50,47 @@ fn rows_scan( if data.is_empty() { return true; } if inner == 1 && len >= SHORT_LANE { return data.chunks(len).fold(true, |fits, l| lane(l, out) & fits); } if inner == 1 { - return appended(out, data.len(), |slots| { - let whole = data.len() / (LANES * len) * (LANES * len); - let (groups, rest) = slots.split_at_mut(whole); - // SAFETY: `side_by_side` gives each position of the data it runs on, which holds as many items as the slots written here. - side_by_side::(&data[..whole], len, |x| first(0, x), step, |at, s| { - unsafe { groups.get_unchecked_mut(at) }.write(s); - }, |_, _| {}) & side_by_side::<1, _, _>(&data[whole..], len, |x| first(0, x), step, |at, s| { - unsafe { rest.get_unchecked_mut(at) }.write(s); - }, |_, _| {}) - }); + return unsafe { + appended(out, data.len(), |slots| { + let whole = data.len() / (LANES * len) * (LANES * len); + let (groups, rest) = slots.split_at_mut(whole); + // SAFETY: `side_by_side` gives each position of the data it runs on, which holds as many items as the slots written here. + side_by_side::(&data[..whole], len, |x| first(0, x), step, |at, s| { + groups.get_unchecked_mut(at).write(s); + }, |_, _| {}) & side_by_side::<1, _, _>(&data[whole..], len, |x| first(0, x), step, |at, s| { + rest.get_unchecked_mut(at).write(s); + }, |_, _| {}) + }) + }; } // Each row steps from the row before it, which `out` already holds. - appended(out, data.len(), |slots| { - let mut fits = true; - let mut at = 0; - while at < data.len() { - let (row, outs) = (&data[at..at + inner], &mut slots[at..at + inner]); - for k in 0..inner { - let (t, f) = first(k, row[k]); - outs[k].write(t); - fits &= f; - } - let mut r = at + inner; - while r < at + len * inner { - let (done, rest) = slots.split_at_mut(r); - let (before, row, outs) = (written(&mut done[r - inner..]), &data[r..r + inner], &mut rest[..inner]); + unsafe { + appended(out, data.len(), |slots| { + let mut fits = true; + let mut at = 0; + while at < data.len() { + let (row, outs) = (&data[at..at + inner], &mut slots[at..at + inner]); for k in 0..inner { - let (t, f) = step(before[k], row[k]); + let (t, f) = first(k, row[k]); outs[k].write(t); fits &= f; } - r += inner; + let mut r = at + inner; + while r < at + len * inner { + let (done, rest) = slots.split_at_mut(r); + let (before, row, outs) = (written(&mut done[r - inner..]), &data[r..r + inner], &mut rest[..inner]); + for k in 0..inner { + let (t, f) = step(before[k], row[k]); + outs[k].write(t); + fits &= f; + } + r += inner; + } + at += len * inner; } - at += len * inner; - } - fits - }) + fits + }) + } } /// A function of two Booleans, held as its truth table: `Logical(t)` gives `t[r][x]` for the running result `r` and the next item `x`. Its scans run any such function. @@ -157,14 +165,16 @@ fn pack(block: &[bool; 64]) -> u64 { bits } -/// Writes the 64 bits of `r` to `out` as Booleans, the lowest bit first. Each byte of `r` becomes 8 bytes of 0 or 1, bit `j` in byte `j`, through one multiplication, two masks, an add and a shift that's the same for every byte. LLVM compiles a test of bit `j` in byte `j` to a different shift for each byte, which wasm's SIMD can only emulate. +/// Expands the 64 bits of `r` into Boolean bytes, the lowest bit first, then converts that block into `out`. Each byte of `r` becomes 8 bytes of 0 or 1, bit `j` in byte `j`, through one multiplication, two masks, an add and a shift that's the same for every byte. LLVM compiles a test of bit `j` in byte `j` to a different shift for each byte, which wasm's SIMD can only emulate. #[inline(always)] -fn unpack(r: u64, out: &mut [std::mem::MaybeUninit; 64]) { - for (k, lanes) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { +fn unpack(r: u64, out: &mut [std::mem::MaybeUninit; 64]) where bool: Source { + let mut block = [false; 64]; + for (k, lanes) in block.as_chunks_mut::<8>().0.iter_mut().enumerate() { let bits = (r >> (8 * k) & 0xff).wrapping_mul(0x0101_0101_0101_0101) & 0x8040_2010_0804_0201; let bytes = ((bits + 0x7f7f_7f7f_7f7f_7f7f) >> 7 & 0x0101_0101_0101_0101).to_le_bytes(); - *lanes = std::array::from_fn(|j| std::mem::MaybeUninit::new(bytes[j] != 0)); + *lanes = std::array::from_fn(|j| bytes[j] != 0); } + crate::slice::pairs_into(Arg::Many(&block), Arg::One(()), 64, out, |b, ()| (b.read(), true)); } /// The results of 64 items, bit by bit, from the running result `carry`, where item `k`'s map is `r ↦ (s ∧ r) ≠ t` for bit `k` of `s` and `t`. Each step composes each bit's map with the map it holds from the bits before it. A bit with none before it takes the identity map, whose `s` is 1 and whose `t` is 0. @@ -192,55 +202,68 @@ fn packed(x: &[bool], start: usize) -> u64 { /// Appends the running results of `n` items to `out`, from the running result `carry` before the first. `maps(start)` gives the maps of the 64 items from `start` as two words of bits `(s, t)`: item `start + k` maps `r` to `(s ∧ r) ≠ t` for bit `k` of each. Bits past the last item are ignored. #[inline(always)] -fn mapped(n: usize, mut carry: bool, maps: impl Fn(usize) -> (u64, u64), out: &mut Vec) { - appended(out, n, |slots| { - let (outs, last) = slots.as_chunks_mut::<64>(); - for (i, o) in outs.iter_mut().enumerate() { - let (s, t) = maps(64 * i); - let r = composed(s, t, carry); - unpack(r, o); - carry = r >> 63 != 0; - } - if !last.is_empty() { - let (s, t) = maps(n - last.len()); - let r = composed(s, t, carry); - for (k, slot) in last.iter_mut().enumerate() { slot.write(r >> k & 1 != 0); } - } - }); +fn mapped(n: usize, mut carry: bool, maps: impl Fn(usize) -> (u64, u64), out: &mut Vec) where bool: Source { + unsafe { + appended(out, n, |slots| { + let (outs, last) = slots.as_chunks_mut::<64>(); + for (i, o) in outs.iter_mut().enumerate() { + let (s, t) = maps(64 * i); + let r = composed(s, t, carry); + unpack(r, o); + carry = r >> 63 != 0; + } + if !last.is_empty() { + let (s, t) = maps(n - last.len()); + let r = composed(s, t, carry); + for (k, slot) in last.iter_mut().enumerate() { slot.write(Source::::read(r >> k & 1 != 0)); } + } + }) + }; } /// Appends a Boolean-valued scan of long consecutive lanes, keeping an unseeded lane's first item as it is. Comparisons at carries 0 and 1 make independent two-state maps, whose packed words compose through `mapped`. A failed speculative comparison leaves `out` unchanged and declines, so the ordered runner checks only steps the scan actually takes. -pub(crate) fn comparison_scan( - data: &[T], len: usize, start: Option, step: impl Fn(T, T) -> (T, Option) + Copy, out: &mut Vec) -> bool where bool: Source { +pub fn comparison_scan( + data: &[T], len: usize, start: Option, step: impl Fn(T, T) -> (T, Option) + Copy, read: impl Fn(T) -> S + Copy, + bulk: impl Fn(T, &[T], &mut Vec) -> Option + Copy, out: &mut Vec) -> bool where bool: Source + Source { if len < SHORT_LANE { return false; } let (zero, one, before) = (false.read(), true.read(), out.len()); - let (mut low, mut high, mut bits) = (Vec::new(), Vec::new(), Vec::new()); + let (mut low, mut high) = (Vec::new(), Vec::new()); for lane in data.chunks(len) { - let (first, used) = match start { Some(seed) => (step(seed, lane[0]), 1), None => { out.push(lane[0]); (step(lane[0], lane[1]), 2) } }; + let (first, used) = match start { Some(seed) => (step(seed, lane[0]), 1), None => { out.push(read(lane[0])); (step(lane[0], lane[1]), 2) } }; let (first, failed) = first; if failed.is_some() { out.truncate(before); return false; } - out.push(first); + out.push(read(first)); let rest = &lane[used..]; low.clear(); high.clear(); - bits.clear(); let transitions = |carry, dest: &mut Vec| { - appended(dest, rest.len(), |slots| { - let mut valid = true; - for (&x, slot) in rest.iter().zip(slots) { - let (value, failed) = step(carry, x); - slot.write(value != zero); - valid &= failed.is_none(); - } - valid - }) + if let Some(valid) = bulk(carry, rest, dest) { return valid; } + unsafe { + appended(dest, rest.len(), |slots| { + let mut valid = true; + for (&x, slot) in rest.iter().zip(slots) { + let (value, failed) = step(carry, x); + slot.write(value != zero); + valid &= failed.is_none(); + } + valid + }) + } }; if !(transitions(zero, &mut low) & transitions(one, &mut high)) { out.truncate(before); return false; } - mapped(rest.len(), first != zero, |at| { - let (l, h) = (packed(&low, at), packed(&high, at)); - (l ^ h, l) - }, &mut bits); - out.extend(bits.iter().copied().map(Source::::read)); + mapped(rest.len(), first != zero, |at| { let (l, h) = (packed(&low, at), packed(&high, at)); (l ^ h, l) }, out); } true } + +/// Appends each cell's left difference scan without materializing its negated items. The first item is unchanged without a seed; every later item is subtracted from the running total. +pub(super) fn negated_cells>( + items: &[T], dims: [usize; 3], seed: Option<&[T]>, out: &mut Vec, neg: impl Fn(T) -> T + Copy) { + let start = |i| seed.map(|s| s[if s.len() == 1 { 0 } else { i }]); + let lane = |lane: &[T], out: &mut Vec| { + let (total, rest) = match start(0) { Some(s) => (s, lane), None => { out.push(lane[0]); (lane[0], &lane[1..]) } }; + grouped(rest, Some(total), out, neg, R::step, |x| x, T::DTYPE.is_integral()); + true + }; + rows_scan(items, dims, out, lane, |i, x| (start(i).map_or(x, |s| R::step(s, neg(x))), true), |t, x| (R::step(t, neg(x)), true)); +} diff --git a/src/reduce/sliding.rs b/core/src/reduce/sliding.rs similarity index 100% rename from src/reduce/sliding.rs rename to core/src/reduce/sliding.rs diff --git a/src/search.rs b/core/src/search.rs similarity index 98% rename from src/search.rs rename to core/src/search.rs index 99fe715..f9a15ce 100644 --- a/src/search.rs +++ b/core/src/search.rs @@ -148,7 +148,7 @@ pub fn index_of(x: &[K], y: &[K]) -> Vec { /// Appends `look(n)` to `found` for each needle `n` of `y`, in order. The lookups split across threads in units of `BETWEEN_CHECKS` needles, as `parallel::split` splits a call, so `look` must read only what no lookup changes. pub(crate) fn look_up(found: &mut Vec, y: &[K], look: impl Fn(K) -> u32 + Sync) { - crate::parallel::extended(found, y.len(), crate::BETWEEN_CHECKS, &|r, slots| { + crate::parallel::extended(found, y.len(), crate::BETWEEN_CHECKS, &|r, slots| unsafe { slots.fill(r.len(), |slots| { let needles = &y[r]; let slots = &mut slots[..needles.len()]; @@ -706,11 +706,16 @@ fn two_groups(items: &[T], group: impl Fn(T) -> usize + Sync, ap let zeros = groups.len() - ones; let buffers = if apart { let (mut d0, mut d1) = (Vec::new(), Vec::new()); - crate::slice::appended(&mut d0, zeros, |a| crate::slice::appended(&mut d1, ones, |b| split2(&groups, a, b))); + unsafe { crate::slice::appended(&mut d0, zeros, |a| crate::slice::appended(&mut d1, ones, |b| split2(&groups, a, b))) }; vec![d0, d1] } else { let mut d = Vec::new(); - crate::slice::appended(&mut d, groups.len(), |d| { let (a, b) = d.split_at_mut(zeros); split2(&groups, a, b) }); + unsafe { + crate::slice::appended(&mut d, groups.len(), |d| { + let (a, b) = d.split_at_mut(zeros); + split2(&groups, a, b) + }) + }; vec![d] }; Groups { sizes: vec![zeros, ones], buffers } @@ -1524,21 +1529,23 @@ pub fn view_argextremes(x: View<'_>, [outer, len, inner]: [usize; 3], down: bool fn extremes(x: &[T], [outer, len, inner]: [usize; 3], down: bool, key: impl Fn(T) -> K + Sync) -> Vec { let mut out = Vec::with_capacity(outer * inner); crate::parallel::extended(&mut out, outer * inner, (crate::CHUNK / len.max(1)).max(1), &|range, slots| { - slots.fill(range.len(), |slots| { - for (lane, slot) in range.zip(slots) { - let start = lane / inner * len * inner + lane % inner; - let (mut best, mut at) = (key(x[start]), 0); - for j in 1..len { - let k = key(x[start + j * inner]); - if if down { k > best } else { k < best } { - best = k; - at = j; + unsafe { + slots.fill(range.len(), |slots| { + for (lane, slot) in range.zip(slots) { + let start = lane / inner * len * inner + lane % inner; + let (mut best, mut at) = (key(x[start]), 0); + for j in 1..len { + let k = key(x[start + j * inner]); + if if down { k > best } else { k < best } { + best = k; + at = j; + } } + slot.write(at as i64); } - slot.write(at as i64); - } - true - }); + true + }) + }; true }); out diff --git a/src/signal.rs b/core/src/signal.rs similarity index 100% rename from src/signal.rs rename to core/src/signal.rs diff --git a/src/slice.rs b/core/src/slice.rs similarity index 94% rename from src/slice.rs rename to core/src/slice.rs index b7c0e94..9c05c63 100644 --- a/src/slice.rs +++ b/core/src/slice.rs @@ -78,12 +78,13 @@ macro_rules! arities { #[inline] fn checked(&self, $($x: T),+) -> Option { let (r, ok) = self.judged($($x),+); ok.then_some(r) } fn many(&self, $($x: Arg<'_, T>,)+ n: usize, out: &mut Vec) -> bool where T: Copy + Sync { - appended(out, n, |slots| crate::parallel::split(slots, CHUNK, &|start, slots| { + unsafe { appended(out, n, |slots| crate::parallel::split(slots, CHUNK, &|start, slots| { crate::parallel::in_chunks(start, slots, |at, part| self.slice([$($x.part(at, part.len())),+], part.len(), part)) - })) + })) } } fn many_into(&self, $($x: Arg<'_, T>,)+ n: usize, out: &mut crate::parallel::Slots<'_, Self::Out>) -> bool where T: Copy { - out.fill(n, |slots| crate::parallel::in_chunks(0, slots, |at, part| self.slice([$($x.part(at, part.len())),+], part.len(), part))) + // SAFETY: the unsafe object trait promises that `slice` initializes every output. + unsafe { out.fill(n, |slots| crate::parallel::in_chunks(0, slots, |at, part| self.slice([$($x.part(at, part.len())),+], part.len(), part))) } } } )+}; @@ -204,7 +205,7 @@ fn items(x: &[Arg<'_, E>; N], i: usize) -> [E; N] { std /// Writes `f` for each of `n` results into `out`, for arguments `x`, and gives whether `f` said `true` for every result. One and two arguments run through [`pairs_into`], and three through `triples_into`, whose loops vectorize. #[inline(always)] -pub(crate) fn mapped(x: &[Arg<'_, E>; N], n: usize, out: &mut [MaybeUninit], f: impl Fn([E; N]) -> (O, bool)) -> bool { +pub fn mapped(x: &[Arg<'_, E>; N], n: usize, out: &mut [MaybeUninit], f: impl Fn([E; N]) -> (O, bool)) -> bool { if const { N == 1 } { pairs_into(x[0], Arg::One(()), n, out, #[inline(always)] |a, ()| f([a; N])) } else if const { N == 2 } { pairs_into(x[0], x[1], n, out, #[inline(always)] |a, b| f(std::array::from_fn(|k| if k == 0 { a } else { b }))) @@ -219,7 +220,9 @@ pub(crate) fn mapped(x: &[Arg<'_, E>; N], n: usize, #[inline(always)] pub(crate) fn each, const N: usize>( x: [Arg<'_, E>; N], n: usize, out: &mut [MaybeUninit], f: impl Fn([E; N]) -> R) -> bool { - mapped(&x, n, out, #[inline(always)] |a| f(a).split()) || (R::RESCUE && written(&mut out[..n]).iter().enumerate().all(|(i, &r)| kept(r, items(&x, i)))) + mapped( + &x, n, out, #[inline(always)] |a| f(a).split() + ) || (R::RESCUE && unsafe { written(&mut out[..n]) }.iter().enumerate().all(|(i, &r)| kept(r, items(&x, i)))) } /// A rare form for `each_usual` that no block takes, for a function with no rare form. @@ -430,11 +433,13 @@ macro_rules! each_triple { /// Appends `f` on each of `n` pairs of items to `out`, and gives whether `f` said `true` for every pair. An `Arg::One` pairs with every item of the other argument. The loops write straight into `out`'s spare capacity, which is never zeroed first. This is the loop that rustymath's slice forms run, and a caller's own kernels can run it too, as basedpl's arithmetic does. A long call splits across threads, as [`parallel`](crate::parallel) describes. #[inline(never)] pub fn pairs(x: Arg<'_, X>, y: Arg<'_, Y>, n: usize, out: &mut Vec, f: impl Fn(X, Y) -> (B, bool) + Sync) -> bool { - appended(out, n, |slots| { - crate::parallel::split(slots, CHUNK, &|start, slots| { - pairs_into(x.part(start, slots.len()), y.part(start, slots.len()), slots.len(), slots, |a, b| f(a, b)) + unsafe { + appended(out, n, |slots| { + crate::parallel::split(slots, CHUNK, &|start, slots| { + pairs_into(x.part(start, slots.len()), y.part(start, slots.len()), slots.len(), slots, |a, b| f(a, b)) + }) }) - }) + } } /// [`pairs`], writing into the `n` slots of `out` in place of appending to a vector. It runs [`triples_into`]'s loops with a unit first argument, which go straight to [`each_pair!`]'s. @@ -458,7 +463,7 @@ pub(crate) fn triples_into( /// Whether `valid` holds for every result in `out` with the items of `x` that gave it, paired as [`mapped`] pairs them. These loops only judge: a loop that also computes each result stops vectorizing for some bodies, such as `Ln`'s. Failures gather as bits, with no branch. #[inline(always)] -pub(crate) fn judged(x: &[Arg<'_, E>; N], out: &[O], valid: impl Fn([E; N], O) -> bool) -> bool { +pub fn judged(x: &[Arg<'_, E>; N], out: &[O], valid: impl Fn([E; N], O) -> bool) -> bool { let (n, mut bad) = (out.len(), 0u8); if const { N == 1 } { each_pair!(x[0], Arg::Many(out), n, |_i, a, r| bad |= u8::from(!valid([a; N], r))); } else if const { N == 2 } { @@ -468,8 +473,11 @@ pub(crate) fn judged(x: &[Arg<'_, E>; N], out: } /// Appends `n` items to `out`, which `write` writes into its spare capacity, never zeroed first, and gives `write`'s result. `write` must write every slot it gets. Each caller's loop does, by its construction. +/// +/// # Safety +/// `write` must initialize every slot it receives before returning. #[inline(always)] -pub(crate) fn appended(out: &mut Vec, n: usize, write: impl FnOnce(&mut [MaybeUninit]) -> T) -> T { +pub unsafe fn appended(out: &mut Vec, n: usize, write: impl FnOnce(&mut [MaybeUninit]) -> T) -> T { out.reserve(n); let result = write(&mut out.spare_capacity_mut()[..n]); // SAFETY: `write` wrote all `n` slots. @@ -478,8 +486,11 @@ pub(crate) fn appended(out: &mut Vec, n: usize, write: impl FnOnce(&mut } /// `out`'s slots as items. Each caller passes only slots that its loop has just written. +/// +/// # Safety +/// Every slot in `out` must hold an initialized `B`. #[inline(always)] -pub(crate) fn written(out: &mut [MaybeUninit]) -> &mut [B] { +pub unsafe fn written(out: &mut [MaybeUninit]) -> &mut [B] { // SAFETY: the caller has written every slot. unsafe { &mut *(std::ptr::from_mut(out) as *mut [B]) } } @@ -496,7 +507,7 @@ pub(crate) fn two_pass( }); if first { return true; } let mut ok = true; - for (i, r) in written(out).iter_mut().enumerate().filter(|(_, r)| r.is_nan()) { + for (i, r) in unsafe { written(out) }.iter_mut().enumerate().filter(|(_, r)| r.is_nan()) { let a = items(&x, i); *r = >::read(f(a)); ok &= kept(*r, a); @@ -510,13 +521,16 @@ pub(crate) fn working( narrow: impl Fn(O) -> P) -> bool { let mut bufs = [(); N].map(|()| Vec::new()); let mut next = bufs.each_mut().into_iter(); - buffered(n, out, |o| f(x.map(|a| a.converted(0, n, next.next().unwrap(), F::working)), n, o), narrow) + unsafe { buffered(n, out, |o| f(x.map(|a| a.converted(0, n, next.next().unwrap(), F::working)), n, o), narrow) } } /// `f`, a slice form writing `n` outputs of type `O`, run into a buffer, with `each` turning each output into the `P` that `out` takes. It gives `f`'s result. -pub(crate) fn buffered(n: usize, out: &mut [MaybeUninit

], f: impl FnOnce(&mut [MaybeUninit]) -> bool, each: impl Fn(O) -> P) -> bool { +/// +/// # Safety +/// `f` must initialize every output slot it receives before returning. +pub unsafe fn buffered(n: usize, out: &mut [MaybeUninit

], f: impl FnOnce(&mut [MaybeUninit]) -> bool, each: impl Fn(O) -> P) -> bool { let mut rs = Vec::new(); - let ok = appended(&mut rs, n, f); + let ok = unsafe { appended(&mut rs, n, f) }; pairs_into(Arg::Many(&rs), Arg::One(()), n, &mut out[..n], |r, ()| (each(r), true)); ok } diff --git a/src/special.rs b/core/src/special.rs similarity index 99% rename from src/special.rs rename to core/src/special.rs index e1a1ae2..2d1f14b 100644 --- a/src/special.rs +++ b/core/src/special.rs @@ -651,7 +651,7 @@ fn erfc_blocks(x: &[R], out: &mut [MaybeUninit] }, near::, far::, #[inline(always)] |xs, os| { if branches::() { let n = placed(0..xs.len() as u16, &mut place, |i| xs[usize::from(i)].abs() < split); - let (nears, fars) = written(&mut place[..xs.len()]).split_at(n); + let (nears, fars) = unsafe { written(&mut place[..xs.len()]) }.split_at(n); gathered(nears, &mut t, os, |i| xs[i], #[inline(always)] |t| { for v in t { *v = near::(*v) } }, |_, v| v); @@ -693,10 +693,11 @@ fn regions(x: &[R], out: &mut [MaybeUninit], f: outside |= !(p < 1.0); p <= 0.5 }); - let (ps, qs, signs): (&[f64], &[f64], &[f64]) = (written(&mut ps[..xs.len()]), written(&mut qs[..xs.len()]), written(&mut signs[..xs.len()])); - let (central, others) = written(&mut place[..xs.len()]).split_at(central); + let (ps, qs, signs): (&[f64], &[f64], &[f64]) = + (unsafe { written(&mut ps[..xs.len()]) }, unsafe { written(&mut qs[..xs.len()]) }, unsafe { written(&mut signs[..xs.len()]) }); + let (central, others) = unsafe { written(&mut place[..xs.len()]) }.split_at(central); let upper = placed(others.iter().copied(), &mut split, |i| qs[usize::from(i)] >= 0.25); - let (upper, below) = written(&mut split[..others.len()]).split_at(upper); + let (upper, below) = unsafe { written(&mut split[..others.len()]) }.split_at(upper); let signed = |i: usize, v: f64| R::narrow(v.copysign(signs[i])); gathered(central, &mut t, os, |i| ps[i], #[inline(always)] |t| { for v in t { *v = maddock_central::(*v) } @@ -728,7 +729,7 @@ fn gathered( ) { let t = &mut t[..positions.len()]; for (v, &i) in t.iter_mut().zip(positions) { v.write(arg(usize::from(i))); } - let t = written(t); + let t = unsafe { written(t) }; formula(t); for (&v, &i) in t.iter().zip(positions) { let i = usize::from(i); diff --git a/src/sqrts.rs b/core/src/sqrts.rs similarity index 98% rename from src/sqrts.rs rename to core/src/sqrts.rs index 3138b51..5f9a18e 100644 --- a/src/sqrts.rs +++ b/core/src/sqrts.rs @@ -38,4 +38,4 @@ pub(crate) fn sqrtx2m1(x: R) -> R { } #[derive(Clone, Copy, Debug)] -pub(crate) struct Sqrtm1mx2(pub(crate) bool); +pub struct Sqrtm1mx2(pub bool); diff --git a/src/stats.rs b/core/src/stats.rs similarity index 100% rename from src/stats.rs rename to core/src/stats.rs diff --git a/src/structure.rs b/core/src/structure.rs similarity index 100% rename from src/structure.rs rename to core/src/structure.rs diff --git a/src/table.rs b/core/src/table.rs similarity index 100% rename from src/table.rs rename to core/src/table.rs diff --git a/src/tolerant.rs b/core/src/tolerant.rs similarity index 97% rename from src/tolerant.rs rename to core/src/tolerant.rs index 93b4c35..404cc78 100644 --- a/src/tolerant.rs +++ b/core/src/tolerant.rs @@ -368,7 +368,7 @@ fn in_band(xs: &[F], lo: F, hi: F, band: Band, out: &mut [std::mem: } /// The float types that `against` compares with one number to give Booleans. The default writes them through `in_band`. Wasm's `f64` and `f32` override it in `narrowed`. -pub(crate) trait Banded: Float { +pub trait Banded: Float { /// Writes into `out` whether each item of `xs` lies in `band` against the range `lo` to `hi`. #[inline(always)] fn banded(xs: &[Self], lo: Self, hi: Self, band: Band, out: &mut [std::mem::MaybeUninit]) { in_band(xs, lo, hi, band, out, |b| b) } @@ -380,7 +380,7 @@ impl Banded for f64 {} /// Compares each item of the `Many` argument with the `One` argument `c`, through the range of floats equal to `c`, worked out once. Each item then takes one or two plain comparisons, which `banded` writes: Booleans through `Banded`, or an `Indicator`'s floats through `in_band`. `left` is the band an item on the left of `c` must lie in, and `right` the band for an item on its right. `None`, writing nothing, unless exactly one argument is `One` and `c` isn't NaN. #[inline(always)] -pub(crate) fn against( +pub fn against( t: Tolerance, x: Arg<'_, F>, y: Arg<'_, F>, n: usize, out: &mut [std::mem::MaybeUninit], left: Band, right: Band, banded: impl FnOnce(&[F], F, F, Band, &mut [std::mem::MaybeUninit])) -> Option { let (items, (lo, hi), band) = match (x, y) { @@ -476,7 +476,7 @@ fn complex_equals( let (equal, decided) = t.squared_equal(a, b); (equal != negate, decided) }) { return true; } - let out = crate::slice::written(out); + let out = unsafe { crate::slice::written(out) }; for i in 0..n { let (a, b) = (x.nth(i), y.nth(i)); if !t.squared_equal(a, b).1 { out[i] = t.scaled_equal(a, b) != negate } } true } @@ -593,7 +593,7 @@ tolerant!(Residue at Complex, Complex: |t, [x, y]| t.complex_residue(x /// Recomputes through [`exact_residue`] the residue in `out` of each item that isn't [`near`]. Of those, only the items that [`far`] picks change. The slice form runs this only where its loop finds an item that isn't near. fn replace_far(x: Arg<'_, F>, y: Arg<'_, F>, n: usize, out: &mut [std::mem::MaybeUninit]) { - for (i, r) in crate::slice::written(&mut out[..n]).iter_mut().enumerate() { + for (i, r) in unsafe { crate::slice::written(&mut out[..n]) }.iter_mut().enumerate() { let (x, y) = (x.nth(i), y.nth(i)); if !near(x, y) { *r = exact_residue(x, y); } } @@ -761,20 +761,22 @@ pub fn wholes(t: impl Tolerant, x: &[F], limit: f64) -> Option<(Vec f64 { if self == Self::Tan { 2.0 * FRAC_1_PI } else { FRAC_1_PI } } + #[inline] fn offset(self) -> f64 { if self == Self::Cos { 0.5 } else { 0.0 } } + #[inline] pub(crate) fn predicate + ?Sized>(self, o: &O, x: O::V) -> O::B { let a = o.magnitude(x); let limit = o.number(F::narrow(if o.width() == FloatWidth::F64 { 8388608.0 } else { 1048576.0 })); @@ -47,6 +50,7 @@ pub(crate) fn cos(x: R) -> R { scalar::( #[inline] pub(crate) fn tan(x: R) -> R { scalar::(x, Kind::Tan) } +#[inline] pub(crate) fn formula + ?Sized>(o: &O, x: O::V, kind: Kind) -> O::V { let zero = o.number(F::zero()); let finite = o.less(o.magnitude(x), o.number(F::infinity())); @@ -69,6 +73,7 @@ const PI_FUSED_32: [f64; 3] = [3.141592502593994, 1.5099578831723193e-7, 1.07806 const PI_SPLIT_64: [f64; 4] = [3.141592651605606, 1.9841871583270443e-9, 1.03403659577805e-18, 5.807711947958721e-28]; const PI_SPLIT_32: [f64; 2] = [3.141592651605606, 1.984187159361081e-9]; +#[inline] fn reduce_fused + ?Sized>(o: &O, x: O::V, kind: Kind) -> (O::V, O::V) { let shift = crate::recipe::rounding_shift(o); let z = if kind.offset() == 0.0 { o.multiply_add(x, o.number(F::narrow(kind.scale())), shift) } @@ -80,6 +85,7 @@ fn reduce_fused + ?Sized>(o: &O, x: O::V, kind: Kind) -> (O: (r, n) } +#[inline] fn reduce_split + ?Sized>(o: &O, x: O::V, kind: Kind) -> (O::V, O::V) { let wide = o.widen(x); let z = o.wide_multiply_add(wide, o.wide_number(kind.scale()), o.wide_number(kind.offset())); @@ -98,6 +104,7 @@ const SIN_64: [f64; 7] = [ -0.1666666666666606, ]; const SIN_32: [f64; 4] = [2.6105958568223286e-6, -0.00019811994570773095, 0.00833310279995203, -0.16666661202907562]; +#[inline] fn sine + ?Sized>(o: &O, r: O::V) -> O::V { let r2 = o.multiply(r, r); let p = crate::recipe::horner(o, r2, if o.width() == FloatWidth::F64 { &SIN_64 } else { &SIN_32 }); @@ -109,6 +116,7 @@ const TAN_64: [f64; 9] = [ 0.13333333333289177, 0.33333333333333504, ]; const TAN_32: [f64; 4] = [0.025565408170223236, 0.053473446518182755, 0.13335898518562317, 0.33333292603492737]; +#[inline] fn tangent + ?Sized>(o: &O, r: O::V, odd: O::B) -> O::V { let u = o.multiply(r, o.number(F::narrow(0.5))); let u2 = o.multiply(u, u); diff --git a/kernels/Cargo.toml b/kernels/Cargo.toml new file mode 100644 index 0000000..f72fdfd --- /dev/null +++ b/kernels/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rustymath-kernels" +version.workspace = true +edition.workspace = true +rust-version = "1.98" +license = "Apache-2.0" +description = "Resolved numerical kernels over Rustymath formulas and storage." +repository = "https://github.com/AnswerDotAI/rustymath" + +[lib] +test = false +doctest = false + +[dependencies] +rustymath-core.workspace = true +half.workspace = true +num-complex.workspace = true +num-rational.workspace = true +num-traits.workspace = true + diff --git a/kernels/src/lib.rs b/kernels/src/lib.rs new file mode 100644 index 0000000..af83363 --- /dev/null +++ b/kernels/src/lib.rs @@ -0,0 +1,5 @@ +//! Resolved numerical kernels over Rustymath types and formulas. +#[macro_use] +extern crate rustymath_core; +pub use rustymath_core::*; +pub mod operation; diff --git a/src/operation.rs b/kernels/src/operation.rs similarity index 88% rename from src/operation.rs rename to kernels/src/operation.rs index aee3c7f..d398951 100644 --- a/src/operation.rs +++ b/kernels/src/operation.rs @@ -5,8 +5,9 @@ use crate::buffer::{Buffer, DType, Element, FloatWidth, Source, View, Width}; use crate::domain::Range; use crate::error::{Error, Kind}; use crate::gather::Writes; -use crate::reduce::native_reduction; +use crate::reduce::{Direction, Piece, Pieces, Seed, native_reduction}; use crate::slice::Arg; +use crate::{FUSED, Native, facts::Formula, recipe::NativeOps, tolerant::Banded}; /// Numeric settings rebound for each execution, outside item loops. `tolerance` is the run's comparison tolerance: tolerant objects compare within it, and a kernel reads a float argument as a whole number within it (`Projection::Wholes`). BPL sets it nonzero only under Mathish, and Python only in Mathish mode. #[derive(Clone, Copy, Debug)] @@ -16,16 +17,6 @@ impl Settings { pub fn ops(self) -> crate::recipe::NativeOps<{ crate::FUSED }> { crate::recipe::NativeOps { tolerance: Some(self.tolerance.value()) } } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Direction { Left, Right } - -#[derive(Clone, Copy)] -pub enum Seed<'a> { Identity, Initial(crate::agreement::Read<'a>), Tail(crate::agreement::Read<'a>) } -impl<'a> Seed<'a> { - /// The seed's items, or `None` for the identity. - pub fn read(self) -> Option> { match self { Self::Initial(read) | Self::Tail(read) => Some(read), Self::Identity => None } } -} - /// How a kernel reads its inputs before its formula runs (`Kernel::projection`, `with_projected`). #[derive(Clone, Copy, PartialEq, Debug)] pub enum Projection { @@ -59,31 +50,6 @@ impl WriteRoot<'_> { } } -/// A compiled fold's total over `items` items, as its 8-byte slot holds it: a float as an f64's bits, an integer extended to 64 bits, and a Boolean as 0 or 1. -#[derive(Clone, Copy, Debug)] -pub struct Total { pub items: usize, pub bits: u64 } -impl Total { - pub fn value(self) -> Result { - use crate::buffer::{Floats, Ints}; - match T::DTYPE { - DType::Boolean => T::cast(View::Booleans(&[self.bits != 0]), 0), - DType::Integer(width) => { - let value = match width { - Width::U8 => self.bits as u8 as i64, Width::I16 => self.bits as i16 as i64, Width::I32 => self.bits as i32 as i64, - Width::I64 => self.bits as i64 - }; - T::cast(View::Integers(Ints::I64(&[value])), 0) - } - DType::Float(_) => T::cast(View::Floats(Floats::F64(&[f64::from_bits(self.bits)])), 0), - _ => Err(Error::new(Kind::Domain, "compiled totals have a real machine dtype")), - } - } -} - -pub enum Piece<'a> { Items(View<'a>), Total(Total) } - -pub type Pieces<'a> = dyn Fn(std::ops::Range, &mut dyn FnMut(usize, Piece<'_>)) -> Result<(), Error> + Sync + 'a; - /// One resolved mathematical definition and its captured numerical policies. #[derive(Clone)] pub struct Kernel { @@ -122,8 +88,8 @@ trait Application: Send + Sync { ) -> Option; /// `judged`: whether each item's result is checked for errors, which a kernel skips where its arguments' types can't make one (`Kernel::may_raise`). fn map( - &self, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, repair: bool, - judged: bool, output: &mut Buffer) -> Result<(), Error>; + &self, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, settings: Settings, repair: bool, judged: bool, + output: &mut Output<'_, '_>) -> Result<(), Error>; /// `route`, an ordered walk by this binary formula's per-item kernels at `variant`'s computing type (`binary_formula`). fn binary(&self, variant: crate::domain::Variant, settings: Settings, route: Route<'_>, output: &mut Buffer) -> Result<(), Error>; } @@ -208,14 +174,18 @@ impl, const N: usize> Application for Definition, boolean_inputs: bool, wanted: Option, settings: Settings ) -> Option { product_for::(&self.object, variant, fold, boolean_inputs, wanted, settings) } fn map( - &self, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, repair: bool, - judged: bool, output: &mut Buffer) -> Result<(), Error> { map_formula::(&self.object, variant, inputs, range, settings, repair, judged, output) } + &self, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, settings: Settings, repair: bool, judged: bool, + output: &mut Output<'_, '_>) -> Result<(), Error> { map_formula::(&self.object, variant, inputs, range, settings, repair, judged, output) } fn binary(&self, variant: crate::domain::Variant, settings: Settings, route: Route<'_>, output: &mut Buffer) -> Result<(), Error> { binary_formula::(&self.object, variant, settings, route, output) } } impl Kernel { + pub fn resolve, const N: usize>(object: K, variant: crate::domain::Variant, types: [DType; N]) -> Result { + Self::try_resolve(object, variant, types)?.ok_or_else(|| Error::new(Kind::Domain, "function has no numerical body for these arguments")) + } + pub fn circle(code: i64, inverse: bool, real_part: bool, variant: crate::domain::Variant, types: [DType; 1]) -> Result, Error> { macro_rules! picked { ($normal:expr, $real:expr) => { @@ -234,7 +204,7 @@ impl Kernel { (1, true) => picked!(crate::Asin, crate::RealAsin), (2, true) => picked!(crate::Acos, crate::RealAcos), (3, true) => picked!(crate::Atan, crate::Atan), (4, true) => picked!(crate::Sqrtx2m1, crate::RealSqrtx2m1), (5, true) => picked!(crate::Asinh, crate::Asinh), (6, true) => picked!(crate::Acosh, crate::RealAcosh), - (7, true) => picked!(crate::Atanh, crate::RealAtanh), (8, inverse) => picked!(crate::sqrts::Sqrtm1mx2(inverse), crate::RealSqrtm1mx2(inverse)), + (7, true) => picked!(crate::Atanh, crate::RealAtanh), (8, inverse) => picked!(crate::Sqrtm1mx2(inverse), crate::RealSqrtm1mx2(inverse)), _ => return Ok(None) })? else { return Ok(None); }; @@ -457,21 +427,43 @@ impl Kernel { }) } pub fn map(&self, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, output: &mut Buffer) -> Result<(), Error> { + output.clear(); + self.mapping(Inputs::Mapped(inputs), range, settings, MapOutput::Buffer(output)) + } + + /// Appends n results from ready inputs into output. A one-item view broadcasts; every other view contains n items. + pub fn map_ready(&self, inputs: &[View<'_>], n: usize, settings: Settings, mut output: MapOutput<'_, '_>) -> Result<(), Error> { + if inputs.iter().any(|v| v.len() != 1 && v.len() != n) { output.fill(n); return Err(Error::new(Kind::Length, "ready kernel input length changed")); } + self.mapping(Inputs::Ready(inputs), 0..n, settings, output) + } + + fn mapping(&self, inputs: Inputs<'_, '_>, range: std::ops::Range, settings: Settings, mut target: MapOutput<'_, '_>) -> Result<(), Error> { use crate::characters::{ByKinds, Policy}; - if inputs.len() != self.arity() { return Err(Error::new(Kind::Length, "kernel operand count changed")); } - let wanted = output.dtype(); - if output.dtype() != self.result { *output = Buffer::with_capacity(self.result, range.len()); } - match self.policy { - Some(Policy::Kinds(ByKinds::Truth(value))) => *output = Buffer::Booleans(vec![value; range.len()]), - Some(Policy::Kinds(side)) => { - let side = usize::from(side == ByKinds::Right); - inputs[side].read_into(range, self.result, self.variant.checked, output)?; + if inputs.len() != self.arity() { target.fill(range.len()); return Err(Error::new(Kind::Length, "kernel operand count changed")); } + if range.is_empty() { return Ok(()); } + let mut output = Output { target, dtype: self.result, checked: self.variant.fails_on_overflow() }; + crate::error::checked(|| match self.policy { + Some(Policy::Kinds(ByKinds::Truth(value))) => map::constant(value, range.len(), &mut output), + Some(Policy::Kinds(side)) => map::copied(inputs.selected(usize::from(side == ByKinds::Right)), range, self.variant.checked, &mut output), + _ => { + let apply = |inputs: Inputs<'_, '_>, output: &mut Output<'_, '_>| { + self.definition.map(self.variant, inputs, range.clone(), settings, self.repair, self.may_raise(), output) + }; + match inputs { + Inputs::Mapped(reads) => { + self.projected(reads, Some(range.clone()), Seed::Identity, settings, |reads, _| apply(Inputs::Mapped(reads), &mut output)) + } + Inputs::Ready(_) if self.projection(settings) == Projection::AsIs => apply(inputs, &mut output), + Inputs::Ready(views) => { + let projected = match views.iter().map(|&v| projected_view(v, self.projection(settings))).collect::, _>>() { + Ok(projected) => projected, Err(error) => { output.target.fill(range.len()); return Err(error); } + }; + let views: Vec<_> = projected.iter().map(|v| v.view()).collect(); + apply(Inputs::Ready(&views), &mut output) + } + } } - _ => self.projected(inputs, Some(range.clone()), Seed::Identity, settings, |inputs, _| { - self.definition.map(self.variant, inputs, range, settings, self.repair, self.may_raise(), output) - })?, - } - finish(output, wanted, self.variant) + }) } pub fn fold_pieces( &self, dims: [usize; 3], seed: Seed<'_>, direction: Direction, settings: Settings, pieces: &Pieces<'_>, output: &mut Buffer) -> Result<(), Error> { @@ -582,29 +574,36 @@ fn choose_typed( { let mut bits = Vec::new(); if let Ok(Arg::Many(mask)) = selector.0.part::(part(selector.0), checked, &mut bits) && mask.len() == range.len() { - return slots.fill(range.len(), |out| { - for (((slot, &m), &n), &y) in out.iter_mut().zip(mask.iter()).zip(no.iter()).zip(yes.iter()) { - slot.write(core::hint::select_unpredictable(m, y, n)); - } - true - }); + return unsafe { + slots.fill(range.len(), |out| { + for (((slot, &m), &n), &y) in out.iter_mut().zip(mask.iter()).zip(no.iter()).zip(yes.iter()) { + slot.write(core::hint::select_unpredictable(m, y, n)); + } + true + }) + }; } } let mut indices = Buffer::with_capacity(selector.1, 0); let positions = selector.0.part_view(part(selector.0), selector.1, false, &mut indices) .and_then(|view| crate::structure::positions(view, choices.len(), crate::tolerant::Tolerance::EXACT)); match positions.and_then(|positions| args.map(|args| (positions, args))) { - Ok((positions, args)) => slots.fill(range.len(), |out| { - for (i, slot) in out.iter_mut().enumerate() { - let position = positions[if positions.len() == 1 { 0 } else { i }]; - let value = match args.get(position) { - Some(Arg::One(value)) => *value, Some(Arg::Many(values)) => values[if values.len() == 1 { 0 } else { i }], None => T::FILL - }; - slot.write(value); - } - true - }), - Err(error) => { crate::error::record(error); slots.fill(range.len(), |out| { for slot in out { slot.write(T::FILL); } false }) } + Ok((positions, args)) => unsafe { + slots.fill(range.len(), |out| { + for (i, slot) in out.iter_mut().enumerate() { + let position = positions[if positions.len() == 1 { 0 } else { i }]; + let value = match args.get(position) { + Some(Arg::One(value)) => *value, Some(Arg::Many(values)) => values[if values.len() == 1 { 0 } else { i }], None => T::FILL + }; + slot.write(value); + } + true + }) + }, + Err(error) => { + crate::error::record(error); + unsafe { slots.fill(range.len(), |out| { for slot in out { slot.write(T::FILL); } false }) } + } } }); Ok(()) @@ -621,12 +620,17 @@ pub fn whole_numbers(t: crate::tolerant::Tolerance, values: &[F } /// `f` on `view` as an integer computation reads it (`with_projected`): characters as their codes under `Projection::Codes`, and floats as the whole numbers they equal within the projection's tolerance (`whole_numbers`). -fn with_projected_view(view: View<'_>, projection: Projection, f: impl FnOnce(View<'_>) -> R) -> Result { +fn projected_view(view: View<'_>, projection: Projection) -> Result, Error> { + use crate::buffer::{IntBuf, Items}; let view = if matches!(projection, Projection::Codes(_)) { crate::characters::as_codes(view) } else { view }; - let Some(t) = projection.wholes().filter(|_| matches!(view.dtype(), DType::Float(_))) else { return Ok(f(view)) }; + let Some(t) = projection.wholes().filter(|_| matches!(view.dtype(), DType::Float(_))) else { return Ok(Items::Borrowed(view)) }; let mut floats = Vec::new(); ::extend_cast(view, view.len(), true, &mut floats)?; - Ok(f(View::Integers(crate::buffer::Ints::I64(&whole_numbers(t, &floats)?)))) + Ok(Items::Owned(Buffer::Integers(IntBuf::I64(whole_numbers(t, &floats)?)))) +} + +fn with_projected_view(view: View<'_>, projection: Projection, f: impl FnOnce(View<'_>) -> R) -> Result { + Ok(f(projected_view(view, projection)?.view())) } /// `call` on `inputs` as an integer computation reads them: characters as their codes under `Projection::Codes`, and floats as the whole numbers they equal within the projection's tolerance (`whole_numbers`), as `i64`, over `range`, or over the whole of each input where `None`. @@ -669,7 +673,7 @@ fn with_projected( } macro_rules! with_binary_body { - ($object:expr, $variant:expr, $settings:expr, |$T:ident, $apply:ident| $body:expr) => {{ + ($object:expr, $variant:expr, $settings:expr, |$T:ident, $apply:ident, $compare:ident| $body:expr) => {{ let (object, variant, settings) = ($object, $variant, $settings); let ops = settings.ops(); match variant.dtype { @@ -678,21 +682,25 @@ macro_rules! with_binary_body { with_width!(width, $T => { let ops = crate::recipe::IntegerOps::<$T>::new(variant); let $apply = { #[inline(always)] |x: $T, y: $T| integer_value(object, variant, &ops, std::array::from_fn(|i| if i == 0 { x } else { y })) }; + let $compare = |_: $T, _: &[$T], _: bool, _: &mut Vec| None; $body }) }, DType::Float(FloatWidth::F16 | FloatWidth::F32) => { type $T = f32; let $apply = { #[inline(always)] move |x: $T, y: $T| real_value(object, variant, &ops, std::array::from_fn(|i| if i == 0 { x } else { y }), true) }; + let $compare = comparison_writer(object, &ops); $body }, DType::Float(FloatWidth::F64) => { type $T = f64; let $apply = { #[inline(always)] move |x: $T, y: $T| real_value(object, variant, &ops, std::array::from_fn(|i| if i == 0 { x } else { y }), true) }; + let $compare = comparison_writer(object, &ops); $body }, DType::Complex(w) => with_complex_width!(w, $T => { let $apply = { #[inline(always)] move |x: $T, y: $T| complex_value(object, variant, &ops, std::array::from_fn(|i| if i == 0 { x } else { y })) }; + let $compare = |_: $T, _: &[$T], _: bool, _: &mut Vec| None; $body }), DType::Character => Err(Error::new(Kind::Domain, "a numerical body needs code-point inputs for characters")), @@ -700,6 +708,23 @@ macro_rules! with_binary_body { }}; } +/// Compares a block against a constant carry through the same tolerance bands as an elementwise map. +fn comparison_writer, F: Native + Banded, const N: usize>( + object: &K, ops: &NativeOps<{ FUSED }>) -> impl Fn(F, &[F], bool, &mut Vec) -> Option + Copy { + let bands = object.bands().and_then(|(t, left, right)| Some((crate::facts::run_tolerance::(ops, t)?, left, right))); + move |carry, items, flipped, out| { + let (t, left, right) = bands?; + let (x, y) = (Arg::One(carry), Arg::Many(items)); + let (x, y) = if flipped { (y, x) } else { (x, y) }; + let (at, n) = (out.len(), items.len()); + out.reserve(n); + let ok = crate::tolerant::against(t, x, y, n, &mut out.spare_capacity_mut()[..n], left, right, F::banded)?; + // SAFETY: `against` writes all `n` slots when it returns `Some`. + unsafe { out.set_len(at + n) }; + Some(ok) + } +} + /// The results of `step` for each pair of items of `xs` and `ys`, written to `out`, in one loop that carries each result's validity as a flag with no branch. A block with a failure finds the first again and gives its kind. fn paired(step: impl Fn(T, T) -> (T, Option)) -> impl Fn(&[T], &[T], &mut [std::mem::MaybeUninit]) -> Result<(), Kind> { move |xs: &[T], ys: &[T], out: &mut [std::mem::MaybeUninit]| { @@ -755,7 +780,7 @@ fn verdict, F: crate::Float, O: crate::recipe::Ops, F: crate::float::Native, const N: usize>( +fn complex_valid, F: crate::Native, const N: usize>( ops: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [num_complex::Complex; N], result: num_complex::Complex) -> bool { !K::FAILS_NAN || crate::facts::nan_valid(ops, x.into_iter().flat_map(|x| [x.re, x.im]), [result.re, result.im]) } @@ -763,7 +788,7 @@ fn complex_valid, F: crate::float::Native, const N: #[inline(always)] fn real_value( object: &K, variant: crate::domain::Variant, ops: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [F; N], repair: bool) -> (F, Option) where - K: crate::facts::Formula, F: crate::float::Native { + K: crate::facts::Formula, F: crate::Native { let result = if repair && object.emit_repair(ops, x) == Some(true) { crate::facts::Formula::repair_routine(object).map(|r| r.apply(x[0])) } else { object.real(ops, x) }; let Some(result) = result else { return (F::zero(), variant.checked.then_some(Kind::Domain)); }; @@ -791,7 +816,7 @@ fn boolean_raises, const N: usize>(object: &K, varia #[inline(always)] fn complex_value( object: &K, variant: crate::domain::Variant, ops: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [num_complex::Complex; N] -) -> (num_complex::Complex, Option) where K: crate::facts::Formula, F: crate::float::Native { +) -> (num_complex::Complex, Option) where K: crate::facts::Formula, F: crate::Native { let Some(result) = object.complex(ops, x) else { return (num_complex::Complex::new(F::zero(), F::zero()), variant.checked.then_some(Kind::Domain)); }; let valid = !variant.checked || complex_valid::(ops, x, result); (result, (!valid).then_some(Kind::Domain)) @@ -806,9 +831,6 @@ fn reduction, const N: usize>(object: &K, variant: c if variant.dtype == DType::Boolean { object.truth().or_else(|| object.fold()) } else { object.fold() } } -/// The float width that a Boolean or integer computes in under rustymath's own policy, NumPy's: the float that [`DType::inexact`] gives it under `f64`. `f64` for any other dtype. -pub fn numpy_float(dtype: DType) -> FloatWidth { match dtype.inexact(FloatWidth::F64) { Some((DType::Float(w), _)) => w, _ => FloatWidth::F64 } } - /// `output`, a kernel's results at `variant`, as `dtype`, which a caller can ask for narrower than the kernel's result type. An integer that doesn't fit fails where `variant.fails_on_overflow()`, and wraps otherwise. fn finish(output: &mut Buffer, dtype: DType, variant: crate::domain::Variant) -> Result<(), Error> { if output.dtype() != dtype { *output = Buffer::cast_results(output.view(), dtype, variant.fails_on_overflow())?; } @@ -820,6 +842,5 @@ mod binary; mod fold; mod map; mod product; -pub(crate) use map::real_slice; -pub use {at::*, product::*}; -use {binary::*, fold::*, map::map_formula}; +pub use {at::*, map::MapOutput, product::*}; +use {binary::*, fold::*, map::{Inputs, Output, map_formula}}; diff --git a/src/operation/at.rs b/kernels/src/operation/at.rs similarity index 100% rename from src/operation/at.rs rename to kernels/src/operation/at.rs diff --git a/src/operation/binary.rs b/kernels/src/operation/binary.rs similarity index 64% rename from src/operation/binary.rs rename to kernels/src/operation/binary.rs index 410e6d2..70f5156 100644 --- a/src/operation/binary.rs +++ b/kernels/src/operation/binary.rs @@ -12,7 +12,7 @@ pub(super) enum Route<'a> { pub(super) fn binary_formula, const N: usize>( object: &K, variant: crate::domain::Variant, settings: Settings, route: Route<'_>, output: &mut Buffer) -> Result<(), Error> { if const { N != 2 || K::MAP_ONLY } { return Err(Error::new(Kind::Length, "an ordered walk needs a binary function")); } - with_binary_body!(object, variant, settings, |T, apply| { + with_binary_body!(object, variant, settings, |T, apply, compare| { // `step(total, item)` for every route, swapped for a right walk, so that the routes share one `paired`. let step = |right: bool| { #[inline(always)] @@ -25,17 +25,35 @@ pub(super) fn binary_formula, const N: usize>( } Route::Scan { dims, seed, direction, input } => { let step = step(direction == Direction::Right); + let both = |carry, items: &[T], out: &mut Vec| compare(carry, items, direction == Direction::Right, out); let (lane, block) = (chained(step), paired(step)); + let ordered = |items: &[T], dims, seed: Option<&[T]>, out: &mut Vec| { + // SAFETY: `paired` initializes every output before reporting an error. + unsafe { crate::reduce::ordered_scan(items, dims, seed, out, &lane, &block) } + .map_or(Ok(()), |kind| Err(Error::new(kind, "ordered scan is outside the function's domain"))) + }; + if K::BOOLEAN && output.dtype() == DType::Boolean { + return crate::reduce::boolean_scan_read::(dims, seed, direction, variant.checked, input, output, &|items, dims, seed, out| { + let zero: T = false.read(); + if dims[2] == 1 && crate::reduce::comparison_scan(items, dims[1], seed.map(|s| s[0]), step, |x| x != zero, both, out) { + return Ok(()); + } + let mut values = Vec::new(); + ordered(items, dims, seed, &mut values)?; + out.extend(values.into_iter().map(|x| x != zero)); + Ok(()) + }); + } crate::reduce::scan_read::(dims, seed, direction, variant.checked, input, output, &|items, dims, seed, out| { - if K::BOOLEAN && dims[2] == 1 && crate::reduce::comparison_scan(items, dims[1], seed.map(|s| s[0]), step, out) { + if K::BOOLEAN && dims[2] == 1 && crate::reduce::comparison_scan(items, dims[1], seed.map(|s| s[0]), step, |x| x, both, out) { return Ok(()); } - crate::reduce::ordered_scan(items, dims, seed, out, &lane, &block) - .map_or(Ok(()), |kind| Err(Error::new(kind, "ordered scan is outside the function's domain"))) + ordered(items, dims, seed, out) }) } Route::Adjacent { dims, seed, flip, input } => { - crate::reduce::adjacent_read::(dims, seed, variant.checked, flip, input, output, &paired(step(false))) + // SAFETY: `paired` initializes every output before reporting an error. + unsafe { crate::reduce::adjacent_read::(dims, seed, variant.checked, flip, input, output, &paired(step(false))) } } Route::At { root, values, ids, checked, projection } => { root.initialize(T::DTYPE, checked, projection, output)?; diff --git a/kernels/src/operation/fold.rs b/kernels/src/operation/fold.rs new file mode 100644 index 0000000..a91fb4e --- /dev/null +++ b/kernels/src/operation/fold.rs @@ -0,0 +1,75 @@ +use super::*; + +fn piece_values(range: std::ops::Range, checked: bool, pieces: &Pieces<'_>, out: &mut Vec) -> Result<(), Error> { + out.clear(); + let (mut next, mut failure) = (range.start, None); + pieces(range.clone(), &mut |at, piece| { + if failure.is_some() { return; } + let Piece::Items(view) = piece else { failure = Some(Error::new(Kind::Length, "ordered fold received a precomputed total")); return; }; + if at != next || at + view.len() > range.end { failure = Some(Error::new(Kind::Length, "fold pieces do not cover the requested range")); return; } + if let Err(error) = T::extend_cast(view, view.len(), checked, out) { failure = Some(error); return; } + next += view.len(); + })?; + if let Some(error) = failure { return Err(error); } + if next != range.end { return Err(Error::new(Kind::Length, "fold pieces left an unread range")); } + Ok(()) +} + +/// The ordered folds of each lane of `[outer, len, inner]`, read from `pieces` a chunk of whole cells or rows at a time. Each chunk folds from the right onto the fold so far: by `lane`, one call for each lane, along the last axis, and by `block`, one call for each row, along any other axis. +pub(super) fn ordered_fold( + [outer, len, inner]: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, identity: Option, pieces: &Pieces<'_>, + lane: &dyn Fn(&[T], T) -> (T, Option), block: &dyn Fn(&[T], &[T], &mut [std::mem::MaybeUninit]) -> Result<(), Kind>, output: &mut Buffer +) -> Result<(), Error> { + let (mut result, mut items, mut seed_scratch) = (Vec::with_capacity(outer * inner), Vec::new(), Vec::new()); + if inner == 0 { *output = T::buffer(result); return Ok(()); } + let (right, cell_items) = (direction == Direction::Right, len * inner); + let read_ahead = crate::CHUNK / cell_items.max(1) * cell_items; + let (mut loaded, mut total, mut next) = (0, Vec::with_capacity(inner), Vec::with_capacity(inner)); + let fold = |mut rows: &[T], total: &mut Vec, next: &mut Vec| { + if total.is_empty() { + let (rest, tail) = rows.split_at(rows.len() - inner); + total.extend_from_slice(tail); + rows = rest; + } + let mut failure = None; + if inner == 1 { (total[0], failure) = lane(rows, total[0]); } else { + for row in rows.chunks_exact(inner).rev() { + next.clear(); + failure = failure.or(unsafe { crate::slice::appended(next, inner, |slots| block(total, row, slots)) }.err()); + std::mem::swap(total, next); + } + } + failure.map_or(Ok(()), |kind| Err(Error::new(kind, "ordered fold is outside the function's domain"))) + }; + for cell in 0..outer { + total.clear(); + let given = seed.read().map(|read| crate::reduce::seed_row(&read, cell, inner, checked, &mut seed_scratch)).transpose()?; + let upfront = len == 0 || match seed { Seed::Tail(_) => right, Seed::Initial(_) => !right, Seed::Identity => false }; + if upfront && let Some(seed) = &given { total.extend_from_slice(seed); } + let rows = (crate::CHUNK / inner).max(1); + let mut remaining = len; + while remaining != 0 { + let n = rows.min(remaining); + let start = if right { remaining - n } else { len - remaining }; + let at = (cell * len + start) * inner; + let end = at + n * inner; + if at < loaded || end > loaded + items.len() { + // Short cells share a read; long cells keep their original row chunks. + piece_values(at..end.max(at + read_ahead).min(outer * cell_items), checked, pieces, &mut items)?; + loaded = at; + } + let chunk = &mut items[at - loaded..end - loaded]; + if !right { crate::reduce::reverse_rows(chunk, [1, n, inner]); } + fold(chunk, &mut total, &mut next)?; + remaining -= n; + } + if total.is_empty() { + let value = identity.ok_or_else(|| Error::new(Kind::Length, "an empty fold needs an identity or a seed"))?; + total.resize(inner, T::cast(View::Floats(crate::buffer::Floats::F64(&[value])), 0)?); + } + if !upfront && let Some(seed) = given { fold(&seed, &mut total, &mut next)?; } + result.extend_from_slice(&total); + } + *output = T::buffer(result); + Ok(()) +} diff --git a/src/operation/map.rs b/kernels/src/operation/map.rs similarity index 67% rename from src/operation/map.rs rename to kernels/src/operation/map.rs index 74236d4..794329c 100644 --- a/src/operation/map.rs +++ b/kernels/src/operation/map.rs @@ -4,32 +4,39 @@ use crate::{FUSED, recipe::NativeOps}; mod complex; mod integer; mod real; -pub(crate) use real::real_slice; use {complex::*, integer::*, real::map_real}; +mod io; +pub use io::MapOutput; +use io::OutputItem; +pub(super) use io::{Inputs, Output}; pub(super) fn map_formula, const N: usize>( - object: &K, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, repair: bool, - judged: bool, output: &mut Buffer) -> Result<(), Error> { + object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, settings: Settings, repair: bool, judged: bool, + output: &mut Output<'_, '_>) -> Result<(), Error> { if inputs.len() != N { return Err(Error::new(Kind::Length, "kernel operand count changed")); } let ops = settings.ops(); crate::events::taken(crate::events::Route::VectorKernel, range.len()); - report_maps(inputs.iter().map(|read| read.plan), range.clone()); + match inputs { + Inputs::Mapped(reads) => report_maps(reads.iter().map(|read| read.plan), range.clone()), + Inputs::Ready(views) if range.len() > 1 && views.iter().any(|v| v.len() == 1) => crate::events::taken(crate::events::Route::MappedKernel, range.len()), + _ => {} + } // A kernel whose arguments' types can't make an error judges no item: `judge` is `variant` unchecked. Reads and `IntegerOps` keep `variant`, so an argument that doesn't convert, or an overflow that `IntegerOps::result` reports, still fails. let judge = crate::domain::Variant { checked: variant.checked && judged, ..variant }; let slices = object.native_slices(); if let Some(slices) = &slices { let valid = |ok: bool| if judge.checked && !ok { Err(Kind::Domain) } else { Ok(()) }; let blocks = match variant.dtype { - DType::Float(FloatWidth::F64) => sliced(object, &ops, slices.double, inputs, range.clone(), variant.checked, &valid, output), - DType::Float(FloatWidth::F32) => sliced(object, &ops, slices.single, inputs, range.clone(), variant.checked, &valid, output), - DType::Float(FloatWidth::F16) => slices.single.map(|single| { + DType::Float(FloatWidth::F64) => sliced(object, &ops, slices.double(), inputs, range.clone(), variant.checked, &valid, output), + DType::Float(FloatWidth::F32) => sliced(object, &ops, slices.single(), inputs, range.clone(), variant.checked, &valid, output), + DType::Float(FloatWidth::F16) => slices.single().map(|single| { map_blocks::(inputs, range.clone(), variant.checked, output, &|args, n, out| { - valid(crate::slice::buffered(n, out, |out| single(object, &ops, args, n, out), half::f16::from_f32)) + valid(unsafe { crate::slice::buffered(n, out, |out| single(object, &ops, args, n, out), half::f16::from_f32) }) }) }), - DType::Integer(Width::U8) => sliced(object, &ops, slices.byte, inputs, range.clone(), variant.checked, &valid, output), - DType::Integer(Width::I16) => sliced(object, &ops, slices.short, inputs, range.clone(), variant.checked, &valid, output), - DType::Integer(Width::I32) => sliced(object, &ops, slices.int, inputs, range.clone(), variant.checked, &valid, output), - DType::Integer(Width::I64) => sliced(object, &ops, slices.long, inputs, range.clone(), variant.checked, &valid, output), + DType::Integer(Width::U8) => sliced(object, &ops, slices.byte(), inputs, range.clone(), variant.checked, &valid, output), + DType::Integer(Width::I16) => sliced(object, &ops, slices.short(), inputs, range.clone(), variant.checked, &valid, output), + DType::Integer(Width::I32) => sliced(object, &ops, slices.int(), inputs, range.clone(), variant.checked, &valid, output), + DType::Integer(Width::I64) => sliced(object, &ops, slices.long(), inputs, range.clone(), variant.checked, &valid, output), _ => None, }; if let Some(blocks) = blocks { return blocks; } @@ -42,47 +49,42 @@ pub(super) fn map_formula, const N: usize>( Some(table) => map_truth::(table, inputs, range, output), None => map_integer::(object, variant, inputs, range, judge, output) }, DType::Integer(width) => with_width!(width, I => map_integer::(object, variant, inputs, range, judge, output)), - DType::Complex(crate::buffer::ComplexWidth::F32) if K::REAL_COMPLEX && real_inputs(inputs) => { + DType::Complex(crate::buffer::ComplexWidth::F32) if K::REAL_COMPLEX && inputs.real() => { map_real_complex::(object, variant, inputs, range, &ops, output) } - DType::Complex(crate::buffer::ComplexWidth::F64) if K::REAL_COMPLEX && real_inputs(inputs) => { + DType::Complex(crate::buffer::ComplexWidth::F64) if K::REAL_COMPLEX && inputs.real() => { map_real_complex::(object, variant, inputs, range, &ops, output) } DType::Complex(crate::buffer::ComplexWidth::F32) => { - map_complex::(object, variant, inputs, range, &ops, slices.and_then(|s| s.complex_single), judge, output) + map_complex::(object, variant, inputs, range, &ops, slices.and_then(|s| s.complex_single()), judge, output) } DType::Complex(crate::buffer::ComplexWidth::F64) => { - map_complex::(object, variant, inputs, range, &ops, slices.and_then(|s| s.complex_double), judge, output) + map_complex::(object, variant, inputs, range, &ops, slices.and_then(|s| s.complex_double()), judge, output) } DType::Character => Err(Error::new(Kind::Domain, "characters are not numerical kernel operands")), } } /// `slice`, a formula's slice form at `T` (`NativeSlices`), on each block of `inputs` over `range`, with `valid` turning each block's validity into its result. `None` where the formula has no form at `T`. -fn sliced( - object: &K, ops: &NativeOps<{ FUSED }>, slice: Option>, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, - checked: bool, valid: &(dyn Fn(bool) -> Result<(), Kind> + Sync), output: &mut Buffer) -> Option> { +fn sliced( + object: &K, ops: &NativeOps<{ FUSED }>, slice: Option>, inputs: Inputs<'_, '_>, range: std::ops::Range, checked: bool, + valid: &(dyn Fn(bool) -> Result<(), Kind> + Sync), output: &mut Output<'_, '_>) -> Option> { let slice = slice?; Some(map_blocks::(inputs, range, checked, output, &|args, n, out| valid(slice(object, ops, args, n, out)))) } /// A function of Booleans by its truth table (`Formula::truth_table`): bit `2x+y` of the table, or bit `x` for one argument, gives its result. A function with a table gives it for every Boolean argument, so no item checks its validity. -fn map_truth(table: u8, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, output: &mut Buffer) -> Result<(), Error> { +fn map_truth(table: u8, inputs: Inputs<'_, '_>, range: std::ops::Range, output: &mut Output<'_, '_>) -> Result<(), Error> { let value = |x: [bool; N]| match N { - 1 => crate::arith::bitwise(table, false, x[0]), 2 => crate::arith::bitwise(table, x[0], x[1]), _ => crate::facts::truth_bit(table, x) + 1 => crate::bitwise(table, false, x[0]), 2 => crate::bitwise(table, x[0], x[1]), _ => crate::facts::truth_bit(table, x) }; map_values::(inputs, range, false, false, output, value, |_, _| None) } -/// Whether no source of `inputs` holds complex items, so that a formula's complex results from real arguments (`Formula::real_complex`) can read them as reals. -fn real_inputs(inputs: &[crate::agreement::Read<'_>]) -> bool { - inputs.iter().all(|read| read.sources.iter().all(|b| !matches!(b.view.dtype(), DType::Complex(_)))) -} - /// `apply` on each item of `inputs` over `range`, written to `output` through `map_written`. #[inline] -fn map_values( - inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, checked: bool, judged: bool, output: &mut Buffer, apply: impl Fn([F; N]) -> O + Sync, +fn map_values( + inputs: Inputs<'_, '_>, range: std::ops::Range, checked: bool, judged: bool, output: &mut Output<'_, '_>, apply: impl Fn([F; N]) -> O + Sync, kind: impl Fn([F; N], O) -> Option + Sync) -> Result<(), Error> { map_written(inputs, range, checked, judged, output, |args, n, out| { crate::slice::mapped(args, n, out, #[inline(always)] |x| (apply(x), true)); @@ -91,12 +93,12 @@ fn map_values( /// Each block of `inputs` over `range` as `write` writes all `n` of its slots, to `output`. Where `judged`, `kind` judges each stored result against its arguments: `None` for a valid result, otherwise its error kind. No body runs twice, because a resolved object's body gives a result for every item. #[inline] -fn map_written( - inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, checked: bool, judged: bool, output: &mut Buffer, +fn map_written( + inputs: Inputs<'_, '_>, range: std::ops::Range, checked: bool, judged: bool, output: &mut Output<'_, '_>, write: impl Fn(&[Arg<'_, F>; N], usize, &mut [std::mem::MaybeUninit]) + Sync, kind: impl Fn([F; N], O) -> Option + Sync) -> Result<(), Error> { map_blocks(inputs, range, checked, output, &|args, n, out| { write(&args, n, out); - let results = crate::slice::written(&mut out[..n]); + let results = unsafe { crate::slice::written(&mut out[..n]) }; if judged && !crate::slice::judged(&args, results, |x, r| kind(x, r).is_none()) { for (i, &r) in results.iter().enumerate() { if let Some(e) = kind(std::array::from_fn(|j| args[j].nth(i)), r) { return Err(e); } } } @@ -105,38 +107,32 @@ fn map_written( } /// `apply` on `inputs` over `range`, written to `output`: by rows where `by_rows` reads them so, and otherwise a part at a time. Where an input reads through scratch, not in place (`Read::in_place`), the part runs `CHUNK` items at a time to keep that scratch in cache. -fn map_blocks( - inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, checked: bool, output: &mut Buffer, +fn map_blocks( + inputs: Inputs<'_, '_>, range: std::ops::Range, checked: bool, output: &mut Output<'_, '_>, apply: &(dyn Fn([Arg<'_, F>; N], usize, &mut [std::mem::MaybeUninit]) -> Result<(), Kind> + Sync)) -> Result<(), Error> { - if output.dtype() != O::DTYPE { *output = Buffer::with_capacity(O::DTYPE, range.len()); } - let values = O::vec_mut(output).expect("kernel output has its selected dtype"); - values.clear(); let recorded = |result: Result<(), Kind>| result.map_err(|kind| crate::error::record(Error::new(kind, "numerical kernel argument is outside its domain"))).is_ok(); - if let Some((len, rows)) = by_rows::(inputs) { - crate::parallel::extended(values, range.len(), crate::CHUNK, &|part, slots| { - slots.fill(part.len(), |out| recorded(fill_rows(&rows, len, part.start + range.start, out, apply))) - }); - return Ok(()); + if let Inputs::Mapped(reads) = inputs && let Some((len, rows)) = by_rows::(reads) { + return output + .write(range.len(), &|part, slots| unsafe { slots.fill(part.len(), |out| recorded(fill_rows(&rows, len, part.start + range.start, out, apply))) }); } - crate::parallel::extended(values, range.len(), crate::CHUNK, &|part, slots| { + output.write(range.len(), &|part, slots| { let part = part.start + range.start..part.end + range.start; - let len = if inputs.iter().all(|input| input.in_place::(&part)) { part.len().max(1) } else { crate::CHUNK }; + let len = if inputs.in_place::(&part) { part.len().max(1) } else { crate::CHUNK }; let mut scratch: [Vec; N] = std::array::from_fn(|_| Vec::new()); part.clone().step_by(len).fold(true, |ok, at| { let run = at..(at + len).min(part.end); let n = run.len(); - let mut args = [Arg::Many(&[]); N]; - for ((arg, input), scratch) in args.iter_mut().zip(inputs).zip(scratch.iter_mut()) { - match input.part::(run.clone(), checked, scratch) { - Ok(read) => *arg = read, - Err(error) => { crate::error::record(error); return slots.fill(n, |out| { for slot in out { slot.write(O::FILL); } false }) & ok; } + let args = match inputs.part::(run, checked, &mut scratch) { + Ok(args) => args, + Err(error) => { + crate::error::record(error); + return unsafe { slots.fill(n, |out| { for slot in out { slot.write(O::FILL); } false }) } & ok; } - } - slots.fill(n, |out| recorded(apply(args, n, out))) & ok + }; + (unsafe { slots.fill(n, |out| recorded(apply(args, n, out))) }) & ok }) - }); - Ok(()) + }) } /// Rows shorter than this can read slower than blocks, which share each call's cost among more items. @@ -171,3 +167,11 @@ fn fill_rows( } result } + +pub(super) fn copied(inputs: Inputs<'_, '_>, range: std::ops::Range, checked: bool, output: &mut Output<'_, '_>) -> Result<(), Error> { + with_dtype!(output.dtype, T => map_values::(inputs, range, checked, false, output, |x| x[0], |_, _| None)) +} + +pub(super) fn constant(value: bool, n: usize, output: &mut Output<'_, '_>) -> Result<(), Error> { + output.write::(n, &|part, slots| slots.pairs(Arg::One(value), Arg::One(()), part.len(), |x, ()| (x, true))) +} diff --git a/src/operation/map/complex.rs b/kernels/src/operation/map/complex.rs similarity index 70% rename from src/operation/map/complex.rs rename to kernels/src/operation/map/complex.rs index 206739d..aaa8b41 100644 --- a/src/operation/map/complex.rs +++ b/kernels/src/operation/map/complex.rs @@ -1,16 +1,16 @@ use super::*; pub(super) fn map_complex( - object: &K, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, ops: &NativeOps<{ FUSED }>, - slice: Option, N, ()>>, judge: crate::domain::Variant, output: &mut Buffer) -> Result<(), Error> where - K: crate::facts::Formula, F: crate::float::Native + crate::buffer::Cast, num_complex::Complex: crate::buffer::Cast { + object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, ops: &NativeOps<{ FUSED }>, + slice: Option, N, ()>>, judge: crate::domain::Variant, output: &mut Output<'_, '_>) -> Result<(), Error> +where K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast + OutputItem, num_complex::Complex: crate::buffer::Cast + OutputItem { let value = |x| object.complex(ops, x).unwrap_or(num_complex::Complex::new(F::zero(), F::zero())); let invalid = |x, r| (!complex_valid::(ops, x, r)).then_some(Kind::Domain); if const { K::BOOLEAN } { map_values::, bool, N>( inputs, range, variant.checked, judge.checked, output, |x| value(x).re != F::zero(), |x, r| invalid(x, num_complex::Complex::new(if r { F::one() } else { F::zero() }, F::zero()))) - } else if output.dtype().is_real() { + } else if output.dtype.is_real() { map_values::, F, N>( inputs, range, variant.checked, judge.checked, output, |x| value(x).re, |x, r| invalid(x, num_complex::Complex::new(r, F::zero()))) } else { @@ -23,13 +23,12 @@ pub(super) fn map_complex( /// A formula's complex results from real arguments (`Formula::real_complex`). An item that it declines takes `Formula::complex` on its arguments as complex numbers. pub(super) fn map_real_complex( - object: &K, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, ops: &NativeOps<{ FUSED }>, - output: &mut Buffer) -> Result<(), Error> where K: crate::facts::Formula, F: crate::float::Native + crate::buffer::Cast, num_complex::Complex: Element -{ + object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, ops: &NativeOps<{ FUSED }>, output: &mut Output<'_, '_> +) -> Result<(), Error> where K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast, num_complex::Complex: OutputItem { map_blocks::, N>(inputs, range, variant.checked, output, &|args, n, out| { let nan = num_complex::Complex::new(F::nan(), F::nan()); if !crate::slice::mapped(&args, n, out, #[inline(always)] |x| object.real_complex(ops, x).map_or((nan, false), |z| (z, true))) { - for (i, z) in crate::slice::written(&mut out[..n]).iter_mut().enumerate() { + for (i, z) in unsafe { crate::slice::written(&mut out[..n]) }.iter_mut().enumerate() { let x = std::array::from_fn(|j| args[j].nth(i)); if object.real_complex(ops, x).is_none() { *z = object.complex(ops, x.map(|v| num_complex::Complex::new(v, F::zero()))).unwrap_or(nan); } } diff --git a/src/operation/map/integer.rs b/kernels/src/operation/map/integer.rs similarity index 68% rename from src/operation/map/integer.rs rename to kernels/src/operation/map/integer.rs index 6a31224..066a02f 100644 --- a/src/operation/map/integer.rs +++ b/kernels/src/operation/map/integer.rs @@ -1,8 +1,8 @@ use super::*; pub(super) fn map_integer( - object: &K, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, judge: crate::domain::Variant, - output: &mut Buffer) -> Result<(), Error> where K: crate::facts::Formula, I: crate::recipe::Overflowing { + object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, judge: crate::domain::Variant, + output: &mut Output<'_, '_>) -> Result<(), Error> where K: crate::facts::Formula, I: crate::recipe::Overflowing + OutputItem { let ops = crate::recipe::IntegerOps::::new(variant); let value = |x| object.integer(&ops, x).unwrap_or(I::FILL); if const { K::BOOLEAN } { diff --git a/kernels/src/operation/map/io.rs b/kernels/src/operation/map/io.rs new file mode 100644 index 0000000..c50d492 --- /dev/null +++ b/kernels/src/operation/map/io.rs @@ -0,0 +1,111 @@ +use super::*; +use crate::{buffer::Cast, parallel::Slots}; + +#[derive(Clone, Copy)] +pub(crate) enum Inputs<'a, 'b> { Mapped(&'a [crate::agreement::Read<'b>]), Ready(&'a [View<'b>]) } + +impl<'a, 'b> Inputs<'a, 'b> { + pub(crate) fn len(self) -> usize { match self { Self::Mapped(x) => x.len(), Self::Ready(x) => x.len() } } + pub(crate) fn selected(self, k: usize) -> Self { match self { Self::Mapped(x) => Self::Mapped(&x[k..k + 1]), Self::Ready(x) => Self::Ready(&x[k..k + 1]) } } + pub(crate) fn real(self) -> bool { + let real = |v: View<'_>| !matches!(v.dtype(), DType::Complex(_)); + match self { Self::Mapped(x) => x.iter().all(|r| r.sources.iter().all(|s| real(s.view))), Self::Ready(x) => x.iter().all(|&v| real(v)) } + } + pub(crate) fn in_place(self, range: &std::ops::Range) -> bool { + match self { Self::Mapped(x) => x.iter().all(|r| r.in_place::(range)), Self::Ready(x) => x.iter().all(|&v| F::of(v).is_some()) } + } + pub(crate) fn part<'s, F: Cast, const N: usize>( + self, range: std::ops::Range, checked: bool, scratch: &'s mut [Vec; N]) -> Result<[Arg<'s, F>; N], Error> where 'a: 's, 'b: 's { + let mut args = [Arg::Many(&[]); N]; + for (k, (arg, scratch)) in args.iter_mut().zip(scratch).enumerate() { + *arg = match self { + Self::Mapped(x) => x[k].part(range.clone(), checked, scratch)?, + Self::Ready(x) => { + let view = if x[k].len() == 1 { x[k] } else { x[k].slice(range.clone()) }; + match F::of(view) { + Some(items) => Arg::of(items), + None => { + scratch.clear(); + F::extend_cast(view, view.len(), checked, scratch)?; + Arg::of(scratch) + } + } + } + }; + } + Ok(args) + } +} + +/// Where a map appends its results: an owned buffer or the next slots of a caller's result. +pub enum MapOutput<'a, 'b> { + Buffer(&'a mut Buffer), F16(&'a mut Slots<'b, half::f16>), F32(&'a mut Slots<'b, f32>), F64(&'a mut Slots<'b, f64>), Bool(&'a mut Slots<'b, bool>), + U8(&'a mut Slots<'b, u8>), I16(&'a mut Slots<'b, i16>), I32(&'a mut Slots<'b, i32>), I64(&'a mut Slots<'b, i64>), + C32(&'a mut Slots<'b, num_complex::Complex32>), C64(&'a mut Slots<'b, num_complex::Complex64>), Char(&'a mut Slots<'b, char>) +} + +pub(crate) trait OutputItem: Element { fn slots<'s, 'b>(output: &'s mut MapOutput<'_, 'b>) -> Option<&'s mut Slots<'b, Self>>; } + +macro_rules! outputs { + ($($v:ident: $t:ty),+ $(,)?) => { + impl MapOutput<'_, '_> { + fn dtype(&self) -> DType { match self { Self::Buffer(b) => b.dtype(), $(Self::$v(_) => <$t>::DTYPE),+ } } + fn copy(&mut self, view: View<'_>) { + match self { Self::Buffer(b) => { b.extend_from(view, 0..view.len()); }, $(Self::$v(s) => s.copy(<$t>::of(view).expect("result conversion has the destination's dtype"))),+ } + } + pub(crate) fn fill(&mut self, n: usize) { + match self { + Self::Buffer(b) => with_dtype!(b.dtype(), T => { let v = T::vec_mut(b).expect("a buffer has its dtype"); v.resize(v.len() + n, T::FILL); }), + $(Self::$v(s) => { s.pairs(Arg::One(<$t>::FILL), Arg::One(()), n, |x, ()| (x, true)); }),+ + } + } + } + $(impl OutputItem for $t { + fn slots<'s, 'b>(output: &'s mut MapOutput<'_, 'b>) -> Option<&'s mut Slots<'b, Self>> { + match output { MapOutput::$v(s) => Some(s), _ => None } + } + } + impl<'a, 'b> From<&'a mut Slots<'b, $t>> for MapOutput<'a, 'b> { + fn from(slots: &'a mut Slots<'b, $t>) -> Self { MapOutput::$v(slots) } + })+ + }; +} +outputs!(F16: half::f16, F32: f32, F64: f64, Bool: bool, U8: u8, I16: i16, I32: i32, I64: i64, + C32: num_complex::Complex32, C64: num_complex::Complex64, Char: char); + +pub(crate) struct Output<'a, 'b> { pub target: MapOutput<'a, 'b>, pub dtype: DType, pub checked: bool } + +impl Output<'_, '_> { + /// The block body is already erased here: output conversion never instantiates a formula at a second computing type. + pub(crate) fn write(&mut self, n: usize, apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>) -> bool + Sync)) -> Result<(), Error> { + let checked = self.checked; + if let MapOutput::Buffer(buffer) = &mut self.target { + if let Some(values) = O::vec_mut(buffer) { crate::parallel::extended(values, n, crate::CHUNK, apply); } else { + with_dtype!(buffer.dtype(), T => { + let values = T::vec_mut(buffer).expect("a buffer has its dtype"); + crate::parallel::extended(values, n, crate::CHUNK, &|part, slots| { + converted::(&mut slots.into(), part, checked, apply).map_err(crate::error::record).is_ok() + }); + }); + } + return Ok(()); + } + if let Some(slots) = O::slots(&mut self.target) { apply(0..n, slots); return Ok(()); } + converted(&mut self.target, 0..n, checked, apply) + } +} + +fn converted( + target: &mut MapOutput<'_, '_>, range: std::ops::Range, checked: bool, apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>) -> bool + Sync) +) -> Result<(), Error> { + let mut scratch = Vec::::new(); + for at in (range.start..range.end).step_by(crate::CHUNK) { + let n = crate::CHUNK.min(range.end - at); + scratch.clear(); + Slots::append(&mut scratch, n, |slots| apply(at..at + n, slots)); + match Buffer::cast_results(O::view(&scratch), target.dtype(), checked) { + Ok(buffer) => target.copy(buffer.view()), Err(error) => { target.fill(range.end - at); return Err(error); } + } + } + Ok(()) +} diff --git a/src/operation/map/real.rs b/kernels/src/operation/map/real.rs similarity index 50% rename from src/operation/map/real.rs rename to kernels/src/operation/map/real.rs index f614440..e423e70 100644 --- a/src/operation/map/real.rs +++ b/kernels/src/operation/map/real.rs @@ -1,12 +1,12 @@ use super::*; pub(super) fn map_real( - object: &K, variant: crate::domain::Variant, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, ops: &NativeOps<{ FUSED }>, repair: bool, - judge: crate::domain::Variant, output: &mut Buffer) -> Result<(), Error> where K: crate::facts::Formula, - F: crate::float::Native + crate::buffer::Cast + Source + crate::tolerant::Banded, Dst: Element + crate::Float { + object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, ops: &NativeOps<{ FUSED }>, repair: bool, + judge: crate::domain::Variant, output: &mut Output<'_, '_>) -> Result<(), Error> where K: crate::facts::Formula, + F: crate::Native + crate::buffer::Cast + Source + crate::tolerant::Banded, Dst: OutputItem + crate::Float { if !K::BOOLEAN && repair && object.repair_routine().is_some() { return map_blocks::(inputs, range, variant.checked, output, &|args, n, out| { - let ok = real_slice(object, ops, args, n, out); + let ok = crate::facts::real_slice(object, ops, args, n, out); if judge.checked && K::FAILS_NAN && !ok { Err(Kind::Domain) } else { Ok(()) } }); } @@ -22,13 +22,3 @@ pub(super) fn map_real( inputs, range, variant.checked, judge.checked, output, #[inline(always)] |x| Source::::read(object.real(ops, x).unwrap_or(F::zero())), |x, r| verdict(object, judge, ops, x, F::narrow(r.wide()))) } - -/// A real formula's slice body and rare-item repairs, without recording errors from intermediate values. -pub(crate) fn real_slice, F: crate::float::Native + Source, O: crate::Float, const N: usize>( - object: &K, ops: &NativeOps<{ FUSED }>, args: [Arg<'_, F>; N], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { - let body = |x| object.real(ops, x).expect("a resolved object has a real body"); - if let Some(repair) = object.repair_routine() { - let needs = |x| object.emit_repair(ops, x).unwrap_or(false); - crate::slice::two_pass(args, n, out, |x| (!needs(x)).then(|| body(x)), |x| if needs(x) { repair.apply(x[0]) } else { body(x) }) - } else { crate::slice::each(args, n, out, #[inline(always)] |x| Source::::read(body(x))) } -} diff --git a/src/operation/product.rs b/kernels/src/operation/product.rs similarity index 98% rename from src/operation/product.rs rename to kernels/src/operation/product.rs index 5020be7..5591629 100644 --- a/src/operation/product.rs +++ b/kernels/src/operation/product.rs @@ -129,12 +129,12 @@ fn product_typed>( let mut result = Vec::with_capacity(plan.layout.count()); let overflow = || if variant.checked { Err(Error::new(Kind::Overflow, "product does not fit its integer result")) } else { Ok(false) }; let mut multiply = |a: Mapped<'_, T>, b: Mapped<'_, T>| -> Result { + let start = result.len(); + result.resize(start + m * n, P::Out::default()); if T::DTYPE.is_integral() { let (mut ab, mut bb) = (Vec::new(), Vec::new()); - let start = result.len(); - result.resize(start + m * n, P::Out::default()); if !pair.checked_into(a.read(0, m * k, &mut ab), b.read(0, k * n, &mut bb), matrix.size, &mut result[start..]) { return overflow(); } - } else { result.extend(pair.inner_mapped(a, b, [Orientation::Normal; 2], matrix.size)); } + } else { pair.inner_mapped_into(a, b, [Orientation::Normal; 2], matrix.size, &mut result[start..]); } Ok(true) }; if let [Some((a, a0, a_steps)), Some((b, b0, b_steps))] = &walks { diff --git a/py/Cargo.toml b/py/Cargo.toml index 0800787..007cdff 100644 --- a/py/Cargo.toml +++ b/py/Cargo.toml @@ -16,6 +16,7 @@ native = ["rustymath/default"] [dependencies] rustymath.workspace = true rustymath-buffer.workspace = true +rustymath-timing.workspace = true pyo3 = { workspace = true, features = ["extension-module", "num-bigint"] } num-bigint = ">=0.4.6, <0.5" num-rational = ">=0.4.2, <1" diff --git a/py/src/functions.rs b/py/src/functions.rs index 0de14a2..f45a67d 100644 --- a/py/src/functions.rs +++ b/py/src/functions.rs @@ -57,9 +57,7 @@ fn axes_of(axis: Option<&Axes>, layout: &Layout) -> PyResult> { fn computes(function: &str, op: Op, arity: usize, dtype: DType, only: Option<&[DType]>) -> PyResult { match only { None => op.computing_dtype(arity, dtype), - Some(allowed) => { - dtype.computes_in(rustymath::buffer::Width::I64, rustymath::operation::numpy_float(dtype), |at| allowed.contains(&at) && op.supports(arity, at)) - } + Some(allowed) => dtype.computes_in(rustymath::buffer::Width::I64, dtype.numpy_float(), |at| allowed.contains(&at) && op.supports(arity, at)), }.ok_or_else(|| missing(function, dtype)) } /// The kernel of `op` on arguments of types `types`, or of its partial in argument `side` (`Kernel::slope`), whose last type is then the result's. @@ -117,7 +115,7 @@ fn totals(function: &str, op: Op, x: ArrayView<'_>, axis: Option<&Axes>, only: O let kernel = select(function, op, None, view.dtype(), &[view.dtype(); 2], true, only)?; let mut buffer = Buffer::with_capacity(kernel.fold_dtype(view.dtype(), axis.map_or(view.len(), |a| shape[a]), None), next.count()); let order = kernel - .fold_read(shape, axis, rustymath::operation::Seed::Identity, rustymath::operation::Direction::Right, settings(false), view.read(), &mut buffer) + .fold_read(shape, axis, rustymath::reduce::Seed::Identity, rustymath::reduce::Direction::Right, settings(false), view.read(), &mut buffer) .map_err(py_err)?; out = Some(Core::new(next, buffer.ordered(order)).map_err(py_err)?); } @@ -142,7 +140,7 @@ fn scan<'py>(py: Python<'py>, function: &str, op: Op, x: &Bound<'py, PyAny>, axi let (dims, layout, _) = along_axis(x.layout(), view.len(), axis)?; let kernel = select(function, op, None, view.dtype(), &[view.dtype(); 2], true, only)?; let mut buffer = Buffer::with_capacity(kernel.scan_dtype(false), view.len()); - released(py, || kernel.scan(dims, rustymath::operation::Seed::Identity, rustymath::operation::Direction::Left, settings(false), view.read(), &mut buffer)) + released(py, || kernel.scan(dims, rustymath::reduce::Seed::Identity, rustymath::reduce::Direction::Left, settings(false), view.read(), &mut buffer)) .map_err(py_err)?; give_items(py, layout, buffer, false) } @@ -532,9 +530,9 @@ fn logical_scan<'py>(py: Python<'py>, x: &Bound<'py, PyAny>, table: [[bool; 2]; let kernel = rustymath::operation::Kernel::new::<_, 2>(rustymath::Bitwise(code), rustymath::domain::Variant::checked(DType::Boolean)); let start = start.map(|s| Buffer::Booleans(vec![s])); let start = start.as_ref().map(|s| ArrayView::compact(&Layout::EMPTY, s.view())); - let seed = start.as_ref().map_or(rustymath::operation::Seed::Identity, |s| rustymath::operation::Seed::Initial(s.read())); + let seed = start.as_ref().map_or(rustymath::reduce::Seed::Identity, |s| rustymath::reduce::Seed::Initial(s.read())); let mut buffer = Buffer::with_capacity(kernel.scan_dtype(start.is_some()), 0); - released(py, || kernel.scan(dims, seed, rustymath::operation::Direction::Left, settings(false), input.read(), &mut buffer)).map_err(py_err)?; + released(py, || kernel.scan(dims, seed, rustymath::reduce::Direction::Left, settings(false), input.read(), &mut buffer)).map_err(py_err)?; give_items(py, layout, buffer, false) } @@ -551,9 +549,9 @@ fn _accumulate<'py>( let (dims, layout, _) = along_axis(x.layout(), view.len(), axis)?; let (function, mode) = function; let kernel = (native_binary(function)?.kernel)(view.dtype(), mode)?; - let seed = start.as_ref().map_or(rustymath::operation::Seed::Identity, |s| rustymath::operation::Seed::Initial(s.read())); + let seed = start.as_ref().map_or(rustymath::reduce::Seed::Identity, |s| rustymath::reduce::Seed::Initial(s.read())); let mut buffer = Buffer::with_capacity(kernel.scan_dtype(start.is_some()), view.len()); - released(py, || kernel.scan(dims, seed, rustymath::operation::Direction::Left, settings(mode), view.read(), &mut buffer)).map_err(py_err)?; + released(py, || kernel.scan(dims, seed, rustymath::reduce::Direction::Left, settings(mode), view.read(), &mut buffer)).map_err(py_err)?; give_items(py, layout, buffer, false) } @@ -587,8 +585,7 @@ fn _sliding<'py>(py: Python<'py>, function: NativeFunction<'_>, x: &Bound<'py, P let dims = [lanes.outer * (lanes.len + 1 - width), width, lanes.inner]; let mut buffer = Buffer::with_capacity(kernel.fold_dtype(view.dtype(), width, None), layout.count()); let order = released(py, || { - kernel.fold_read( - &dims, Some(1), rustymath::operation::Seed::Identity, rustymath::operation::Direction::Right, settings(mode), input.read(), &mut buffer) + kernel.fold_read(&dims, Some(1), rustymath::reduce::Seed::Identity, rustymath::reduce::Direction::Right, settings(mode), input.read(), &mut buffer) }).map_err(py_err)?; give_items(py, layout, buffer.ordered(order), false) } diff --git a/py/src/linalg.rs b/py/src/linalg.rs index 618d542..29af274 100644 --- a/py/src/linalg.rs +++ b/py/src/linalg.rs @@ -6,7 +6,6 @@ use rustymath::array::{Array as Core, ArrayView}; use rustymath::buffer::{Buffer, DType, FloatWidth, Width}; use rustymath::error::{Error, Kind}; use rustymath::linalg::{self, Stacked}; -use rustymath::operation::numpy_float; use rustymath_buffer::py_err; use crate::array::give; @@ -17,7 +16,7 @@ use crate::functions::{NativeFunction, native_binary}; fn on_stacked<'py, const N: usize, R: Send>( py: Python<'py>, args: [&Bound<'py, PyAny>; N], f: impl for<'c> FnOnce(Stacked, [ArrayView<'c>; N]) -> Result + Send) -> PyResult { let call = convert(&args)?; - let (s, views) = (Stacked::new(numpy_float(call.args[0].view().dtype())), std::array::from_fn(|k| call.args[k].view())); + let (s, views) = (Stacked::new(call.args[0].view().dtype().numpy_float()), std::array::from_fn(|k| call.args[k].view())); released(py, || f(s, views)).map_err(py_err) } diff --git a/py/src/timing.rs b/py/src/timing.rs index 29a197b..3c69388 100644 --- a/py/src/timing.rs +++ b/py/src/timing.rs @@ -1,36 +1,40 @@ -//! The machine's timing lock, copy times and cost lines for Python, as `rustymath::timing` describes. +//! The machine's timing lock, copy times and cost lines for Python, as `rustymath_timing` describes. use pyo3::prelude::*; -/// `rustymath::timing`'s `$f` at the float type whose items take `$bytes` bytes, f32 or f64. Returns a ValueError from the calling function for any other size. +/// `rustymath_timing`'s `$f` at the float type whose items take `$bytes` bytes, f32 or f64. Returns a ValueError from the calling function for any other size. macro_rules! at_width { ($bytes:expr, $f:ident($($arg:expr),*)) => { match $bytes { - 4 => rustymath::timing::$f::($($arg),*), 8 => rustymath::timing::$f::($($arg),*), + 4 => rustymath_timing::$f::($($arg),*), 8 => rustymath_timing::$f::($($arg),*), bytes => return Err(pyo3::exceptions::PyValueError::new_err(format!("no copy time for {bytes}-byte items"))), } }; } -/// A hold on the timing lock that `timing` names, taken exclusively when a `with` block starts and released when it ends, as `rustymath::timing::hold_at` takes it. While the block runs, the calling thread runs rustymath's calls on one thread, and on macOS it asks for user-interactive QoS. `pauses` gives the pauses, in seconds, before each retry of a case over its budget, `column` which of a budget file's two native budgets this machine compares costs with, or `None` on a machine with no budgets, as `rustymath::timing::NATIVE_COLUMN` gives it, and `checks_budgets` whether a case still over its budget fails, as `rustymath::timing::checks_budgets` decides. `warm` keeps the thread busy after each pause. `budgets` reads a budget file, and `copy`, `peak_held`, `footprint` and `line` give what the speed test measures and prints. +/// A hold on the timing lock that `timing` names, taken exclusively, or `shared` for a speed test's ordinary run, when a `with` block starts and released when it ends, as `rustymath_timing::hold_at` takes it. While the block runs, the calling thread runs rustymath's calls on one thread, and on macOS an exclusive hold asks for user-interactive QoS. `pauses` gives the pauses, in seconds, before each retry of a case over its budget in a calibration run, `column` which of a budget file's two native budgets this machine compares costs with, or `None` on a machine with no budgets, as `rustymath_timing::NATIVE_COLUMN` gives it, and `checks_budgets` whether a case still over its budget fails, as `rustymath_timing::checks_budgets` decides. `measures` names what a budget file counts for each item, and `count_margin` is the usual margin over a recorded count. `warm` keeps the thread busy after each pause. `budgets` reads a budget file, and `copy`, `peak_held`, `footprint`, `instructions` and `line` give what the speed test measures and prints. #[pyclass(module = "rustymath")] -struct Timing { lock: String, hold: Option } +struct Timing { lock: String, shared: bool, hold: Option } #[pymethods] impl Timing { #[classattr] - fn pauses() -> Vec { rustymath::timing::RETRY_PAUSES.to_vec() } + fn pauses() -> Vec { rustymath_timing::RETRY_PAUSES.to_vec() } #[classattr] - fn column() -> Option { rustymath::timing::NATIVE_COLUMN } + fn column() -> Option { rustymath_timing::NATIVE_COLUMN } #[classattr] - fn checks_budgets() -> bool { rustymath::timing::checks_budgets() } - /// Keeps the calling thread busy for 0.1 s after a retry's pause, as `rustymath::timing::warm` does. + fn checks_budgets() -> bool { rustymath_timing::checks_budgets() } + #[classattr] + fn measures() -> Vec<&'static str> { rustymath_timing::MEASURES.to_vec() } + #[classattr] + fn count_margin() -> f64 { rustymath_timing::COUNT_MARGIN } + /// Keeps the calling thread busy for 0.1 s after a retry's pause, as `rustymath_timing::warm` does. #[staticmethod] - fn warm() { rustymath::timing::warm() } - /// One copy's time for each of `items` items of `bytes` bytes, in seconds, and the line that the speed tests print for it, as `rustymath::timing::copy_time` and `copy_line` give them. + fn warm() { rustymath_timing::warm() } + /// One copy's time for each of `items` items of `bytes` bytes, in seconds, and the line that the speed tests print for it, as `rustymath_timing::copy_time` and `copy_line` give them. #[staticmethod] fn copy(bytes: usize, items: usize) -> PyResult<(f64, String)> { let t = at_width!(bytes, copy_time(items)); - Ok((t.seconds, rustymath::timing::copy_line(&t))) + Ok((t.seconds, rustymath_timing::copy_line(&t))) } /// What `run` gives when called, and the most bytes that it held at once, as `rustymath::memory::peak_held` counts them. #[staticmethod] @@ -38,35 +42,48 @@ impl Timing { let (result, held) = rustymath::memory::peak_held(|| run.call0()); Ok((result?.unbind(), held)) } - /// A case's footprint in bytes and the time for each item of a copy of it, measured now, as `rustymath::timing::footprint` gives them for `items` items of `bytes` bytes and `held` bytes held at once, or `None` for a footprint within one core's cache. + /// A case's footprint in bytes and the time for each item of a copy of it, measured now, as `rustymath_timing::footprint` gives them for `items` items of `bytes` bytes and `held` bytes held at once, or `None` for a footprint within one core's cache. #[staticmethod] fn footprint(bytes: usize, items: usize, held: usize) -> PyResult> { Ok(at_width!(bytes, footprint(items, held)).map(|f| (f.bytes, f.seconds))) } - /// A case's line in a speed test's log, from its time for each item in `seconds` and the `footprint` that `footprint` gives, as `rustymath::timing::line` gives it. + /// A case's line in a speed test's log, from its time for each item in `seconds`, the `footprint` that `footprint` gives and its `counts` for each item in the order of `measures`, as `rustymath_timing::line` gives it. + #[staticmethod] + #[pyo3(signature = (section, code, cost, budget, seconds, footprint=None, counts=(None, None)))] + fn line(section: &str, code: &str, cost: f64, budget: &str, seconds: f64, footprint: Option<(usize, f64)>, counts: (Option, Option)) -> String { + let footprint = footprint.map(|(bytes, seconds)| rustymath_timing::Footprint { bytes, seconds }); + rustymath_timing::line(section, code, cost, budget, seconds, footprint, [counts.0, counts.1]) + } + /// Each case of `text`, a budget file such as `tests/speed_python.txt`, as `(section, code, budget, counts)`. `budget` is its `Budget` in the column that this machine checks, or `None` on a machine with no budgets, as `rustymath_timing::budgeted` gives them. `counts` holds the `Count` that the case records in that column for each of `measures`, or `None`. #[staticmethod] - #[pyo3(signature = (section, code, cost, budget, seconds, footprint=None))] - fn line(section: &str, code: &str, cost: f64, budget: &str, seconds: f64, footprint: Option<(usize, f64)>) -> String { - rustymath::timing::line(section, code, cost, budget, seconds, footprint.map(|(bytes, seconds)| rustymath::timing::Footprint { bytes, seconds })) + fn budgets(text: &str) -> PyResult, Vec>)>> { + let column = rustymath_timing::NATIVE_COLUMN; + let cases = rustymath_timing::budgeted(text, column).map_err(pyo3::exceptions::PyValueError::new_err)?; + Ok(cases.into_iter().map(|(case, budget)| { + let counts = rustymath_timing::MEASURES.iter().map(|m| case.count(m, column).map(Count)).collect(); + (case.section.into(), case.code.into(), budget.map(Budget), counts) + }).collect()) } - /// Each case of `text`, a budget file such as `tests/speed_python.txt`, as `(section, code, budget)`, where `budget` is its `Budget` in the column that this machine checks, or `None` on a machine with no budgets, as `rustymath::timing::budgeted` gives them. + /// What `run` gives when called, and the instructions retired while it ran, as `rustymath_timing::instructions` counts them, or `None` where this machine can't count them. #[staticmethod] - fn budgets(text: &str) -> PyResult)>> { - let cases = rustymath::timing::budgeted(text, rustymath::timing::NATIVE_COLUMN).map_err(pyo3::exceptions::PyValueError::new_err)?; - Ok(cases.into_iter().map(|(section, code, budget)| (section.into(), code.into(), budget.map(Budget))).collect()) + fn instructions(run: &Bound<'_, PyAny>) -> PyResult<(Py, Option)> { + let (result, count) = rustymath_timing::instructions(|| run.call0()); + Ok((result?.unbind(), count)) } - /// Waits for the lock without holding the GIL, as long as `rustymath::timing::hold_at` waits. + /// Waits for the lock without holding the GIL, as long as `rustymath_timing::hold_at` waits, unless `try_timing` already holds it. fn __enter__(mut slf: PyRefMut<'_, Self>) -> PyResult> { - let lock = slf.lock.clone(); - slf.hold = Some(slf.py().detach(|| rustymath::timing::hold_at(&lock, false))?); + if slf.hold.is_none() { + let (lock, shared) = (slf.lock.clone(), slf.shared); + slf.hold = Some(slf.py().detach(|| rustymath_timing::hold_at(&lock, shared).map(rustymath_timing::Hold::on_one_thread))?); + } Ok(slf) } fn __exit__(&mut self, _kind: &Bound<'_, PyAny>, _value: &Bound<'_, PyAny>, _traceback: &Bound<'_, PyAny>) { self.hold = None; } } -/// A case's budget in the column that this machine checks, with any hold, as `rustymath::timing::Budget` gives it. `over(cost)` says whether a cost is over it, `stale(cost)` whether the hold of a gap has outlived the gap, and `str()` gives it as a speed test's line does. +/// A case's budget in the column that this machine checks, with any hold, as `rustymath_timing::Budget` gives it. `over(cost)` says whether a cost is over it, `stale(cost)` whether the hold of a gap has outlived the gap, and `str()` gives it as a speed test's line does. #[pyclass(module = "rustymath", frozen)] -struct Budget(rustymath::timing::Budget); +struct Budget(rustymath_timing::Budget); #[pymethods] impl Budget { @@ -75,13 +92,32 @@ impl Budget { fn __str__(&self) -> String { self.0.to_string() } } -/// A hold on the machine's timing lock, for `with rustymath.timing():` around a timing, or on the lock whose file is `lock`, as a test of the protocol holds a file of its own. +/// A count that a budget file records for each item of a case, in the column that this machine checks, as `rustymath_timing::Count` gives it. `value` is the count, and `over(count, margin)` says whether a count is more than `margin` times it allows, as `rustymath_timing::Count::over` decides. +#[pyclass(module = "rustymath", frozen)] +struct Count(rustymath_timing::Count); + +#[pymethods] +impl Count { + #[getter] + fn value(&self) -> f64 { self.0.value } + fn over(&self, count: f64, margin: f64) -> bool { self.0.over(count, margin) } +} + +/// A hold on the machine's timing lock, for `with rustymath.timing():` around a timing, or on the lock whose file is `lock`, as a test of the protocol holds a file of its own. A `shared` hold is a speed test's ordinary run, which times beside builds and other tests. #[pyfunction] -#[pyo3(signature = (lock=None))] -fn timing(lock: Option) -> Timing { Timing { lock: lock.unwrap_or_else(|| rustymath::timing::LOCK.into()), hold: None } } +#[pyo3(signature = (lock=None, shared=false))] +fn timing(lock: Option, shared: bool) -> Timing { Timing { lock: lock.unwrap_or_else(|| rustymath_timing::LOCK.into()), shared, hold: None } } + +/// The machine's timing lock, held exclusively, where no build, test run or other timing holds it, or else `None` at once, as `rustymath_timing::try_exclusive` takes it. A `with` block around the hold keeps it, and the block's end releases it. A speed test's ordinary run uses it to confirm a case over its budget, after its shared hold ends. +#[pyfunction] +fn try_timing() -> PyResult> { + Ok(rustymath_timing::try_exclusive()?.map(|hold| Timing { lock: rustymath_timing::LOCK.into(), shared: false, hold: Some(hold) })) +} pub fn add(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - m.add_function(wrap_pyfunction!(timing, m)?) + m.add_class::()?; + m.add_function(wrap_pyfunction!(timing, m)?)?; + m.add_function(wrap_pyfunction!(try_timing, m)?) } diff --git a/pyproject.toml b/pyproject.toml index a1caa16..88ab320 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,13 +36,11 @@ module-name = "rustymath._core" macos-deployment-target = "15.0" [tool.uv] -cache-keys = [{ file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "py/Cargo.toml" }] +cache-keys = [{ file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "core/Cargo.toml" }, { file = "kernels/Cargo.toml" }, { file = "py/Cargo.toml" }] [tool.pytest.ini_options] testpaths = ["tests"] session_timeout = 30 -# The timing suite runs only when named: `pytest -q tests/test_speed.py`. -addopts = "--ignore=tests/test_speed.py" asyncio_default_fixture_loop_scope = "function" [tool.nbdev] diff --git a/python/rustymath/__init__.py b/python/rustymath/__init__.py index 3873dd4..bc1612c 100644 --- a/python/rustymath/__init__.py +++ b/python/rustymath/__init__.py @@ -36,7 +36,7 @@ betaincinv, binom, binom_dk, binom_dn, cbrt, ceil, cis, cos, cosh, cumprod, cumsum, digamma, distribution, divide, equal, erf, erfc, erfcinv, erfinv, exp, expm1, factorial, floor, gamma, gammainc, gammaincc, gammaincinv, gammaln, gcd, greater, greater_equal, hypot, lambertw, lcm, less, less_equal, log, log10, log1p, log2, logb, loggamma, logical_and, logical_or, logical_scan, logical_xor, logical_xnor, max, maximum, min, minimum, minmax, multiply, not_equal, number_theory, polygamma, power, - prod, remainder, root, set_threads, sign, sin, sinh, sqrt, sqrt1mx2, sqrt1px2, sqrtx2m1, subtract, sum, tan, tanh, threads, thread_setting, timing, explain) + prod, remainder, root, set_threads, sign, sin, sinh, sqrt, sqrt1mx2, sqrt1px2, sqrtx2m1, subtract, sum, tan, tanh, threads, thread_setting, timing, try_timing, explain) from .create import arange, finfo, full, full_like, iinfo, issubdtype, linspace, logspace, ones, ones_like, result_type, zeros, zeros_like from ._core import clip, conj, imag, isfinite, isinf, isnan, real, rint, trunc from ._core import bitwise_and, bitwise_or, bitwise_xor, bitwise_not, left_shift, right_shift, floor_divide, round, degrees, radians diff --git a/src/array.rs b/src/array.rs index 576d1e3..5514ade 100644 --- a/src/array.rs +++ b/src/array.rs @@ -5,58 +5,9 @@ use crate::error::{Error, Kind}; use crate::gather::{Steps, Writes}; use crate::layout::{Keys, Lanes, Layout, element_count}; use crate::operation::{Kernel, Settings, WriteRoot}; -use std::borrow::Cow; +pub use rustymath_core::array::ArrayView; use std::sync::Arc; -/// Sources lent for one operation. The plan carries positions, never data or a materialization cache. -#[derive(Clone)] -pub struct ArrayView<'a> { layout: &'a Layout, sources: Vec>, plan: ReadPlan } - -impl<'a> ArrayView<'a> { - pub fn new(layout: &'a Layout, sources: Vec>, plan: ReadPlan) -> Result { - if layout.count() != plan.len() { return Err(Error::new(Kind::Length, "read plan doesn't fit its layout")); } - Ok(Self { layout, sources, plan }) - } - pub fn compact(layout: &'a Layout, storage: View<'a>) -> Self { - Self { layout, sources: vec![Binding::full(storage)], plan: ReadPlan::input(0, storage.len()) } - } - pub fn layout(&self) -> &'a Layout { self.layout } - pub fn dtype(&self) -> DType { self.sources[0].view.dtype() } - pub fn len(&self) -> usize { self.plan.len() } - pub fn is_empty(&self) -> bool { self.plan.is_empty() } - pub fn read(&self) -> Read<'_> { Read { sources: &self.sources, plan: &self.plan } } - pub fn plan(&self) -> &ReadPlan { &self.plan } - pub fn sources(&self) -> &[Binding<'a>] { &self.sources } - /// Changes metadata, retaining the logical sequence. - pub fn with_layout<'b>(&self, layout: &'b Layout) -> Result, Error> where 'a: 'b { - ArrayView::new(layout, self.sources.clone(), self.plan.clone()) - } - /// Composes a call's positional read. Missing positions read the prototype (`Missing::prototype`). - pub fn remapped(&self, mapping: Mapping, items: usize) -> Self { - let mut sources = self.sources.clone(); - let missing = Missing::prototype(self.dtype(), &mut sources); - Self { layout: self.layout, sources, plan: self.plan.remapped(mapping, items, missing) } - } - pub fn contiguous(&self) -> Option> { - if self.is_empty() { return Some(self.sources[0].view.slice(0..0)); } - let (source, range) = self.plan.contiguous(0..self.len())?; - self.sources.get(source)?.borrow(range) - } - pub fn affine(&self) -> Option<(View<'a>, usize, Vec)> { - let (source, base, strides) = self.plan.affine(self.layout.shape())?; - let (view, base) = self.sources.get(source)?.affine(base, &strides, self.layout.shape())?; - Some((view, base, strides)) - } - /// Borrows a contiguous read at its dtype, otherwise prepares exactly these logical items. - pub fn cast(&self, dtype: DType) -> Result, Error> { - if dtype == self.dtype() && let Some(view) = self.contiguous() { return Ok(Items::Borrowed(view)); } - let mut buffer = Buffer::with_capacity(dtype, self.len()); - self.read().read_into(0..self.len(), dtype, true, &mut buffer)?; - Ok(Items::Owned(buffer)) - } - pub fn items(&self) -> Result, Error> { Ok(self.cast(T::DTYPE)?.items()) } -} - /// A value keeps its sources alive; reading a view never replaces its plan or caches a gathered copy. #[derive(Clone, Debug)] pub struct Array { layout: Layout, sources: Arc<[Arc]>, plan: ReadPlan } @@ -71,7 +22,8 @@ impl Array { pub fn set_names(&mut self, names: Vec>>) -> Result<(), Error> { self.layout = self.layout.clone().with_names(names)?; Ok(()) } pub fn set_keys(&mut self, keys: Vec>>) -> Result<(), Error> { self.layout = self.layout.clone().with_keys(keys)?; Ok(()) } pub fn view(&self) -> ArrayView<'_> { - ArrayView { layout: &self.layout, sources: self.sources.iter().map(|s| Binding::full(s.view())).collect(), plan: self.plan.clone() } + ArrayView::new(&self.layout, self.sources.iter().map(|s| Binding::full(s.view())).collect(), self.plan.clone()) + .expect("an array's plan fits its layout") } pub fn dtype(&self) -> DType { self.sources[0].dtype() } pub fn len(&self) -> usize { self.plan.len() } @@ -137,12 +89,12 @@ impl Array { /// Joins arrays of one dtype along an axis, retaining the usual layout rules. pub fn concatenate(arrays: &[ArrayView<'_>], axis: usize) -> Result { let Some(source) = arrays.first() else { return Err(Error::new(Kind::Length, "concatenate needs an array")) }; - let mut layout = source.layout.clone(); + let mut layout = source.layout().clone(); for view in &arrays[1..] { if view.dtype() != source.dtype() { return Err(Error::new(Kind::Domain, "concatenate items must have one dtype")); } - layout = crate::structure::catenate(&layout, view.layout, axis)?; + layout = crate::structure::catenate(&layout, view.layout(), axis)?; } - let lanes = arrays.iter().map(|a| Lanes::new(a.layout.shape(), axis)).collect::, _>>()?; + let lanes = arrays.iter().map(|a| Lanes::new(a.layout().shape(), axis)).collect::, _>>()?; let lens: Vec = lanes.iter().map(|l| l.len * l.inner).collect(); let (outer, stride) = (lanes[0].outer, lens.iter().sum::()); let buffer = with_dtype!(source.dtype(), T => { @@ -160,8 +112,8 @@ impl Array { pub fn choose(condition: ArrayView<'_>, x: ArrayView<'_>, y: ArrayView<'_>) -> Result { if condition.dtype() != DType::Boolean { return Err(Error::new(Kind::Domain, "condition must be Boolean")); } if x.dtype() != y.dtype() { return Err(Error::new(Kind::Domain, "chosen items must have one dtype")); } - let pair = Agreement::new(x.layout, y.layout)?; - let whole = Agreement::onto(&pair.layout, condition.layout)?; + let pair = Agreement::new(x.layout(), y.layout())?; + let whole = Agreement::onto(&pair.layout, condition.layout())?; let x = x.remapped(pair.left, pair.len).remapped(whole.left.clone(), whole.len); let y = y.remapped(pair.right, pair.len).remapped(whole.left, whole.len); let condition = condition.remapped(whole.right, whole.len); @@ -170,7 +122,7 @@ impl Array { Self::new(whole.layout, buffer) } pub fn broadcasted(source: ArrayView<'_>, shape: &[usize]) -> Result { - let agreement = Agreement::new(&Layout::from(shape), source.layout)?; + let agreement = Agreement::new(&Layout::from(shape), source.layout())?; if agreement.layout.shape() != shape { return Err(Error::new(Kind::Length, "source doesn't broadcast to this shape")); } let source = source.remapped(agreement.right, agreement.len); let buffer = source.cast(source.dtype())?; @@ -178,7 +130,7 @@ impl Array { } /// Adjacent differences read two selections through the ordinary kernel. Integer differences wrap at the input width. pub fn differences(source: ArrayView<'_>, axis: usize) -> Result { - let layout = source.layout; + let layout = source.layout(); let &size = layout.shape().get(axis).ok_or_else(|| Error::new(Kind::Index, "difference axis is out of range"))?; if source.dtype() == DType::Character { return Err(Error::new(Kind::Domain, "differences need numerical items")); } let n = size.saturating_sub(1); diff --git a/src/lib.md b/src/lib.md index e96f48c..a93270b 100644 --- a/src/lib.md +++ b/src/lib.md @@ -36,14 +36,14 @@ let mut squares = Vec::new(); Pow.many(Arg::Many(&[1.5, 2.0, 3.0]), Arg::One(2.0), 3, &mut squares); // squares: [2.25, 4.0, 9.0] ``` -`facts::Formula` holds an object's numerical and kind rules. `Formula::resolve` selects a working dtype and returns an `operation::Kernel` for callers choosing dtypes at run time. Its `map`, `fold_read`, `fold_pieces` and `scan` methods consume shared `agreement::Read` plans. `Kernel::real`, `complex` and `exact` evaluate scalar arguments through the same object. +`facts::Formula` holds an object's numerical and kind rules. `Kernel::resolve` selects a working dtype and returns an `operation::Kernel` for callers choosing dtypes at run time. Its `map`, `fold_read`, `fold_pieces` and `scan` methods consume shared `agreement::Read` plans. `Kernel::real`, `complex` and `exact` evaluate scalar arguments through the same object. ```rust -use rustymath::{Exp, domain::Variant, facts::Formula, buffer::{DType, FloatWidth, Width}}; +use rustymath::{Exp, domain::Variant, operation::Kernel, buffer::{DType, FloatWidth, Width}}; let double = DType::Float(FloatWidth::F64); -let exp = Exp.resolve(Variant::checked(double), [DType::Integer(Width::I32)]).unwrap(); -assert_eq!(exp.variant.dtype, double); +let exp = Kernel::resolve(Exp, Variant::checked(double), [DType::Integer(Width::I32)]).unwrap(); +assert_eq!(exp.variant().dtype, double); ``` `Sin`, `Cos` and `Tan` use short reduction and polynomials for their common range. An object's repair predicate marks arguments beyond 2^23 at f64 or 2^20 at f32. Its native slice driver applies the object's scalar repair routine to those items before returning the results. `Kernel::ranged` omits repair when the same predicate is proved false over the supplied ranges. @@ -87,7 +87,6 @@ Equal(Tolerance::DEFAULT).many(Arg::Many(&[0.3, 0.1 + 0.2, 0.31]), Arg::One(0.3) - `structure`: plans for structural operations. A `Plan` holds the result's `Layout` and the `Steps` that `walk` reads it by. `select` applies independent axis selectors for positions, ranges, index arrays and inserted axes. Index arrays pair, all of them by broadcasting as `.[]` reads them, or those that share an axis name as `⌷` reads them. `addressed` gives the axis that each selector selects along. `take` takes or drops items, `rotate` rotates or reverses an axis, with one count or one for each lane, `transpose` sends axes to new positions, and `catenate` gives the layout of two arrays joined along an axis. `Windows` plans windows over leading axes, centred on each position of a padded axis, which reads past its ends by an `Edge` mode. `Walk` classifies how a copy walks its source: the source axis that each entry walks, and, through `read` and `range`, what the entries along one source axis read as windows. `Walk::windows` recognises a copy of windows laid out as `Windows::plan` lays them out without padding, and gives each axis's frames, window size and move. `Transpose` gives the kernel that adds a copy's cotangent back to its source: a sum along its spread axes, then reads that send each item back, overlap-add for windows that move by more than 1, or a scatter-add. Each plan checks its arguments. Keys and names follow their positions. - `signal`: correlations and convolutions, which slide a kernel along an array's leading axes, and overlap-add. `correlation` gives the plans that a correlation reads: the array with the fills its `Mode` adds, its windows, and the kernel, reversed for a convolution. `correlate` runs them through an `Inner` pair's `inner_mapped`. Float and complex `PlusTimes` read the windows a block at a time. `overlap_add` adds each frame back where its window was read, the transpose of taking windows. - `convert`: `int` converts a whole float to an integer type. It gives `None` outside the type's range, and for `i64` and `u32` beyond a magnitude of 2^51 - 1. `float` converts an integer to the nearest float. Loops that call either vectorize on every target. AVX2 and wasm lack a vector instruction between `f64` and 64-bit integers. On those targets a loop that converts with `as` runs one item at a time. -- `timing`: the machine's timing lock, which keeps timings apart from builds and test runs. `exclusive` takes it for a timing and runs the calling thread's calls on that thread alone while it's held. `shared` takes it for a build or a test run. The browser build doesn't have it. The special functions `gamma`, `lgamma`, `digamma`, `erf`, `erfc`, their inverses, `binom` with its slopes `binom_dn` and `binom_dk`, the incomplete gamma and beta functions and Lambert W have free functions and objects at the crate root, as the elementary functions do. diff --git a/src/lib.rs b/src/lib.rs index e418759..53d4222 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,213 +1,9 @@ -//! Maths in plain Rust. Its float functions take `f16`, `f32` and `f64`, each within 4 units in the last place (ulps) of the correctly rounded result. The README lists the functions with wider bounds. The crate also has APL's tolerant comparison, exact rationals, linear algebra, number theory and polynomials. Its Python functions evaluate eagerly and return Rustymath arrays or Python scalars. +//! Numerical kernels and value-semantic arrays. #![doc = include_str!("lib.md")] #[macro_use] -mod slice; -mod arc; -mod arith; -#[macro_use] -pub mod buffer; -pub mod agreement; +extern crate rustymath_core; +pub use rustymath_core::*; +pub use rustymath_kernels::operation; pub mod array; -pub mod characters; -mod complex; -pub mod convert; pub mod create; -mod dd; -pub mod derivative; -pub mod distribution; -pub mod domain; -pub mod error; -pub mod events; -pub mod exact; -mod exp; -pub mod facts; -mod field; -mod float; -pub mod gather; -pub mod hashing; -mod hyperbolic; -pub mod integer; -pub mod interrupt; -mod lambert; -pub mod layout; pub mod linalg; -mod ln; -pub mod memory; -pub mod number_theory; -pub mod operation; -pub mod optimize; -pub mod parallel; -mod parts; -pub mod polynomial; -mod pow; -pub mod random; -pub mod recipe; -pub mod reduce; -pub mod search; -pub mod signal; -mod special; -mod sqrts; -pub mod stats; -pub mod structure; -mod table; -pub mod timing; -pub mod tolerant; -mod trig; - -pub use arc::{RealAcos, RealAsin}; -pub use arith::{ - Add, BitAnd, BitOr, Bitwise, Divide, LeftShift, MathishDivide, Multiply, Nand, Negate, Nor, Not, PiTimes, Reciprocal, RightShift, Subtract, bitwise, -}; -pub use complex::{Cis, divide}; -pub use exp::{Exp, Expm1}; -pub use float::{FUSED, Float, Native}; -pub use hyperbolic::{RealAcosh, RealAtanh}; -pub use ln::{Ln, Log1p, Log2, Log10, Logb, MathishLogb, RealLn}; -pub use parts::{Abs, Angle, Choose, Clip, Conj, Imag, Isfinite, Isinf, Isnan, Real, Rint, Round, Scale, Sign, Trunc}; -pub use pow::{Exponent, Pow, RealPow, Root}; -pub use slice::{Arg, Binary, CHUNK, Flipped, KernelItem, Results, Ternary, Unary, Wrapped, pairs}; -pub use special::{BetaincAt, Erfc, Erfcinv, Erfinv}; -pub use sqrts::{RealSqrt, RealSqrt1mx2, RealSqrtm1mx2, RealSqrtx2m1, Sqrt}; -pub use trig::{Cos, Sin, Tan}; -/// The crates whose types compact buffers hold, which the exported width macros name. -pub use {half, num_complex}; - -/// How many items a function handles between checks for an interrupt ([`interrupt::check`]), which keeps the check out of the loop over items. -pub const BETWEEN_CHECKS: usize = 1024; - -/// Declares each function at the crate root, where it multiplies and adds as the target's `FUSED` says, and in `fused` and `unfused`, which choose for themselves. Each entry names the module whose function of the same name implements it, generic over `R: Native` and `const FUSED: bool`. Every wrapper is `#[inline(always)]`, because LLVM leaves a large function such as `gamma` out of line in a caller's loop, and the call stops that loop vectorizing. An entry marked `as` also declares a function object of that name, for the facts table's formula. An object with forms, such as [`Pow`] or [`Atan`], is written with `forms!` beside its function. `function_doc!(ln)` gives an entry's doc by its name. The Python package's functions take their docs from it. -macro_rules! functions { - ($($(#[doc = $doc:literal])* $name:ident($($x:ident),+) in $module:ident $(as $object:ident)?;)+) => { - $( - $(#[doc = $doc])* - #[inline(always)] - pub fn $name($($x: F),+) -> F { functions!(@call FUSED, $module, $name, ($($x),+)) } - )+ - $(functions!(@object $($object)?; $name, $module, ($($x),+));)+ - /// Each function's doc by its name, as `function_doc!(ln)`, with its lines joined by newlines. - #[macro_export] - macro_rules! function_doc { - $(($name) => { concat!($($doc, "\n"),*) };)+ - } - /// rustymath's functions with `mla` fused, whatever the target's `FUSED`. Code compiled with FMA enabled for itself, such as an AVX2 version of a loop chosen when the program runs, calls these on a target whose baseline lacks FMA. - pub mod fused { - use crate::Float; - $( - #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), "), with `mla` fused.")] - #[inline(always)] - pub fn $name($($x: F),+) -> F { functions!(@call true, $module, $name, ($($x),+)) } - )+ - } - /// rustymath's functions with `mla` unfused, whatever the target's `FUSED`. - pub mod unfused { - use crate::Float; - $( - #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), "), with `mla` unfused.")] - #[inline(always)] - pub fn $name($($x: F),+) -> F { functions!(@call false, $module, $name, ($($x),+)) } - )+ - } - }; - (@object; $name:ident, $module:ident, ($($x:ident),+)) => {}; - (@object $object:ident; $name:ident, $module:ident, ($($x:ident),+)) => { - #[doc = concat!("[`", stringify!($name), "`](crate::", stringify!($name), ") as a function object.")] - #[derive(Clone, Copy, Debug)] - pub struct $object; - }; - (@call $fused:expr, $module:ident, $name:ident, ($($x:ident),+)) => { F::from_working(crate::$module::$name::<_, { $fused }>($($x.working()),+)) }; -} - -functions! { - /// e^x. `exp(∞)` is ∞, `exp(-∞)` is 0, and `exp(NaN)` is NaN. - exp(x) in exp; - /// e^x - 1, without the loss of digits near 0 that `exp(x) - 1` has. - expm1(x) in exp; - /// The natural logarithm of x. `ln(0)` is -∞, `ln(∞)` is ∞, and a negative real x or NaN gives NaN. - ln(x) in ln; - /// ln(1 + x), without the loss of digits near 0 that `ln(1 + x)` has. `log1p(-1)` is -∞. - log1p(x) in ln; - /// The logarithm of x to base 2. It is k exactly at 2^k. - log2(x) in ln; - /// The logarithm of x to base 10. It is k exactly at 10^k, and at the float nearest 10^k where 10^k isn't a float. - log10(x) in ln; - /// The logarithm of x to base b, ln x / ln b. Base 2 and base 10 take `log2` and `log10`, which are exact at powers of their base. - logb(b, x) in ln; - /// sin x. `sin(±0)` is ±0, and `sin(±∞)` and `sin(NaN)` are NaN. Beyond a magnitude of 2^23 at f64 or 2^20 at f32, x takes a slower, exact reduction. - sin(x) in trig; - /// cos x. `cos(±0)` is 1, and `cos(±∞)` and `cos(NaN)` are NaN. Beyond the magnitude that `sin` gives, x takes the same exact reduction. - cos(x) in trig; - /// tan x. `tan(±0)` is ±0, and `tan(±∞)` and `tan(NaN)` are NaN. Beyond the magnitude that `sin` gives, x takes the same exact reduction. - tan(x) in trig; - /// asin x. A real x gives a result from -π/2 to π/2 and NaN beyond a magnitude of 1. - asin(x) in arc as Asin; - /// acos x. A real x gives a result from 0 to π and NaN beyond a magnitude of 1. - acos(x) in arc as Acos; - /// atan x. A real x gives a result from -π/2 to π/2. - atan(x) in arc as Atan; - /// The angle of the point (x, y) from the positive x axis, from -π to π, with y first as in C's `atan2`. Zeros and infinities give C99's angles: atan2(±0, -0) is ±π, and atan2(±∞, ±∞) is an odd multiple of ±π/4. - atan2(y, x) in arc as Atan2; - /// sinh x. It stays finite up to where sinh itself overflows, beyond the point where e^x does. - sinh(x) in hyperbolic as Sinh; - /// cosh x. It stays finite up to where cosh itself overflows, beyond the point where e^x does. - cosh(x) in hyperbolic as Cosh; - /// tanh x. - tanh(x) in hyperbolic as Tanh; - /// asinh x. - asinh(x) in hyperbolic as Asinh; - /// acosh x. A real x below 1 gives NaN. - acosh(x) in hyperbolic as Acosh; - /// atanh x. ±1 gives ±∞, and a real x of magnitude beyond 1 gives NaN. - atanh(x) in hyperbolic as Atanh; - /// x^y. Special values follow IEEE 754's `pow`: x^0 and 1^y are 1, even for NaN, and a negative x has a power only for a whole y. - pow(x, y) in pow; - /// The real nth root of y, computed without rounding 1/n first. A negative y has one only for an odd whole n. n = 0, an infinite n and NaN give NaN. - root(n, y) in pow; - /// The cube root of x. - cbrt(x) in pow as Cbrt; - /// √(x² + y²), with no overflow or underflow in between. An infinite argument gives ∞, even beside NaN. - hypot(x, y) in pow as Hypot; - /// √(1 - x²), without the digits that squaring x first loses near |x| = 1. A real x of magnitude beyond 1 gives NaN. - sqrt1mx2(x) in sqrts as Sqrt1mx2; - /// √(1 + x²), with no overflow where x² would overflow. - sqrt1px2(x) in sqrts as Sqrt1px2; - /// √(x - 1) √(x + 1): √(x² - 1) with the sign of a real x, without the digits that squaring x first loses near |x| = 1. A real x of magnitude below 1 gives NaN. - sqrtx2m1(x) in sqrts as Sqrtx2m1; - /// Γ(x). Γ(±0) is ±∞, a negative whole x and -∞ give NaN, and Γ(∞) is ∞. - gamma(x) in special as Gamma; - /// Γ(x + 1), which is x! for a whole x from 0. - factorial(x) in special as Factorial; - /// ln |Γ(x)|. It is ∞ at 0, at the negative whole numbers and at ±∞. Near its zeros at 1 and 2 its error is within 4 ulps of 2, not of the result. - lgamma(x) in special as Lgamma; - /// ψ(x), the digamma function: the derivative of ln Γ(x), Γ′(x)/Γ(x). ψ(+0) is -∞, ψ(-0) is ∞, a negative whole x and -∞ give NaN, and ψ(∞) is ∞. Near its zeros its error is within 4 ulps of 1, not of the result, and below -20 within 4 ulps of ln(1 - x). - digamma(x) in special as Digamma; - /// ψ⁽ⁿ⁾(x), the polygamma function: the nth derivative of ψ, for a whole n from 0 to 170. For n from 1, x from 0 down gives NaN, as do a fractional or negative n and an n above 170. - polygamma(n, x) in special as Polygamma; - /// B(a, b) = Γ(a) Γ(b)/Γ(a + b), the beta function. A pole of Γ(a) or Γ(b) gives NaN, and a pole of Γ(a + b) alone gives 0. - beta(a, b) in special as Beta; - /// C(n, k), the binomial coefficient. Whole n and k from 0 multiply out exactly while the result times |n| + k is a whole number the type holds, and a negative whole n gives (-1)^k C(k - n - 1, k). Other n with a whole k from 0 to 64 multiply out n(n - 1)…(n - k + 1)/k!, within 2k ulps. Other real arguments take Γ(n + 1)/(Γ(k + 1) Γ(n - k + 1)) in double words, within 4 ulps in tests against mpmath, for large n and negative arguments too. - binom(n, k) in special as Binom; - /// ∂C(n, k)/∂n, the binomial coefficient's slope in n. For a whole k from 0 to 64 it is the slope of the polynomial n(n - 1)…(n - k + 1)/k! by the product rule, within 2k ulps. Other arguments take C(n, k)(ψ(n + 1) - ψ(n - k + 1)), with the ψ difference in double words where both arguments are from 10 up, with the polynomial's slope where a whole n below a whole k puts those ψ at a pole. Other poles of ψ give NaN. - binom_dn(n, k) in special as BinomDn; - /// ∂C(n, k)/∂k, the binomial coefficient's slope in k: C(n, k)(ψ(n - k + 1) - ψ(k + 1)). At a whole n from 0 below a whole k, it is the negative of `binom_dn`. Other poles of ψ, as at a negative whole k, give NaN. For a whole k from 0 to 64, its error is within 2k ulps of C(n, k) times ψ(n - k + 1) - ψ(k + 1), plus the error of that difference, so near n = 2k, where the two ψ cancel, its relative error has no bound. - binom_dk(n, k) in special as BinomDk; - /// The error function, 2/√π times the integral of e^-t² from 0 to x. - erf(x) in special as Erf; - /// 1 - erf(x), without the loss of digits for large x that `1 - erf(x)` has. - erfc(x) in special; - /// The x with erf(x) = p. ±1 gives ±∞, and a magnitude beyond 1 gives NaN. - erfinv(p) in special; - /// The x with erfc(x) = q. 0 gives ∞, 2 gives -∞, and q outside 0 to 2 gives NaN. - erfcinv(q) in special; - /// W0(x), the principal branch of the Lambert W function: the w of at least -1 with w e^w = x. A real x below -1/e gives NaN, except within half an ulp of -1/e, where W0 is -1. - lambertw(x) in lambert as LambertW; - /// The regularized lower incomplete gamma function P(a, x): the integral of t^(a-1) e^-t from 0 to x, divided by Γ(a). a must be positive and x nonnegative, or the result is NaN. - gammainc(a, x) in special as Gammainc; - /// The regularized upper incomplete gamma function Q(a, x) = 1 - P(a, x), without the loss of digits that `1 - gammainc(a, x)` has where Q is small. - gammaincc(a, x) in special as Gammaincc; - /// The x with P(a, x) = p, for p from 0 to 1. - gammaincinv(a, p) in special as Gammaincinv; - /// The regularized incomplete beta function I_x(a, b): the integral of t^(a-1) (1-t)^(b-1) from 0 to x, divided by B(a, b). a and b must be positive and x from 0 to 1, or the result is NaN. - betainc(a, b, x) in special as Betainc; - /// The x with I_x(a, b) = p, for p from 0 to 1. - betaincinv(a, b, p) in special as Betaincinv; -} diff --git a/src/linalg.rs b/src/linalg.rs index 02427b2..f17a787 100644 --- a/src/linalg.rs +++ b/src/linalg.rs @@ -1,1437 +1,4 @@ -//! Inner products of row-major matrices. The inner product `f.g` of an `m`×`k` matrix `a` and a `k`×`n` matrix `b` is the `m`×`n` matrix whose item at row `i` and column `j` folds `f` over `g(a[i][p], b[p][j])` for each `p`. `+.×` is the matrix product. The specialized pairs have associative `f`, and their loops combine its steps in whatever order suits them. A float result can therefore differ in its last bits from a fold taken in order. [`inner_with`] runs other native pairs through bounded term blocks and a caller's right fold, which preserves the association of non-associative functions. -//! -//! The loops follow GotoBLAS and BLIS. Packing copies a block of each argument into a contiguous buffer sized for a cache level. The blocking reuses each buffer while it is in cache. A micro-kernel computes a small tile of the result in registers, one step of `p` at a time. A micro-kernel in plain Rust vectorizes through LLVM's SLP vectorizer, which the builds of rustymath and basedpl turn on at opt-level 2. It vectorizes only tiles of up to 32 items, too few to keep the multiply-add units busy. On AArch64, in wasm and on x86-64 with AVX2 and FMA, the float pairs run micro-kernels on explicit vectors instead. Their maximum and minimum are AArch64's `fmaxnm`, wasm's `pmax` and x86-64's `maxpd`, with the running value as the argument each gives beside NaN. A tile starts at `-∞` or `∞`, so its running value is never NaN and a NaN term leaves it unchanged, as `Core::maximum` does. Targets without these vectors run the plain kernels. -//! -//! A product large enough splits its rows across the threads that [`crate::parallel::set_threads`] sets. On macOS, Accelerate computes real and complex plus-times products instead. `Tiled` also sends the decompositions' triangular solves, SVDs and Hermitian eigen decompositions to it, and the general eigen decompositions of real matrices from `accelerate::GEEV_FROM` rows. macOS builds with the `native` feature also compile rustymath's own kernels, and `RUSTYMATH_NATIVE=1` selects them for the whole process. -//! -//! The private module `factor` holds decompositions of real and complex matrices, whose blocked updates run on these loops. This module re-exports its public items, and [`Scalar`], the trait of the item types they take. [`transpose_into`] transposes a matrix through 8×8 tiles. -use crate::{ - Float, agreement::{Mapping, Walked, expanded_items}, buffer::{Element, Source, read_all}, convert, error::Error, float::{FUSED, sealed::Core}, - parallel, slice -}; -use num_complex::Complex; -use num_traits::{One, Zero}; -use std::{borrow::Cow, mem::MaybeUninit, sync::LazyLock}; - -#[cfg(accelerate)] -mod accelerate; -mod factor; -mod inner; -mod scalar; +//! Matrix algorithms and their array interface. +pub use rustymath_core::linalg::*; mod stacked; -// The platform's own BLAS and LAPACK, which every `Tiled` type implements: Accelerate on macOS, and none elsewhere. -#[cfg(accelerate)] -pub(crate) use accelerate::Accelerated as Vendor; -#[cfg(not(accelerate))] -pub(crate) trait Vendor {} -#[cfg(not(accelerate))] -impl Vendor for T {} - -/// Whether this process selects rustymath's own kernels where Accelerate is also compiled in: in macOS builds with the `native` feature, when `RUSTYMATH_NATIVE=1`. It's read once. -fn switched() -> bool { - static NATIVE: LazyLock = LazyLock::new(|| std::env::var_os("RUSTYMATH_NATIVE").is_some_and(|v| v == "1")); - cfg!(all(accelerate, native)) && *NATIVE -} - -/// Whether Accelerate serves this process's products, triangular solves and decompositions. -pub(crate) fn accelerated() -> bool { cfg!(accelerate) && !switched() } - -/// `$vendor` where Accelerate serves this process, and `$native` where rustymath's own kernels do. Each compiles only where its kernels do. -macro_rules! dispatch { - ($vendor:expr, $native:expr) => {{ - #[cfg(all(accelerate, native))] - let r = if accelerated() { $vendor } else { $native }; - #[cfg(all(accelerate, not(native)))] - let r = $vendor; - #[cfg(not(accelerate))] - let r = $native; - r - }}; -} -/// `$vendor` where Accelerate serves this process and the matrix's order `$n` is at least `$from`, and `$native` otherwise. Accelerate's LAPACK pays a fixed cost for each call, which makes the native solver faster for small matrices. Use it only in a function body: it returns `$vendor`'s result. -macro_rules! from_order { - ($n:expr, $from:expr, $vendor:expr, $native:expr) => {{ - #[cfg(accelerate)] - if $n >= $from && accelerated() { - return $vendor; - } - $native - }}; -} -pub mod exact; -pub use factor::*; -pub use inner::inner_with; -pub use scalar::Scalar; pub use stacked::{Stacked, cells, paired, stored}; - -/// The `n`×`n` identity matrix. -pub(crate) fn identity(n: usize) -> Vec { - let mut x = Vec::new(); - identity_into(&mut x, n); - x -} - -/// [`identity`], written into `x` in place of its old items. -pub(crate) fn identity_into(x: &mut Vec, n: usize) { - x.clear(); - x.extend(std::iter::repeat_with(T::zero).take(n * n)); - for i in 0..n { x[i * (n + 1)] = T::one() } -} - -/// Writes the transpose of the `rows`×`cols` matrix in `src`, whose rows start `ls` items apart, into `dst`, whose rows start `ld` items apart. Each item passes through `f`: `dst[c * ld + r]` is `f(src[r * ls + c])`. Whole 8×8 tiles go through [`tiles`], and the items past the last whole tile one at a time. -#[inline] -pub fn transpose_into(src: &[S], ls: usize, dst: &mut [D], ld: usize, [rows, cols]: [usize; 2], f: impl Fn(S) -> D) { - if rows == 0 || cols == 0 { return; } - assert!(src.len() >= (rows - 1) * ls + cols && dst.len() >= (cols - 1) * ld + rows, "both matrices lie in their slices"); - let (whole_rows, whole_cols) = (rows / 8 * 8, cols / 8 * 8); - // Each write moves 16 bytes where the items allow, which is at most 8 items, as the registers that the compiler shuffles hold. - match 16 / size_of::().max(1) { - 8.. => tiles::<8, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), 4.. => tiles::<4, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), - 2.. => tiles::<2, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f), _ => tiles::<1, S, D>(src, ls, dst, ld, [whole_rows, whole_cols], &f) - } - if whole_cols < cols { for r in 0..whole_rows { for c in whole_cols..cols { dst[c * ld + r] = f(src[r * ls + c]) } } } - for r in whole_rows..rows { for c in 0..cols { dst[c * ld + r] = f(src[r * ls + c]) } } -} - -/// The whole 8×8 tiles of a transpose, the first `rows` rows and `cols` columns of each matrix, which its caller checked lie in their slices. Each tile reads `P` of its runs of 8 adjacent items at a time, and writes `P` adjacent items of each result run, one from each run read. The compiler shuffles the runs' lanes in registers, as `zip` instructions do for two runs of 8-byte items. -#[inline(always)] -fn tiles(src: &[S], ls: usize, dst: &mut [D], ld: usize, [rows, cols]: [usize; 2], f: &impl Fn(S) -> D) { - let (from, to) = (src.as_ptr(), dst.as_mut_ptr()); - for r0 in (0..rows).step_by(8) { - for c0 in (0..cols).step_by(8) { - for i in (0..8).step_by(P) { - // SAFETY: the tile's rows and result rows lie in `src` and `dst`, as the caller checked. - let read = |k: usize| unsafe { from.add((r0 + i + k) * ls + c0).cast::<[S; 8]>().read() }; - let mut runs = [read(0); P]; - for k in 1..P { runs[k] = read(k) } - for j in 0..8 { let out = unsafe { &mut *to.add((c0 + j) * ld + r0 + i).cast::<[D; P]>() }; for k in 0..P { out[k] = f(runs[k][j]) } } - } - } - } -} - -/// An inner product `f.g` at item type `T`. Each pair object implements it at the item types listed here. Its loops compile in rustymath, as slice forms do. -pub trait Inner { - /// The type of each result. - type Out: Copy + Default; - /// The `m`×`n` inner product of `a`, an `m`×`k` matrix, and `b`, a `k`×`n` matrix, both row-major, written row-major into `out`, which holds `m`×`n` items. `size` is `[m, k, n]`. Where `k` is 0, each result is the identity of `f`. An integer result that doesn't fit its type panics, where `checked_into` gives `false`. - fn inner_into(&self, a: &[T], b: &[T], size: [usize; 3], out: &mut [Self::Out]); - /// `inner_into`'s results, as a new row-major matrix. - fn inner(&self, a: &[T], b: &[T], size @ [m, _, n]: [usize; 3]) -> Vec { - let mut c = filled(m * n, Self::Out::default()); - self.inner_into(a, b, size, &mut c); - c - } - /// `inner_into`, or `false` where an integer result doesn't fit its type, for callers that keep integers exact or report overflow. Where it gives `true`, `out` holds the exact results. Floats always give `true`. - #[inline] - fn checked_into(&self, a: &[T], b: &[T], size: [usize; 3], out: &mut [Self::Out]) -> bool { self.inner_into(a, b, size, out); true } - /// `inner` of the matrices that `a` and `b` give in the orientations given, with `size` `[m, k, n]` for those matrices. A transposed `a` holds a `k`×`m` matrix, row-major, and gives its `m`×`k` transpose. This default copies a transposed operand into its orientation. Float and complex `PlusTimes` read it in place. - fn inner_ops(&self, a: &[T], b: &[T], [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3]) -> Vec where T: Copy { - fn oriented(x: &[T], o: Orientation, [rows, cols]: [usize; 2]) -> std::borrow::Cow<'_, [T]> { - match o { - Orientation::Normal => std::borrow::Cow::Borrowed(x), - Orientation::Transpose => (0..rows * cols).map(|ij| x[ij % cols * rows + ij / cols]).collect(), - } - } - self.inner(&oriented(a, oa, [m, k]), &oriented(b, ob, [k, n]), [m, k, n]) - } - /// `inner_ops` of operands whose items come through mappings, as [`Mapped`] describes. This default reads each operand whole, then calls `inner_ops`. Float and complex `PlusTimes` read an operand that isn't held in order one block of its stored rows at a time and multiply each block as it's read. Only one block of that operand then exists at once. - fn inner_mapped(&self, a: Mapped<'_, T>, b: Mapped<'_, T>, orientations: [Orientation; 2], size @ [m, k, n]: [usize; 3]) -> Vec where T: Element - { - let (mut ab, mut bb) = (Vec::new(), Vec::new()); - self.inner_ops(a.read(0, m * k, &mut ab), b.read(0, k * n, &mut bb), orientations, size) - } -} - -/// `+.×`, the matrix product, at `i64`, `f32` and `f64`, and complex numbers with parts of either type. Native float steps use fused multiply-add where the target has one; Accelerate computes them on macOS. Nonfinite complex results need not match scalar arithmetic. Native complex products form each part from two real products of the parts, as scalar multiplication does. A small imaginary part beside a large real part keeps its own relative accuracy. -#[derive(Default)] -pub struct PlusTimes; -/// `⌈.+`, the max-plus product of longest-path problems, at `i64`, `f32` and `f64`. The module doc says how each target treats a NaN term. -#[derive(Default)] -pub struct MaxPlus; -/// `⌊.+`, the min-plus product of shortest-path problems, at `i64`, `f32` and `f64`. Its minimum treats NaN as `MaxPlus`'s maximum does. -#[derive(Default)] -pub struct MinPlus; -/// `∨.∧` on Booleans: whether some `p` has both `a[i][p]` and `b[p][j]`. -#[derive(Default)] -pub struct OrAnd; -/// `∧.∨` on Booleans: whether every `p` has `a[i][p]` or `b[p][j]`. -#[derive(Default)] -pub struct AndOr; -/// `+.=`, at `i16`, `i32`, `i64`, `f32` and `f64`: the number of `p` where `a[i][p]` equals `b[p][j]`. The count has the width of the items, as `i16`, `i32` or `i64`, so an `i16` count needs `k` below 2^15 and an `i32` count below 2^31. Floats compare exactly, so NaN equals nothing. -#[derive(Default)] -pub struct PlusEqual; -/// `+.≠`, at `i16`, `i32`, `i64`, `f32` and `f64`: the number of `p` where `a[i][p]` differs from `b[p][j]`, with `+.=`'s count type. -#[derive(Default)] -pub struct PlusNotEqual; - -/// A native inner-product pair. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Product { PlusTimes, MaxPlus, MinPlus, OrAnd, AndOr, PlusEqual, PlusNotEqual } - -impl Product { pub fn compares_exactly(self) -> bool { matches!(self, Self::PlusEqual | Self::PlusNotEqual) } } - -/// Layout and reads of an inner product. Contracted axes follow output axes in `iteration`. -#[derive(Clone, Debug)] -pub struct ProductPlan { pub layout: crate::layout::Layout, pub iteration: crate::layout::Layout, pub mappings: [Mapping; 2], pub contracted: usize } - -#[derive(Clone, Copy, Debug)] -pub struct MatrixProduct { pub frame_rank: usize, pub row_rank: usize, pub size: [usize; 3] } -impl MatrixProduct { - fn of(shape: &[usize], owners: &[u8]) -> Option { - let frame_rank = owners.iter().take_while(|&&owner| owner == 3).count(); - let row_rank = owners[frame_rank..].iter().take_while(|&&owner| owner == 1).count(); - if !owners[frame_rank + row_rank..].iter().all(|&owner| owner == 2) { return None; } - let sizes = [&shape[frame_rank..frame_rank + row_rank], &shape[owners.len()..], &shape[frame_rank + row_rank..owners.len()]] - .map(|shape| crate::layout::element_count(shape).ok()); - match sizes { [Some(m), Some(k), Some(n)] => Some(Self { frame_rank, row_rank, size: [m, k, n] }), _ => None } - } -} - -impl ProductPlan { - /// The number of pairings each result folds: the product of the contracted axes' lengths. - pub fn contracted_len(&self) -> usize { let shape = self.iteration.shape(); crate::layout::count(&shape[shape.len() - self.contracted..]) } - pub fn folded(iteration: crate::layout::Layout, mappings: [Mapping; 2], contracted: usize) -> Result { - let rank = iteration.shape().len(); - if contracted > rank { return Err(Error::new(crate::error::Kind::Rank, "too many contracted axes")); } - let layout = iteration.axes(0..rank - contracted); - Ok(Self { layout, iteration, mappings, contracted }) - } - - /// The matrix product that runs this plan, from the iteration axes each operand steps along: its read's steps where the read reaches its items positionally, and otherwise its `mappings` entry's, which says which items it reads. An axis that no read steps along, as a broadcast gives, takes the class of the operands whose free axis `mappings` makes it, and an axis of one item takes the class of the axis before it. `None` where the axes don't fall into frames, rows and columns, in that order. - pub fn matrix_for(&self, inputs: [crate::agreement::Read<'_>; 2]) -> Option { - let shape = self.iteration.shape(); - let mut owners = vec![0; shape.len() - self.contracted]; - let mut nominal = owners.clone(); - for (side, input) in inputs.iter().enumerate() { - let plan = input.plan.remapped(self.mappings[side].clone(), self.iteration.count(), crate::agreement::Missing::Invalid); - let map = - crate::agreement::Read { sources: input.sources, plan: &plan }.positional(shape).map_or_else(|| self.mappings[side].clone(), |(_, map)| map); - let (_, steps) = map.affine(shape)?; - for (axis, owner) in owners.iter_mut().enumerate() { if steps[axis] != 0 { *owner |= 1 << side; } } - if let Some((_, free)) = self.mappings[side].affine(shape) { - for (named, &step) in nominal.iter_mut().zip(&free) { if step != 0 { *named |= 1 << side; } } - } - } - // An axis of one item adds nothing to any size, so any class that keeps the order serves. - let mut last = 3; - for ((owner, named), &len) in owners.iter_mut().zip(nominal).zip(shape) { - if *owner == 0 { *owner = if len == 1 { last } else if named != 0 { named } else { 3 }; } - last = *owner; - } - MatrixProduct::of(shape, &owners) - } - pub fn dot([left, right]: [&crate::layout::Layout; 2]) -> Result { - let rank = left.shape().len(); - if rank == 0 || right.shape().is_empty() { return Err(Error::new(crate::error::Kind::Rank, "an inner product needs vectors or matrices")); } - Self::paired([left, right], &[(rank - 1, 0)], None) - } - pub fn matrices(inputs: [&crate::layout::Layout; 2], frame: usize) -> Result { - let (contracted, destinations) = Self::matrix_axes(inputs, frame)?; - Self::paired(inputs, &[contracted], Some(destinations)) - } - /// The contracted pair and the destinations of each input's remaining axes, in source order. - pub fn matrix_axes([left, right]: [&crate::layout::Layout; 2], frame: usize) -> Result<((usize, usize), [Vec; 2]), Error> { - use crate::agreement::Agreement; - let [lr, rr] = [left, right].map(|l| l.shape().len()); - if lr <= frame || rr <= frame { return Err(Error::new(crate::error::Kind::Rank, "a matrix product needs vector or matrix cells")); } - let [lc, rc] = [(lr - frame).min(2), (rr - frame).min(2)]; - let [lf, rf] = [left.axes(0..lr - lc), right.axes(0..rr - rc)]; - let [mut xa, mut ya] = Agreement::axis_pairs(&lf, &rf); - let first = xa.iter().chain(&ya).max().map_or(0, |a| a + 1); - if lc == 2 { xa.push(first); } - if rc == 2 { ya.push(first + usize::from(lc == 2)); } - Ok(((lr - 1, rr - rc), [xa, ya])) - } - /// Contracts `pairs`; optional destinations pair the remaining axes, in each input's axis order. Without destinations, remaining axes concatenate left then right. - pub fn paired(inputs @ [left, right]: [&crate::layout::Layout; 2], pairs: &[(usize, usize)], destinations: Option<[Vec; 2]>) -> Result { - use crate::{agreement::Agreement, error::Kind, layout::{element_count, generated_len}}; - let contracted: [Vec; 2] = [pairs.iter().map(|p| p.0).collect(), pairs.iter().map(|p| p.1).collect()]; - for (input, axes) in inputs.iter().zip(&contracted) { - if axes.iter().enumerate().any(|(i, a)| *a >= input.shape().len() || axes[..i].contains(a)) { - return Err(Error::new(Kind::Rank, "contracted axes must be distinct axes of their inputs")); - } - } - for &(a, b) in pairs { - let (x, y) = (left.shape()[a], right.shape()[b]); - if x != y { - return Err(Error::new(Kind::Length, format!("an inner product needs matching contracted lengths, not {x} and {y}"))); - } - Mapping::contract(left, a, right, b).map_err(|_| Error::new(Kind::Length, "an inner product needs the same keys on its contracted axes"))?; - } - let free: [Vec; 2] = std::array::from_fn(|i| (0..inputs[i].shape().len()).filter(|a| !contracted[i].contains(a)).collect()); - let free_layouts = std::array::from_fn::<_, 2, _>(|i| inputs[i].axes(free[i].iter().copied())); - let total = free[0].len() + free[1].len(); - let destinations = destinations.unwrap_or_else(|| [(0..free[0].len()).collect(), (free[0].len()..total).collect()]); - for (axes, positions) in free.iter().zip(&destinations) { - if axes.len() != positions.len() || positions.iter().enumerate().any(|(i, p)| *p >= total || positions[..i].contains(p)) { - return Err(Error::new(Kind::Rank, "free axes need distinct output destinations")); - } - } - let agreement = Agreement::mapped(&free_layouts[0], &free_layouts[1], &destinations[0], &destinations[1], false)?; - if agreement.left.fills() || agreement.right.fills() { return Err(Error::new(Kind::Length, "keyed frames need a cell that an argument lacks")); } - let layout = agreement.layout; - generated_len(layout.shape())?; - let first = layout.shape().len(); - let iteration = layout.concat(&left.axes(contracted[0].iter().copied())); - let items = element_count(iteration.shape())?; - let mappings = std::array::from_fn(|i| { - let mut axes = vec![0; inputs[i].shape().len()]; - for (&a, &p) in free[i].iter().zip(&destinations[i]) { axes[a] = p; } - for (p, &a) in contracted[i].iter().enumerate() { axes[a] = first + p; } - Mapping::new(inputs[i], &axes, &iteration, items) - }); - Ok(Self { layout, iteration, mappings, contracted: pairs.len() }) - } -} - -/// The scalar operations of a pair at item type `T`. -trait Pair { - type Out: Copy + Send; - /// The identity of `f`: the result of folding no steps. - const UNIT: Self::Out; - fn g(a: T, b: T) -> Self::Out; - fn f(x: Self::Out, y: Self::Out) -> Self::Out; - /// `f(acc, g(a, b))`, which `+.×` computes as one fused multiply-add. - #[inline(always)] - fn step(acc: Self::Out, a: T, b: T) -> Self::Out { Self::f(acc, Self::g(a, b)) } -} - -/// `Pair` impls, one for each entry: the object, the item type, the result type and `UNIT`, then `g`, `f` and an optional fused `step`. -macro_rules! pairs { - ($($obj:ty: $t:ty => $out:ty = $unit:expr, |$a:ident, $b:ident| $g:expr, |$x:ident, $y:ident| $f:expr $(, |$c:ident, $p:ident, $q:ident| $step:expr)?;)+) => {$( - impl Pair<$t> for $obj { - type Out = $out; - const UNIT: $out = $unit; - #[inline(always)] - fn g($a: $t, $b: $t) -> $out { $g } - #[inline(always)] - fn f($x: $out, $y: $out) -> $out { $f } - $(#[inline(always)] - fn step($c: $out, $p: $t, $q: $t) -> $out { $step })? - } - )+}; -} -macro_rules! floats { - ($($t:ty),+) => {$(pairs! { - PlusTimes: $t => $t = 0.0, |a, b| a * b, |x, y| x + y, |acc, a, b| a.mla::(b, acc); - MaxPlus: $t => $t = <$t>::NEG_INFINITY, |a, b| a + b, |acc, term| Core::maximum(term, acc); - MinPlus: $t => $t = <$t>::INFINITY, |a, b| a + b, |acc, term| Core::minimum(term, acc); - PlusTimes: Complex<$t> => Complex<$t> = Complex::new(0.0, 0.0), |a, b| a * b, |x, y| x + y, |acc, a, b| a.mul_acc(b, acc); - })+}; -} -floats!(f32, f64); - -pairs! { - OrAnd: bool => bool = false, |a, b| a & b, |x, y| x | y; - AndOr: bool => bool = true, |a, b| a | b, |x, y| x & y; -} - -impl Inner for OrAnd { - type Out = bool; - #[inline] - fn inner_into(&self, a: &[bool], b: &[bool], size: [usize; 3], out: &mut [bool]) { - if size[0] * size[1] * size[2] <= DIRECT_WORK { direct::(a, b, size, out) } else { packed::(a, b, size, out) } - } -} - -impl Inner for AndOr { - type Out = bool; - #[inline] - fn inner_into(&self, a: &[bool], b: &[bool], size: [usize; 3], out: &mut [bool]) { - if size[0] * size[1] * size[2] <= DIRECT_WORK { direct::(a, b, size, out) } else { packed::(a, b, size, out) } - } -} - -/// `∨.∧` of `a` and `b`, or with `NOT` set, `∧.∨`, which is `~(~a)∨.∧(~b)`, written into `c`. `NOT` flips each item of `a` and `b` as it is read, and each result as it is written. Each row of the result ORs together the rows of `b` where the row of `a` is true. The rows of `b` are packed 64 items to a word, in blocks of `WORDS` words, and each block of a row's results folds in a local array that stays in registers. An item of `a` becomes a mask of all ones or none, so the loop over it has no branch. -fn packed(a: &[bool], b: &[bool], [m, k, n]: [usize; 3], c: &mut [bool]) { - const WORDS: usize = 4; - assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); - if k == 0 || n == 0 { return c.fill(NOT); } - let blocks = n.div_ceil(64 * WORDS); - let mut packed = vec![[0u64; WORDS]; k * blocks]; - for (row, bits) in b.chunks_exact(n).zip(packed.chunks_exact_mut(blocks)) { - for (j, chunk) in row.chunks(64).enumerate() { - bits[j / WORDS][j % WORDS] = chunk.iter().enumerate().fold(0, |w, (t, &v)| w | u64::from(v ^ NOT) << t); - } - } - for (row, out) in a.chunks_exact(k).zip(c.chunks_exact_mut(n)) { - for (block, out) in out.chunks_mut(64 * WORDS).enumerate() { - let mut acc = [0u64; WORDS]; - for (&x, bits) in row.iter().zip(packed[block..].iter().step_by(blocks)) { - let mask = u64::from(x ^ NOT).wrapping_neg(); - for t in 0..WORDS { acc[t] |= bits[t] & mask } - } - for (j, o) in out.iter_mut().enumerate() { *o = (acc[j / 64] >> (j % 64) & 1 != 0) ^ NOT } - } - } -} -// The pairs at `i64` and `+.×`'s pair at `i32`. `tiered` runs the pairs at `i64` only where every value that a product forms fits `i64`. Their wrapping operations then never wrap. The pair at `i32` wraps, as the products of [`Wrapped`] do. The pairs at `i128`, which `checked_into` uses where a result could pass `i64`. Each step of `+.×` checks that its running total fits 128 bits. -pairs! { - PlusTimes: i64 => i64 = 0, |a, b| a.wrapping_mul(b), |x, y| x.wrapping_add(y); - PlusTimes: i32 => i32 = 0, |a, b| a.wrapping_mul(b), |x, y| x.wrapping_add(y); - MaxPlus: i64 => i64 = i64::MIN, |a, b| a.wrapping_add(b), |x, y| x.max(y); - MinPlus: i64 => i64 = i64::MAX, |a, b| a.wrapping_add(b), |x, y| x.min(y); - PlusTimes: i128 => Option = Some(0), |a, b| a.checked_mul(b), |x, y| x?.checked_add(y?); - MaxPlus: i128 => Option = Some(i128::MIN), |a, b| a.checked_add(b), |x, y| Some(x?.max(y?)); - MinPlus: i128 => Option = Some(i128::MAX), |a, b| a.checked_add(b), |x, y| Some(x?.min(y?)); -} -macro_rules! counts { - ($($t:ty => $out:ty),+) => {$(pairs! { PlusEqual: $t => $out = 0, |a, b| <$out>::from(a == b), |x, y| x + y; })+}; -} -counts!(i64 => i64, i32 => i32, i16 => i16, f64 => i64, f32 => i32); - -/// Steps of `p` in each packed block. -#[doc(hidden)] -pub const KC: usize = 256; -/// Rows of `a` in each packed block. -#[doc(hidden)] -pub const MC: usize = 128; -/// Columns of `b` in each packed block. -#[doc(hidden)] -pub const NC: usize = 2048; - -/// The tile shapes, `[MR, NR]`, and the bytes in a vector. A tile holds the rows and columns of the result that a micro-kernel keeps in registers. AArch64 has 32 vector registers of 16 bytes, and AVX2 has 16 registers of 32 bytes. wasm has 16-byte vectors, and V8 gives it 16 registers on x86-64. The extremes and the counts compare at each step, and on AVX2 they need registers for those comparisons, so their tiles have fewer rows. `TIMES32` serves both `f32` and `i32` products, whose items take 4 bytes. A plain kernel's tile holds at most 32 items. On the Mac, plain tiles of 48 or more floats compiled to scalar code and ran 4 to 5 times slower. -#[cfg(target_arch = "aarch64")] -mod tiles { - pub(super) const BYTES: usize = 16; - #[cfg(native)] - pub(super) const TIMES64: [usize; 2] = [8, 4]; - pub(super) const TIMES32: [usize; 2] = [8, 8]; - pub(super) const COMPARES64: [usize; 2] = [6, 8]; - pub(super) const COMPARES32: [usize; 2] = [6, 16]; - pub(super) const COMPARES16: [usize; 2] = [6, 32]; -} -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma"))] -mod tiles { - pub(super) const BYTES: usize = 32; - #[cfg(native)] - pub(super) const TIMES64: [usize; 2] = [6, 8]; - pub(super) const TIMES32: [usize; 2] = [6, 16]; - pub(super) const COMPARES64: [usize; 2] = [4, 8]; - pub(super) const COMPARES32: [usize; 2] = [4, 16]; - pub(super) const COMPARES16: [usize; 2] = [4, 32]; -} -#[cfg(not(any(target_arch = "aarch64", all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma"))))] -mod tiles { - pub(super) const BYTES: usize = 16; - pub(super) const TIMES64: [usize; 2] = [4, 4]; - pub(super) const TIMES32: [usize; 2] = [4, 8]; - pub(super) const COMPARES64: [usize; 2] = [4, 4]; - pub(super) const COMPARES32: [usize; 2] = [4, 8]; - pub(super) const COMPARES16: [usize; 2] = [4, 16]; -} - -/// The tile of the plain kernel on 64-bit integers, which only products too large for f64's 53 bits reach. -const PLAIN: [usize; 2] = [4, 8]; - -macro_rules! inner { - ($($object:ident at $t:ty => $out:ty: $tile:expr, $kernel:ident;)+) => {$( - impl Inner<$t> for $object { - type Out = $out; - fn inner_into(&self, a: &[$t], b: &[$t], size: [usize; 3], out: &mut [Self::Out]) { - const MR: usize = $tile[0]; - const NR: usize = $tile[1]; - tiled_into::<$t, $t, $object, MR, NR>(a, b, size, out, $kernel::() / tiles::BYTES }>) - } - } - )+}; -} -/// `n` copies of `x`. `vec!` of zeros allocates through calloc, whose `madvise` on a large block costs more than writing its items. -fn filled(n: usize, x: T) -> Vec { - let mut v = Vec::with_capacity(n); - v.resize(n, x); - v -} -/// `P`'s inner product of `a` and `b`, as `Inner::inner_into` gives it, written into `c`, which holds `m`×`n` items, through `kernel` on tiles of `MR`×`NR`. Each item reads as the kernel's type `T` through [`Source`] as it's packed, so an operand of narrower items is never copied whole. -#[inline(always)] -fn tiled_into + Copy + Sync, T: Copy + Default + Send + Sync, P: Pair, const MR: usize, const NR: usize>( - a: &[S], b: &[S], [m, k, n]: [usize; 3], c: &mut [P::Out], kernel: impl Fn(&[T], &[T], &mut [[P::Out; NR]; MR], bool) + Sync) { - assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); - if m * k * n <= DIRECT_WORK { return direct::(a, b, [m, k, n], c); } - let f = >::read; - product::( - c, n, |rows, p0, out| strip(a, k, false, rows, p0, out, f), |columns, p0, out| strip(b, n, true, columns, p0, out, f), [m, k, n], true, Part::Whole, - kernel); -} - -/// The most steps, `m`×`k`×`n`, of a product that [`direct`] computes, where packing the operands for the tiled kernels, or calling Accelerate, costs more than the steps. Measured on 22 October 2026 on an M5 Max, a stack of 4,096 products of 4×4 matrices cost 9.3 copies this way, against 25 on Accelerate and 38 on the packed kernels. A stack of 1,024 products of 8×8 matrices cost 10.1 copies this way, against 10.3 on Accelerate and 9.7 on the packed kernels, so the threshold stays below them. -const DIRECT_WORK: usize = 64; - -/// `P`'s inner product of `a` and `b`, written into `c`, which holds `m`×`n` items, by a plain loop. Each item reads as `T` through [`Source`]. Each result folds its row of `a` with its column of `b` in order, in a register, as the tiled kernels do on every target, NaN included. Four results of a row fold side by side, which vectorises, and the row's last columns fold one at a time. Nothing is packed or allocated. -// A stack of tiny products calls this once for each cell. Once the loop vectorizer ran before thin LTO's import step, this body grew too large to import into `product_at`, and the call cost a stack of 64 2×2 Boolean products about 600 instructions. -#[inline] -fn direct + Copy, T: Copy, P: Pair>(a: &[S], b: &[S], [m, k, n]: [usize; 3], c: &mut [P::Out]) { - let f = >::read; - const SIDE: usize = 4; - assert!(a.len() >= m * k && b.len() >= k * n && c.len() >= m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); - // SAFETY: every index below is `i * k + p`, `p * n + j` or `i * n + j` with `i < m`, `p < k` and `j < n`, which the assert bounds. Checking each one cost about half of `direct`'s instructions on a 4×4 product. - for i in 0..m { - let mut j = 0; - while j + SIDE <= n { - let mut acc = [P::UNIT; SIDE]; - for p in 0..k { - let x = f(unsafe { *a.get_unchecked(i * k + p) }); - for s in 0..SIDE { acc[s] = P::step(acc[s], x, f(unsafe { *b.get_unchecked(p * n + j + s) })) } - } - for s in 0..SIDE { unsafe { *c.get_unchecked_mut(i * n + j + s) = acc[s] } } - j += SIDE; - } - for j in j..n { - let mut acc = P::UNIT; - for p in 0..k { acc = P::step(acc, f(unsafe { *a.get_unchecked(i * k + p) }), f(unsafe { *b.get_unchecked(p * n + j) })) } - unsafe { *c.get_unchecked_mut(i * n + j) = acc } - } - } -} -inner! { - MaxPlus at f64 => f64: tiles::COMPARES64, max_plus64; - MaxPlus at f32 => f32: tiles::COMPARES32, max_plus32; - MinPlus at f64 => f64: tiles::COMPARES64, min_plus64; - MinPlus at f32 => f32: tiles::COMPARES32, min_plus32; - PlusEqual at i32 => i32: tiles::COMPARES32, equal_i32; - PlusEqual at i16 => i16: tiles::COMPARES16, equal_i16; - PlusEqual at f64 => i64: tiles::COMPARES64, equal64; - PlusEqual at f32 => i32: tiles::COMPARES32, equal32; -} - -/// `+.=` at `i64` counts at `i16` where `windowed` maps both arguments into it, and otherwise at `i32` where every item fits in 32 bits. A narrower count puts more items in each vector. -impl Inner for PlusEqual { - type Out = i64; - fn inner_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) { - if let Some(c) = windowed(a, b, size) { return out.iter_mut().zip(&c).for_each(|(o, &x)| *o = i64::from(x)); } - let narrow = |x: &[i64]| x.iter().map(|&v| i32::try_from(v).ok()).collect::>>(); - if let (Some(a), Some(b)) = (narrow(a), narrow(b)) { return out.iter_mut().zip(&self.inner(&a, &b, size)).for_each(|(o, &x)| *o = i64::from(x)); } - const MR: usize = tiles::COMPARES64[0]; - const NR: usize = tiles::COMPARES64[1]; - tiled_into::(a, b, size, out, equal_i64::() / tiles::BYTES }>) - } -} - -/// `+.=` of `a` and `b` counted at `i16`, where `k` is below 2^15 and every item of both lies in one window of 2^16 values. Each item becomes its distance from the window's start, read as an `i16`, so two items are equal exactly where they were before. -fn windowed(a: &[i64], b: &[i64], size: [usize; 3]) -> Option> { - if size[1] > i16::MAX as usize { return None; } - let ((la, ha), (lb, hb)) = (crate::reduce::extremes(a), crate::reduce::extremes(b)); - let (lo, hi) = (la.min(lb), ha.max(hb)); - if i128::from(hi) - i128::from(lo) > i128::from(u16::MAX) { return None; } - let shifted = |x: &[i64]| x.iter().map(|&v| v.wrapping_sub(lo) as u16 as i16).collect::>(); - Some(PlusEqual.inner(&shifted(a), &shifted(b), size)) -} - -/// `+.≠` as `k` less `+.=`. -macro_rules! differences { - ($($t:ty => $out:ty),+) => {$( - impl Inner<$t> for PlusNotEqual { - type Out = $out; - fn inner_into(&self, a: &[$t], b: &[$t], size: [usize; 3], out: &mut [$out]) { - PlusEqual.inner_into(a, b, size, out); - let k = size[1] as $out; - for x in out { *x = k - *x } - } - } - )+}; -} -differences!(i64 => i64, i32 => i32, i16 => i16, f64 => i64, f32 => i32); - -/// The largest magnitude among the items of `x`. -fn magnitude(x: &[T]) -> u128 where i64: From { u128::from(x.iter().fold(0, |m, &v| m.max(i64::from(v).unsigned_abs()))) } - -/// The integer inner products of the float pairs. Each computes at the narrowest item type that holds every value the product forms exactly: `f32` where none exceeds 2^24 in magnitude, `f64` where all lie below 2^51, `i64` where all fit it, and otherwise 128 bits through `wide`. `$bound` gives the largest magnitude that a product can form, from the largest magnitudes in `a` and `b` and from `k`. `checked_into` gives `false` where a result doesn't fit `i64`, and `inner` and `inner_into` panic there. -macro_rules! integers { - ($($object:ident: $bound:expr;)+) => {$( - impl Inner for $object { - type Out = i64; - fn inner_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) { - assert!(self.checked_into(a, b, size, out), "an integer inner product fits i64"); - } - fn checked_into(&self, a: &[i64], b: &[i64], size: [usize; 3], out: &mut [i64]) -> bool { - let bound = $bound(magnitude(a), magnitude(b), size[1] as u128); - if bound <= i64::MAX as u128 { tiered(self, a, b, size, bound, out); return true; } - wide::(a, b, size).map(|c| out.copy_from_slice(&c)).is_some() - } - } - )+}; -} -integers! { - PlusTimes: |a: u128, b: u128, k: u128| a.saturating_mul(b).saturating_mul(k); - MaxPlus: |a: u128, b: u128, _| a + b; - MinPlus: |a: u128, b: u128, _| a + b; -} - -/// `P`'s inner product of `a` and `b`, computed in 128 bits through `P`'s [`Pair`] at `i128`. `None` where a result doesn't fit `i64`, or where a running total doesn't fit 128 bits. -fn wide>>(a: &[i64], b: &[i64], [m, k, n]: [usize; 3]) -> Option> { - (0..m * n).map(|c| { - let (i, j) = (c / n, c % n); - let total = (0..k).fold(P::UNIT, |t, p| P::step(t, a[i * k + p].into(), b[p * n + j].into()))?; - i64::try_from(total).ok() - }).collect() -} - -/// `integers!`'s inner product of `pair`, whose values never exceed `bound` in magnitude, written into `out`. -fn tiered

(pair: &P, a: &[i64], b: &[i64], [m, k, n]: [usize; 3], bound: u128, out: &mut [i64]) where - P: Inner + Inner + Pair { - assert!(a.len() == m * k && b.len() == k * n, "`a` must hold `m`×`k` items and `b` `k`×`n`"); - if k == 0 { return out.fill(

>::UNIT); } - if exactly(pair, a, b, [m, k, n], bound, true, out, |x| x) { return; } - tiled_into::(a, b, [m, k, n], out, micro::) -} - -/// `pair`'s inner product of `a` and `b`, whose values never exceed `bound` in magnitude, in a float that holds each of them exactly: `f32` up to 2^24, and `f64` up to `WIDE_LIMIT` where `wide` holds. Each item reads through [`Source`], and each result, a whole number, converts from `i64` to the type of `out` through `whole`. `false`, with `out` untouched, where neither float holds every value. -fn exactly(pair: &P, a: &[S], b: &[S], size: [usize; 3], bound: u128, wide: bool, out: &mut [O], whole: impl Fn(i64) -> O) -> bool where - P: Inner + Inner, S: Source + Source { - fn at, S: Source, O>( - pair: &P, a: &[S], b: &[S], size: [usize; 3], out: &mut [O], whole: impl Fn(i64) -> O) -> bool { - for (o, x) in out.iter_mut().zip(pair.inner(&read_all(a), &read_all(b), size)) { *o = whole(convert::shifted(x.wide())) } - true - } - if bound <= 1 << 24 { at::(pair, a, b, size, out, whole) } - else { wide && bound <= convert::WIDE_LIMIT as u128 && at::(pair, a, b, size, out, whole) } -} - -/// `+.×` of `a` and `b` with wrapping `i32` steps, written into `out`, on the `i32` kernel. The kernel converts each item to `i32` through [`Source`] as it packs it. -fn times_i32_into>(a: &[S], b: &[S], size: [usize; 3], out: &mut [i32]) { - const MR: usize = tiles::TIMES32[0]; - const NR: usize = tiles::TIMES32[1]; - tiled_into::(a, b, size, out, times_i32::() / tiles::BYTES }>) -} - -/// [`Wrapped`] on `u8`, `i16` and `i32` items. Each result is its exact sum of products wrapped to 32 bits, as static kinds and fixed widths keep integer results. Nothing fails. [`Product::run_wrapped`] runs these products. `$largest` gives the largest magnitude that a product of two items can form: from the item type for `u8` and `i16`, and from the items' largest magnitudes for `i32`. Where `k` such products sum exactly in `f32`, the product computes in `f32` and wraps each whole result, as `integers!`'s products do. Where Accelerate serves the product, a sum that `f64` holds exactly computes in `f64` the same way. Every other product runs on the `i32` kernel, whose wrapping steps are exact to 32 bits whatever the items. On an M5 Max on 25 October 2026, against the `i32` kernel's time, a product of two 512×512 `u8` matrices took a fifth through Accelerate's `f64` product and 1.6 times as long through rustymath's own. A 512×256 by 256×512 product took 0.85 of the `i32` kernel's time through rustymath's own `f32` product. -macro_rules! wrapped_narrow { - ($($t:ty: $largest:expr;)+) => {$( - impl Inner<$t> for crate::Wrapped { - type Out = i32; - fn inner_into(&self, a: &[$t], b: &[$t], size @ [m, k, n]: [usize; 3], out: &mut [i32]) { - assert!(a.len() == m * k && b.len() == k * n && out.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `out` `m`×`n`"); - if m * k * n > DIRECT_WORK && exactly(&PlusTimes, a, b, size, $largest(a, b) * k as u128, accelerated(), out, |x| x as i32) { return } - times_i32_into(a, b, size, out) - } - } - )+}; -} -wrapped_narrow! { - u8: |_, _| 255 * 255; - i16: |_, _| 1 << 30; - i32: |a, b| magnitude(a) * magnitude(b); -} - -/// `+.×` of f16 matrices, summed and returned at f32. rustymath's own kernels convert each item to f32 as they pack it, so neither operand is copied whole. Accelerate has no f16 product, so where it serves the process, both operands convert to f32 copies for its f32 product. -impl Inner for PlusTimes { - type Out = f32; - fn inner_into(&self, a: &[half::f16], b: &[half::f16], size @ [m, k, n]: [usize; 3], c: &mut [f32]) { - assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); - if m * k * n <= DIRECT_WORK { return direct::(a, b, size, c); } - dispatch!(half_products_accelerated(a, b, size, c), half_products_native(a, b, size, c)) - } -} - -/// [`PlusTimes`] at f16 on Accelerate's f32 product, through f32 copies of both operands. -#[cfg(accelerate)] -fn half_products_accelerated(a: &[half::f16], b: &[half::f16], size @ [_, k, n]: [usize; 3], c: &mut [f32]) { - f32::gemm(c, n, Matrix::rows(&read_all(a), k), Matrix::rows(&read_all(b), n), size, 1.0, 0.0) -} - -/// [`PlusTimes`] at f16 on rustymath's own f32 kernels, which convert each item as they pack it. -#[cfg(native)] -fn half_products_native(a: &[half::f16], b: &[half::f16], size: [usize; 3], c: &mut [f32]) { - const MR: usize = tiles::TIMES32[0]; - const NR: usize = tiles::TIMES32[1]; - tiled_into::(a, b, size, c, times32::() / tiles::BYTES }>) -} - -/// A strided matrix borrowed in its stored, transposed, conjugated or adjoint orientation. -#[derive(Clone, Copy)] -pub(crate) struct Matrix<'a, T> { pub data: &'a [T], pub ld: usize, pub op: Op } -#[derive(Clone, Copy)] -pub(crate) enum Op { Normal, Transpose, Adjoint, Conjugate } -/// Whether an operand of [`Inner::inner_ops`] holds its matrix as stored or transposed. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Orientation { Normal, Transpose } -/// A row-major matrix whose items `mapping` reads from `data`, as [`expanded_items`] lays an argument over a frame. [`Inner::inner_mapped`] takes its operands this way. A matrix held in order has `Mapping::Linear(1)`. -#[derive(Clone, Copy)] -pub struct Mapped<'a, T> { pub data: &'a [T], pub mapping: &'a Walked<'a> } - -impl<'a, T: Element> Mapped<'a, T> { - /// The matrix in orientation `o`, with no copy, where the mapping reads its stored matrix at positive strides, one of them 1: row by row, or column by column as a transpose. `None` for any other mapping. - fn matrix(&self, o: Orientation, [rows, cols]: [usize; 2]) -> Option> { - let [r, c] = if o == Orientation::Normal { [rows, cols] } else { [cols, rows] }; - let (first, strides) = self.mapping.affine(&[r, c])?; - // An axis of length 1 reads one item, so any stride serves it. A single row whose columns are strided reads as a transposed column. - let sc = if c == 1 { 1 } else { strides[1] }; - let sr = if r > 1 { strides[0] } else if sc == 1 { c as isize } else { 1 }; - let data = self.data.get(first..)?; - let stored = match (sr, sc) { - (s, 1) if s >= c as isize => Matrix::rows(data, s as usize), (1, s) if s >= r as isize => Matrix { data, ld: s as usize, op: Op::Transpose }, - _ => return None - }; - Some(if o == Orientation::Normal { stored } else { stored.transpose() }) - } - /// Whether [`Inner::inner_mapped`] reads this matrix where it lies, in orientation `o` with `size` rows and columns, rather than one block at a time. - pub fn in_place(&self, o: Orientation, size: [usize; 2]) -> bool { self.matrix(o, size).is_some() } - /// Items `start..start + n` of the matrix, as [`expanded_items`] reads them. - pub(crate) fn read<'b>(&'b self, start: usize, n: usize, buf: &'b mut Vec) -> &'b [T] { expanded_items(self.mapping, self.data, start, n, buf) } -} -#[derive(Clone, Copy)] -pub(crate) enum Side { Left, Right } -/// The stored triangle, before applying the matrix view's transpose or adjoint. -#[derive(Clone, Copy)] -pub(crate) enum Triangle { Lower, UnitUpper } -impl Op { - #[inline(always)] - fn transposed(self) -> bool { matches!(self, Self::Transpose | Self::Adjoint) } -} -impl<'a, T: Copy> Matrix<'a, T> { - pub(crate) fn rows(data: &'a [T], ld: usize) -> Self { Self { data, ld, op: Op::Normal } } - pub(crate) fn transpose(self) -> Self { - Self { op: match self.op { - Op::Normal => Op::Transpose, Op::Transpose => Op::Normal, Op::Adjoint => Op::Conjugate, Op::Conjugate => Op::Adjoint - }, ..self } - } - /// Starts at a row and column in the logical orientation. - pub(crate) fn offset(self, row: usize, col: usize) -> Self { - let at = if self.op.transposed() { col * self.ld + row } else { row * self.ld + col }; - Self { data: &self.data[at..], ..self } - } - /// The `rows`×`cols` matrix that an operand of [`Inner::inner_ops`] gives in orientation `o`. A transposed operand stores the `cols`×`rows` matrix, row-major. - pub(crate) fn oriented(data: &'a [T], o: Orientation, [rows, cols]: [usize; 2]) -> Self { - match o { Orientation::Normal => Self::rows(data, cols), Orientation::Transpose => Self { data, ld: rows, op: Op::Transpose } } - } - /// Checks the stored extent once, before any foreign call. - fn check(self, mut rows: usize, mut cols: usize) { - if self.op.transposed() { std::mem::swap(&mut rows, &mut cols) } - if rows == 0 || cols == 0 { return; } - let end = (rows - 1).checked_mul(self.ld).and_then(|n| n.checked_add(cols)); - assert!(self.ld >= cols && end.is_some_and(|n| n <= self.data.len()), "matrix view must hold its rows and columns"); - } -} -impl Matrix<'_, T> { - pub(crate) fn conjugate(self) -> Self { - // Conjugation is an identity for real scalars; don't make their BLAS views more restrictive. - if !T::COMPLEX { return self; } - Self { op: match self.op { - Op::Normal => Op::Conjugate, Op::Transpose => Op::Adjoint, Op::Adjoint => Op::Transpose, Op::Conjugate => Op::Normal - }, ..self } - } - pub(crate) fn adjoint(self) -> Self { self.transpose().conjugate() } - #[cfg(native)] - #[inline(always)] - pub(crate) fn at(self, i: usize, j: usize) -> T { - let x = self.data[if self.op.transposed() { j * self.ld + i } else { i * self.ld + j }]; - if matches!(self.op, Op::Adjoint | Op::Conjugate) { x.conj() } else { x } - } - /// Writes a packing strip of this view, times `scale`, as the free function `strip` does: item `[p][l]` is item `(r0 + l, p0 + p)`. A product packs strips of `a` from `a`, and of `b` from `b`'s transpose. - #[cfg(native)] - #[inline(always)] - fn strip(self, rows: [usize; 2], p0: usize, scale: T, out: &mut [[MaybeUninit; W]]) where T: Default { - let (transposed, one) = (self.op.transposed(), scale == T::one()); - if !T::COMPLEX || !matches!(self.op, Op::Adjoint | Op::Conjugate) { - if one { strip(self.data, self.ld, transposed, rows, p0, out, |x| x) } else { strip(self.data, self.ld, transposed, rows, p0, out, |x| scale * x) } - } else if one { strip(self.data, self.ld, transposed, rows, p0, out, |x: T| x.conj()) } - else { strip(self.data, self.ld, transposed, rows, p0, out, |x: T| scale * x.conj()) } - } -} -/// Writes all `W` items of each step of a packing strip, through `f`. Item `[p][l]` is item `(r0 + l, p0 + p)` of the matrix that `data` holds at `r * ld + p`, or at `p * ld + r` where `transposed`, for `l` below `lines`, and the default after that. Transposed data gives each step from one run, and other data from `lines` rows read side by side. Every step writes a fixed `W` items, which keeps short strips from calling `memcpy` or `memset` at each step. -#[inline(always)] -fn strip( - data: &[S], ld: usize, transposed: bool, [r0, lines]: [usize; 2], p0: usize, out: &mut [[MaybeUninit; W]], f: impl Fn(S) -> T) { - assert!(lines <= W); - let kc = out.len(); - let rows: [&[S]; W] = std::array::from_fn(|l| if transposed || l >= lines { &[] } else { &data[(r0 + l) * ld + p0..][..kc] }); - match (transposed, lines == W) { - (true, true) => { - for (p, step) in out.iter_mut().enumerate() { - let run: &[S; W] = data[(p0 + p) * ld + r0..].first_chunk().expect("a whole step lies in `data`"); - for l in 0..W { step[l] = MaybeUninit::new(f(run[l])) } - } - } - (true, false) => { - for (p, step) in out.iter_mut().enumerate() { - let run = &data[(p0 + p) * ld + r0..][..lines]; - for l in 0..W { step[l] = MaybeUninit::new(if l < lines { f(run[l]) } else { T::default() }) } - } - } - (false, true) => { for (p, step) in out.iter_mut().enumerate() { for l in 0..W { step[l] = MaybeUninit::new(f(rows[l][p])) } } } - (false, false) => { - for (p, step) in out.iter_mut().enumerate() { for l in 0..W { step[l] = MaybeUninit::new(if l < lines { f(rows[l][p]) } else { T::default() }) } } - } - } -} -/// Item types whose matrix products run on the tiled `+.×` kernels: `f32` and `f64`, and complex numbers with parts of either type. The decompositions in `factor` build on them. -pub(crate) trait Tiled: Scalar + Default + Vendor { - /// The row-major matrix product, allocated once through the same GEMM operation as updates. - #[inline] - fn multiply(a: Matrix<'_, Self>, b: Matrix<'_, Self>, size: [usize; 3]) -> Vec { - let mut c = filled(size[0] * size[2], Self::zero()); - Self::gemm(&mut c, size[2], a, b, size, Self::one(), Self::zero()); - c - } - /// `c = alpha a b + beta c`, with `size = [m, k, n]`. Dispatch precedes native packing and row splitting. - #[inline] - fn gemm(c: &mut [Self], ldc: usize, a: Matrix<'_, Self>, b: Matrix<'_, Self>, size: [usize; 3], alpha: Self, beta: Self) { - gemm(c, ldc, a, b, size, alpha, beta, Part::Whole) - } - /// The lower triangle of `c = alpha x xᴴ + beta c`, for `x` of size `[n, k]`. Native diagonal tiles may also change upper items. - #[inline] - fn gram(c: &mut [Self], ldc: usize, x: Matrix<'_, Self>, [n, k]: [usize; 2], alpha: Self::Real, beta: Self::Real) { - gemm(c, ldc, x, x.adjoint(), [n, k, n], Self::from_real(alpha), Self::from_real(beta), Part::Lower) - } - /// Solves `a x = b` or `x a = b` in place, with `triangle` describing the stored square matrix `a`. - fn triangular(x: &mut [Self], ldx: usize, a: Matrix<'_, Self>, [m, n]: [usize; 2], side: Side, triangle: Triangle) { - let order = match side { Side::Left => m, Side::Right => n }; - a.check(order, order); - Matrix::rows(x, ldx).check(m, n); - if m == 0 || n == 0 { return; } - dispatch!(accelerate::triangular(x, ldx, a, [m, n], side, triangle), factor::triangular_native(x, ldx, a, [m, n], side, triangle)) - } - /// `alpha a b` on rustymath's own packed kernels, written over `c` where `start` is true and added to it otherwise. Complex items run four real products through `complex_product`. - #[cfg(native)] - fn native_product(c: &mut [Self], ldc: usize, a: Matrix<'_, Self>, alpha: Self, b: Matrix<'_, Self>, size: [usize; 3], start: bool, part: Part); - /// The thin SVD of the finite, nonempty `m`×`n` matrix `a` as `(U, s, V)`, with its singular vectors only where `vectors` is true, or `None` where it doesn't converge. - fn svd(a: &[Self], size: [usize; 2], vectors: bool) -> Option<(Vec, Vec, Vec)>; - /// The eigen decomposition of the nonempty Hermitian `n`×`n` matrix whose finite lower triangle `a` holds, with its eigenvectors only where `vectors` is true, or `None` where it doesn't converge. - fn eigh(a: &[Self], n: usize, vectors: bool) -> Option>; - /// The eigen decomposition of the finite, nonempty `n`×`n` matrix `a`, with its eigenvectors only where `vectors` is true, or `None` where it doesn't converge. - fn eig(a: &[Self], n: usize, vectors: bool) -> Option>; -} - -fn gemm(c: &mut [T], ldc: usize, a: Matrix<'_, T>, b: Matrix<'_, T>, [m, k, n]: [usize; 3], alpha: T, beta: T, part: Part) { - a.check(m, k); - b.check(k, n); - Matrix::rows(c, ldc).check(m, n); - if m == 0 || n == 0 { return; } - if k == 0 || alpha == T::zero() { return scale(c, ldc, [m, n], beta, part); } - if by_vector(c, ldc, a, alpha, b, [m, k, n], beta, part) { return; } - dispatch!(accelerate::product(c, ldc, a, b, [m, k, n], alpha, beta, part), native_gemm(c, ldc, a, b, [m, k, n], alpha, beta, part)) -} - -/// The fewest items that [`blocked`] reads at a time from an operand that isn't held in order, rounded to whole stored rows. A block this size stays in cache while its product runs. -const BLOCK_ITEMS: usize = 1 << 18; -/// The most items of `b` for which [`blocked`] gives each thread its own piece of the result's rows, multiplied a block at a time. Each thread's product packs its own copy of `b`, and larger copies crowd the cache that the threads share. On one thread, no other copy of `b` shares the cache. A product there takes pieces whatever the size of `b`. With threads on an M5 Max, for a strided 4096×2048 view times a 2048×n matrix (28 October 2026), pieces took 3% to 14% less time than one copy and one product at 64 columns, 2^17 items, and up to 37% more from 128 columns on rustymath's own kernels. -const PIECE_MATRIX: usize = 1 << 17; -/// The most items of an operand that isn't held in order that [`blocked`] reads whole, where `b` is too large for [`PIECE_MATRIX`]. A larger operand is read in blocks of this many items, rounded up to whole rows, each multiplied in one threaded call. With threads on an M5 Max (28 October 2026), products of 2048 and 4096 rows of 2048 items took within 1% as long for each row at 512 columns, and 1024 rows took 4% longer on Accelerate and 8% longer on rustymath's own kernels. -const WHOLE_ITEMS: usize = 1 << 24; - -/// The stored rows of `cols` items in each block that [`blocks`] reads: at least `least` items and at least [`BLOCK_ITEMS`], in whole rows. -fn block_rows(cols: usize, least: usize) -> usize { least.max(BLOCK_ITEMS).div_ceil(cols) } - -/// Calls `f` on each block of an operand's stored rows `rows`, each row of `cols` items, with the block's first row, its number of rows and its items. Each block holds [`block_rows`] rows, apart from the last. -fn blocks(op: Mapped<'_, T>, rows: std::ops::Range, cols: usize, least: usize, mut f: impl FnMut(usize, usize, &[T])) { - let (step, mut block) = (block_rows(cols, least), Vec::new()); - for first in rows.clone().step_by(step) { - let len = step.min(rows.end - first); - f(first, len, op.read(first * cols, len * cols, &mut block)); - } -} - -/// Sets the result `c` to the sum of the products of each block of an operand's stored rows, `[rows, cols]` as stored, where those rows run along the contraction. `f(first, len, block, c, beta)` sets `c` to the product of the block of `len` rows from `first` plus `beta` times `c`. The block at row 0 replaces what `c` held, with `beta` 0. Every other block adds, with `beta` 1. A result of at most [`BLOCK_ITEMS`] items splits the rows into one piece of at least one block for each thread ([`parallel::summed`]). Each thread reads its rows in blocks of [`BLOCK_ITEMS`] items into its own copy of the result, and the copies then add into `c`. A larger result takes blocks of at least as many items as it holds, in turn. -fn contracted(op: Mapped<'_, T>, [rows, cols]: [usize; 2], c: &mut [T], f: &(dyn Fn(usize, usize, &[T], &mut [T], T) + Sync)) { - let beta = |first: usize| if first == 0 { T::zero() } else { T::one() }; - if c.len() > BLOCK_ITEMS { return blocks(op, 0..rows, cols, c.len(), |first, len, block| f(first, len, block, c, beta(first))); } - parallel::summed(rows, block_rows(cols, 0), c, &mut Vec::new(), &|range, sums| { - blocks(op, range, cols, 0, |first, len, block| f(first, len, block, sums, beta(first))) - }); -} - -/// The most rows of the result in one piece of [`by_columns`]'s split across threads. -const COLUMN_ROWS: usize = 4096; -/// The results that [`scaled_runs`] keeps in registers while every column adds into them. -const COLUMN_STRIP: usize = 32; -/// The results in each strip that [`scaled_runs`] takes after its strips of [`COLUMN_STRIP`], before it takes one result at a time. The windows of an image 256 items wide leave 30 results in each row after its strips of 32. -const TAIL_STRIP: usize = 8; -/// The fewest adjacent rows of `a` that [`by_columns`] reads at once. Shorter runs cost more in loop overhead than gathering saves. -const COLUMN_RUN: usize = 64; - -/// `+.×` of `a` and a vector `b`, where each row of `a` starts one item after the row before it in storage, as the windows of a vector or a single-channel image do. Each column of `a` is then a run of storage, which [`scaled_runs`] reads where it lies and scales by its item of `b`. `a` is never gathered. `None` where it doesn't split into entries for its rows and entries for its columns, or where its rows aren't adjacent in runs of at least [`COLUMN_RUN`]. The loop reads a fill as 0, the item type's own fill. -fn by_columns(a: Mapped<'_, T>, b: Mapped<'_, T>, oa: Orientation, [m, k]: [usize; 2]) -> Option> { - use crate::gather::Steps; - let Mapping::Steps { base, steps } = &**a.mapping else { return None }; - if m * k == 0 { return None; } - // The entries of the stored matrix's leading axis come first, and give its first `lead` items. - let lead = if oa == Orientation::Normal { m } else { k }; - let (mut split, mut size) = (0, 1); - while size < lead && split < steps.len() { - size *= steps[split].len(); - split += 1; - } - if size != lead { return None; } - let (outer, inner) = steps.split_at(split); - let (rows, cols) = if oa == Orientation::Normal { (outer, inner) } else { (inner, outer) }; - let Steps::Stride { step: 1, len: width, .. } = *rows.iter().rev().find(|s| s.len() > 1)? else { return None }; - if width < COLUMN_RUN { return None; } - let mut whole = Vec::new(); - let weights = b.read(0, k, &mut whole); - let columns: Vec<(T, usize)> = (0..k).filter_map(|p| Some((weights[p], Steps::reach(0, cols, p)?))).collect(); - let mut c = filled(m, T::zero()); - parallel::split(&mut c, COLUMN_ROWS, &|first, part| { - let mut r = first; - while r < first + part.len() { - let len = (width - r % width).min(COLUMN_ROWS).min(first + part.len() - r); - // A row whose outer entries give a fill reads only fills, and its result stays 0. - if let Some(at) = Steps::reach(*base, rows, r) { scaled_runs(&mut part[r - first..r - first + len], a.data, at, &columns, true) } - r += len; - } - true - }); - Some(c) -} - -/// For adjacent rows of a matrix whose first row starts at `data[at]` and each next row one item later, adds to each of `out`'s results the sum over `columns` of each column's weight times the row's item at the column's offset. Where `start` is true, it writes the sums in place of `out`'s results. Strips of [`COLUMN_STRIP`] results, then of [`TAIL_STRIP`], stay in registers while every column adds into them ([`scaled_strips`]). The rest take one result at a time. -fn scaled_runs(out: &mut [T], data: &[T], at: usize, columns: &[(T, usize)], start: bool) { - let wide = scaled_strips::(out, data, at, columns, start); - let done = wide + scaled_strips::(&mut out[wide..], data, at + wide, columns, start); - let at = at + done; - for (t, y) in out[done..].iter_mut().enumerate() { - *y = columns.iter().fold(if start { T::zero() } else { *y }, |acc, &(w, o)| w.mul_acc(data[at + o + t], acc)) - } -} - -/// [`scaled_runs`] for the whole strips of `N` results at the start of `out`, each kept in registers while every column adds into it. Gives how many results the strips hold. -fn scaled_strips(out: &mut [T], data: &[T], at: usize, columns: &[(T, usize)], start: bool) -> usize { - let (strips, _) = out.as_chunks_mut::(); - for (s, strip) in strips.iter_mut().enumerate() { - let (at, mut acc) = (at + s * N, if start { [T::zero(); N] } else { *strip }); - for &(w, o) in columns { let x = &data[at + o..][..N]; for t in 0..N { acc[t] = w.mul_acc(x[t], acc[t]) } } - *strip = acc; - } - strips.len() * N -} - -/// Writes `out`'s results for stored rows `first..` of a matrix whose rows start `ld` items apart in `data`: each the sum of products of its row with `x`. Four rows at a time share each read of `x`, each row in eight running sums, which vectorize and then add pairwise. -fn dots(out: &mut [T], data: &[T], ld: usize, first: usize, x: &[T]) { - let k = x.len(); - let (quads, tail) = out.as_chunks_mut::<4>(); - let (xc, xr) = x.as_chunks::<8>(); - let sum = |s: &[T; 8]| ((s[0] + s[4]) + (s[2] + s[6])) + ((s[1] + s[5]) + (s[3] + s[7])); - for (q, quad) in quads.iter_mut().enumerate() { - let rows: [&[T]; 4] = std::array::from_fn(|r| &data[(first + 4 * q + r) * ld..][..k]); - let mut s = [[T::zero(); 8]; 4]; - for (c, xs) in xc.iter().enumerate() { for r in 0..4 { let a = &rows[r][c * 8..][..8]; for l in 0..8 { s[r][l] = a[l].mul_acc(xs[l], s[r][l]) } } } - let at = xc.len() * 8; - for r in 0..4 { quad[r] = xr.iter().enumerate().fold(sum(&s[r]), |t, (p, &b)| rows[r][at + p].mul_acc(b, t)) } - } - let first = first + 4 * quads.len(); - for (i, y) in tail.iter_mut().enumerate() { *y = factor::dot(&data[(first + i) * ld..][..k], x) } -} - -/// The most items in the stored rows of a matrix that [`by_vector`] takes where Accelerate serves the process. Accelerate pays a fixed cost for each stored row. [`narrow_sums`] takes up to this many results, and [`narrow_dots`] stored rows of up to this many items. -const NARROW: usize = 8; - -/// [`dots`] for stored rows of `K` items, at most [`NARROW`], where `x` holds as many. Each row's products add into two running sums. Each pair of rows then adds its sums pairwise into both results in one vector step. -fn narrow_dots(out: &mut [T], data: &[T], ld: usize, first: usize, x: &[T]) { - let x: &[T; K] = x.try_into().expect("`x` must hold `K` items"); - let sums = |i: usize| { - let a = &data[i * ld..][..K]; - let mut s = [T::zero(); 2]; - for l in 0..K { s[l % 2] = a[l].mul_acc(x[l], s[l % 2]) } - s - }; - let last = first + out.len().saturating_sub(1); - let (pairs, odd) = out.as_chunks_mut::<2>(); - for (p, pair) in pairs.iter_mut().enumerate() { - let (s, t) = (sums(first + 2 * p), sums(first + 2 * p + 1)); - *pair = [s[0] + s[1], t[0] + t[1]]; - } - if let [y] = odd { let s = sums(last); *y = s[0] + s[1] } -} - -/// Writes each of `out`'s results, at most [`NARROW`], as the sum over the stored rows of `data`, which start `ld` items apart, of the row's item in `w` times its item at the result's position: a vector times the matrix whose columns those rows hold. Every sum stays in registers across all the rows, in four sets that each take every fourth row. A row's multiply-adds therefore don't wait on the row before. Each row adds [`NARROW`] items at once into its set, and the items past the results, which belong to the next row, reach no result. The last rows, whose [`NARROW`] items would pass the end of `data`, add one result at a time. -fn narrow_sums(out: &mut [T], data: &[T], ld: usize, w: &[T]) { - let n = out.len(); - let whole = if data.len() < NARROW { 0 } else { ((data.len() - NARROW) / ld.max(1) + 1).min(w.len()) }; - let mut acc = [[T::zero(); NARROW]; 4]; - let (quads, rest) = w[..whole].as_chunks::<4>(); - for (q, ws) in quads.iter().enumerate() { - for s in 0..4 { let a = &data[(4 * q + s) * ld..][..NARROW]; for t in 0..NARROW { acc[s][t] = a[t].mul_acc(ws[s], acc[s][t]) } } - } - for (p, &wp) in rest.iter().enumerate() { - let a = &data[(4 * quads.len() + p) * ld..][..NARROW]; - for t in 0..NARROW { acc[0][t] = a[t].mul_acc(wp, acc[0][t]) } - } - for (p, &wp) in w.iter().enumerate().skip(whole) { for t in 0..n { acc[0][t] = data[p * ld + t].mul_acc(wp, acc[0][t]) } } - for (t, y) in out.iter_mut().enumerate() { *y = (acc[0][t] + acc[2][t]) + (acc[1][t] + acc[3][t]) } -} - -/// `+.×` of mapped operands on the tiled kernels, for [`Inner::inner_mapped`]. An operand whose mapping reads it at positive strides, one of them 1, is used where it lies, row by row or as a transpose. Any other operand is read a block of its stored rows at a time, and each block's product goes straight into the result. Where `a`'s stored rows are the result's rows, and the product runs on one thread or `b` has at most [`PIECE_MATRIX`] items, the result's rows split into one piece for each thread. Each thread multiplies its piece's rows itself, reading them in blocks that hold at least [`BLOCK_ITEMS`] items and at least as many as `b`. Otherwise an operand of up to [`WHOLE_ITEMS`] items is read whole, and a larger one in blocks of at least that many. Each block is gathered across threads and multiplied in one threaded call. Rows along the contraction add into the whole result, as [`contracted`] reads them. Blocks of the result's columns hold at least as many items as the result. Where neither operand can be used where it lies, `b` is read whole. A vector `b` beside an `a` whose rows are adjacent in storage goes to [`by_columns`], which reads `a` where it lies. Reading an operand larger than the result through its map, by blocks or by columns, records the route `mapped product`. -fn blocked(a: Mapped<'_, T>, b: Mapped<'_, T>, [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3]) -> Vec { - let (ma, mb) = (a.matrix(oa, [m, k]), b.matrix(ob, [k, n])); - if let (Some(a), Some(b)) = (ma, mb) { return T::multiply(a, b, [m, k, n]); } - if ma.is_none() && n == 1 && let Some(c) = by_columns(a, b, oa, [m, k]) { crate::events::taken(crate::events::Route::MappedProduct, m * k); return c; } - let mut c = filled(m * n, T::zero()); - if m == 0 || k == 0 || n == 0 { return c; } - let read = if ma.is_none() { m * k } else { k * n }; - if read > m * n { crate::events::taken(crate::events::Route::MappedProduct, read); } - let (one, zero) = (T::one(), T::zero()); - let mut whole = Vec::new(); - match ma { - None => { - let b = match mb { Some(b) => b, None => Matrix::oriented(b.read(0, k * n, &mut whole), ob, [k, n]) }; - match oa { - Orientation::Normal => { - let (rows, least) = - if parallel::threads() == 1 || k * n <= PIECE_MATRIX { (m.div_ceil(parallel::threads()), k * n) } else { (m, WHOLE_ITEMS) }; - parallel::pieces_of(&mut c, rows * n, usize::MAX, &|start, part| { - let from = start / n; - blocks(a, from..from + part.len() / n, k, least, |first, len, block| { - T::gemm(&mut part[(first - from) * n..], n, Matrix::rows(block, k), b, [len, k, n], one, zero) - }); - true - }); - } - Orientation::Transpose => contracted(a, [k, m], &mut c, &|first, len, part, c, beta| { - T::gemm(c, n, Matrix::oriented(part, oa, [m, len]), b.offset(first, 0), [m, len, n], one, beta) - }), - } - } - Some(a) => match ob { - Orientation::Normal => { - contracted(b, [k, n], &mut c, &|first, len, part, c, beta| T::gemm(c, n, a.offset(0, first), Matrix::rows(part, n), [m, len, n], one, beta)) - } - Orientation::Transpose => { - blocks(b, 0..n, k, m * n, |first, len, block| T::gemm(&mut c[first..], n, a, Matrix::oriented(block, ob, [k, len]), [m, k, len], one, zero)) - } - }, - } - c -} - -/// `c = beta c` for the items of `c` in `part`, where `c` is an `m`×`n` matrix whose rows start `ldc` items apart. A zero `beta` writes zeros, whatever `c` held. -fn scale(c: &mut [T], ldc: usize, [m, n]: [usize; 2], beta: T, part: Part) { - if beta == T::one() { return; } - for i in 0..m { - for j in part.columns(i, n) { - let x = &mut c[i * ldc + j]; - *x = if beta == T::zero() { T::zero() } else { beta * *x }; - } - } -} - -/// The multiply-adds in each unit of [`by_vector`]'s split across threads. `parallel::split` times the first unit to choose the pieces. -const VECTOR_WORK: usize = 1 << 15; -/// The results that [`by_vector`] keeps in the first-level cache while the stored rows of a transposed matrix add into them. -const VECTOR_STRIP: usize = 2048; -/// The stored rows of a transposed matrix that [`by_vector`] adds into its results together, read side by side in storage order. -const VECTOR_ROWS: usize = 8; -/// The fewest multiply-adds in each piece of [`by_vector`]'s split of stored rows across threads: twice what waking one more thread costs ([`THREAD_WORK`]), so that every split gains. At 4 threads on an M5 Max, a split cost about 10 to 12 µs whether it made two pieces or four, and pieces of 2^15 to 2^17 made a vector times a matrix of 1024 columns up to 76% slower than one piece where their work came to less than that. With pieces of 2^18, 128 and 256 rows ran as one piece, and 512 and 1024 rows ran 20 and 28% faster than before in two and four pieces. -const VECTOR_PIECE: usize = 2 * THREAD_WORK; - -/// `alpha a b + beta c`, where `m` or `n` is 1, for a nonzero `k` and `alpha`. The packed kernels would pack the whole matrix and pad the vector to a tile. As a matrix times a vector, `a b` or the transpose of `bᵀ aᵀ`, each result is the sum of products of a stored row with the vector where the matrix's stored rows run along the contraction: [`narrow_dots`] for stored rows of up to [`NARROW`] items, and [`dots`] for longer ones. Otherwise each result reads one item of every stored row. At most [`NARROW`] results keep their sums in registers across every stored row ([`narrow_sums`]). More results take strips of [`VECTOR_STRIP`]. [`scaled_runs`] adds [`VECTOR_ROWS`] stored rows at a time into each strip, in storage order. The results split across threads, or, where they fit one strip of a transposed matrix, the stored rows do, each piece adding into results of its own. The vector is read where it lies where its items are adjacent and need no conjugate, and the results are written straight into `c` where they need no scaling, conjugate or step. `false` for any other size and for `Part::Lower`. Accelerate multiplies longer stored rows faster: where it serves the process, `by_vector` gives `false` for stored rows of more than [`NARROW`] items. -fn by_vector(c: &mut [T], ldc: usize, a: Matrix<'_, T>, alpha: T, b: Matrix<'_, T>, [m, k, n]: [usize; 3], beta: T, part: Part) -> bool { - let (matrix, vector, rows, step) = match (m, n, part) { - (_, 1, Part::Whole) => (a, b.transpose(), m, ldc), (1, _, Part::Whole) => (b.transpose(), a, n, 1), _ => return false - }; - let narrow = if matrix.op.transposed() { rows } else { k } <= NARROW; - if accelerated() && !narrow { return false; } - // A conjugated matrix's results are the conjugates of its stored items' results with the conjugated vector. - let conj = matches!(matrix.op, Op::Adjoint | Op::Conjugate); - let flip = T::COMPLEX && conj != matches!(vector.op, Op::Adjoint | Op::Conjugate); - let stride = if vector.op.transposed() { vector.ld } else { 1 }; - let x: Cow<'_, [T]> = if stride == 1 && !flip { Cow::Borrowed(&vector.data[..k]) } - else { (0..k).map(|p| if flip { vector.data[p * stride].conj() } - else { vector.data[p * stride] }).collect() }; - let (data, ld, unit) = (matrix.data, matrix.ld, (VECTOR_WORK / k).max(1).next_multiple_of(4)); - let (one, start) = (alpha == T::one(), beta == T::zero()); - let direct = step == 1 && one && start && !conj; - let mut kept = if direct { Vec::new() } else { filled(rows, T::zero()) }; - let y = if direct { &mut c[..rows] } else { &mut kept[..] }; - if matrix.op.transposed() { - // Stored row `p` holds the matrix's column `p`, so the matrix's row `i` starts at `data[i]`. Results that fit one strip leave each thread a narrow part of every stored row, so the stored rows split instead. Each piece writes its rows' sums into results of its own, which add into `y` ([`parallel::summed`]). - let piece = (VECTOR_PIECE / rows).max(1); - if narrow { - parallel::summed(k, piece, y, &mut Vec::new(), &|range, part| narrow_sums(part, &data[range.start * ld..], ld, &x[range])); - } else { - let columns: Vec<(T, usize)> = x.iter().enumerate().map(|(p, &w)| (w, p * ld)).collect(); - if rows <= VECTOR_STRIP { - parallel::summed(columns.len(), piece, y, &mut Vec::new(), &|range, part| { - for (i, block) in columns[range].chunks(VECTOR_ROWS).enumerate() { scaled_runs(part, data, 0, block, i == 0) } - }); - } else { - parallel::split(y, unit, &|first, part| { - for (s, strip) in part.chunks_mut(VECTOR_STRIP).enumerate() { - for (i, block) in columns.chunks(VECTOR_ROWS).enumerate() { scaled_runs(strip, data, first + s * VECTOR_STRIP, block, i == 0) } - } - true - }); - } - } - } else { - let dots = match k { - 1 => narrow_dots::, 2 => narrow_dots::, 3 => narrow_dots::, 4 => narrow_dots::, 5 => narrow_dots::, - 6 => narrow_dots::, 7 => narrow_dots::, 8 => narrow_dots::, _ => dots:: - }; - parallel::split(y, unit, &|first, part| { dots(part, data, ld, first, &x); true }); - } - if direct { return true; } - // As the packed kernels do, `alpha` scales only where it isn't 1, which keeps a complex result's infinite part from making NaN. - for (i, &y) in kept.iter().enumerate() { - let y = if conj { y.conj() } else { y }; - let (y, z) = (if one { y } else { alpha * y }, &mut c[i * step]); - *z = if start { y } else { beta.mul_acc(*z, y) }; - } - true -} - -/// `gemm` on rustymath's own packed kernels, for a nonzero `k` and `alpha`. -#[cfg(native)] -fn native_gemm(c: &mut [T], ldc: usize, a: Matrix<'_, T>, b: Matrix<'_, T>, [m, k, n]: [usize; 3], alpha: T, beta: T, part: Part) { - let start = beta == T::zero(); - if !start { scale(c, ldc, [m, n], beta, part) } - T::native_product(c, ldc, a, alpha, b, [m, k, n], start, part) -} -/// The items of a product's result that it must fold: all of them, or those on and below its diagonal. -#[derive(Clone, Copy)] -pub(crate) enum Part { Whole, Lower } -impl Part { - /// The columns of row `i` that this part holds, in a row of `n` items. - #[inline(always)] - fn columns(self, i: usize, n: usize) -> std::ops::Range { match self { Part::Whole => 0..n, Part::Lower => 0..(i + 1).min(n) } } -} -macro_rules! tiled { - (@real $t:ty, $tile:ident, $kernel:ident; $c:ident, $ldc:ident, $a:ident, $alpha:ident, $b:ident, $size:ident, $start:ident, $part:ident) => {{ - const MR: usize = tiles::$tile[0]; - const NR: usize = tiles::$tile[1]; - let one = <$t>::one(); - product::<$t, PlusTimes, MR, NR>($c, $ldc, |rows, p0, out| $a.strip(rows, p0, $alpha, out), |columns, p0, out| $b.transpose().strip(columns, p0, one, out), $size, $start, $part, $kernel::() / tiles::BYTES }>) - }}; - (@complex $r:ty; $c:ident, $ldc:ident, $a:ident, $alpha:ident, $b:ident, $size:ident, $start:ident, $part:ident) => { - complex_product::<$r>($c, $ldc, $a, $alpha, $b, $size, $start, $part) - }; - (@eig real; $a:ident, $n:ident, $vectors:ident) => { from_order!($n, accelerate::GEEV_FROM, accelerate::eig($a, $n, $vectors), factor::general::native($a, $n, $vectors)) }; - (@eig complex; $a:ident, $n:ident, $vectors:ident) => { factor::general::native($a, $n, $vectors) }; - ($($t:ty: $kind:ident($($args:tt)*);)+) => {$( - impl Tiled for $t { - #[cfg(native)] - #[inline] - fn native_product(c: &mut [$t], ldc: usize, a: Matrix<'_, $t>, alpha: $t, b: Matrix<'_, $t>, size: [usize; 3], start: bool, part: Part) { - tiled!(@$kind $($args)*; c, ldc, a, alpha, b, size, start, part) - } - fn svd(a: &[$t], size: [usize; 2], vectors: bool) -> Option<(Vec<$t>, Vec<<$t as Scalar>::Real>, Vec<$t>)> { - from_order!(size[0].max(size[1]), accelerate::GESDD_FROM, accelerate::svd(a, size, vectors), factor::svd::native(a, size, vectors)) - } - fn eigh(a: &[$t], n: usize, vectors: bool) -> Option> { - from_order!(n, accelerate::HEEVD_FROM, accelerate::eigh(a, n, vectors), factor::eigen::native(a, n, vectors)) - } - fn eig(a: &[$t], n: usize, vectors: bool) -> Option::Real>> { - tiled!(@eig $kind; a, n, vectors) - } - } - impl Inner<$t> for PlusTimes { - type Out = $t; - fn inner_ops(&self, a: &[$t], b: &[$t], [oa, ob]: [Orientation; 2], [m, k, n]: [usize; 3]) -> Vec<$t> { - assert!(a.len() == m * k && b.len() == k * n, "`a` must hold `m`×`k` items and `b` `k`×`n`"); - <$t>::multiply(Matrix::oriented(a, oa, [m, k]), Matrix::oriented(b, ob, [k, n]), [m, k, n]) - } - #[inline] - fn inner_into(&self, a: &[$t], b: &[$t], [m, k, n]: [usize; 3], c: &mut [$t]) { - assert!(a.len() == m * k && b.len() == k * n && c.len() == m * n, "`a` must hold `m`×`k` items, `b` `k`×`n` and `c` `m`×`n`"); - if m * k * n <= DIRECT_WORK { return direct::<$t, $t, PlusTimes>(a, b, [m, k, n], c) } - <$t>::gemm(c, n, Matrix::rows(a, k), Matrix::rows(b, n), [m, k, n], <$t>::one(), <$t>::zero()) - } - fn inner_mapped(&self, a: Mapped<'_, $t>, b: Mapped<'_, $t>, orientations: [Orientation; 2], size: [usize; 3]) -> Vec<$t> { blocked(a, b, orientations, size) } - } - )+}; -} -tiled! { - f32: real(f32, TIMES32, times32); - f64: real(f64, TIMES64, times64); - Complex: complex(f32); - Complex: complex(f64); -} -/// `alpha a b`, from four real products of the parts: real parts `aᵣbᵣ - aᵢbᵢ` and imaginary parts `aᵣbᵢ + aᵢbᵣ`, as scalar multiplication forms them. `parts` splits each operand once, before the real products. -#[cfg(native)] -fn complex_product( - c: &mut [Complex], ldc: usize, a: Matrix<'_, Complex>, alpha: Complex, b: Matrix<'_, Complex>, size: [usize; 3], start: bool, part: Part) { - let [m, k, n] = size; - if m == 0 || k == 0 || n == 0 { return; } - let ([ar, ai], [br, bi]) = (parts(a, [m, k], alpha), parts(b, [k, n], Complex::new(R::one(), R::zero()))); - let real = |c: &mut [R], x: &[R], y: &[R], scale: R, start: bool| R::native_product(c, n, Matrix::rows(x, k), scale, Matrix::rows(y, n), size, start, part); - let (mut re, mut im) = (filled(m * n, R::zero()), filled(m * n, R::zero())); - real(&mut re, &ar, &br, R::one(), true); - real(&mut re, &ai, &bi, -R::one(), false); - real(&mut im, &ar, &bi, R::one(), true); - real(&mut im, &ai, &br, R::one(), false); - for i in 0..m { - for j in part.columns(i, n) { - let (t, x) = (i * n + j, &mut c[i * ldc + j]); - *x = if start { Complex::new(re[t], im[t]) } else { Complex::new(x.re + re[t], x.im + im[t]) }; - } - } -} - -/// The real parts and the imaginary parts of `scale` times the `rows`×`cols` matrix that `x` views, each row by row. -#[cfg(native)] -fn parts(x: Matrix<'_, Complex>, [rows, cols]: [usize; 2], scale: Complex) -> [Vec; 2] { - let (one, conj) = (scale == Complex::new(R::one(), R::zero()), matches!(x.op, Op::Adjoint | Op::Conjugate)); - let value = |v: Complex| { let v = if conj { v.conj() } else { v }; if one { v } else { scale * v } }; - let mut z = filled(rows * cols, Complex::default()); - if x.op.transposed() { transpose_into(x.data, x.ld, &mut z, cols, [cols, rows], value) } else { - for (row, out) in z.chunks_exact_mut(cols).enumerate() { for (o, &v) in out.iter_mut().zip(&x.data[row * x.ld..][..cols]) { *o = value(v) } } - } - [z.iter().map(|v| v.re).collect(), z.iter().map(|v| v.im).collect()] -} - -/// The micro-kernels in plain Rust, for targets without the explicit ones. Each takes the number of vectors in a tile row, `V`, as the explicit ones do, and ignores it. -#[cfg(not(any( - target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") -)))] -mod plain { - use super::*; - macro_rules! plain { - ($($name:ident: $t:ty, $object:ident;)+) => {$( - #[inline(always)] - pub(super) fn $name( - a: &[$t], b: &[$t], t: &mut [[<$object as Pair<$t>>::Out; NR]; MR], first: bool, - ) { - micro::<$t, $object, MR, NR>(a, b, t, first) - } - )+}; - } - plain! { - times64: f64, PlusTimes; - times32: f32, PlusTimes; - times_i32: i32, PlusTimes; - max_plus64: f64, MaxPlus; - max_plus32: f32, MaxPlus; - min_plus64: f64, MinPlus; - min_plus32: f32, MinPlus; - equal64: f64, PlusEqual; - equal32: f32, PlusEqual; - equal_i64: i64, PlusEqual; - equal_i32: i32, PlusEqual; - equal_i16: i16, PlusEqual; - } -} -#[cfg(not(any( - target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") -)))] -use plain::*; - -/// Micro-kernels on explicit vectors: 16 bytes on AArch64 and in wasm, and 32 with AVX2. Each is `super::micro` on vectors, with each row of the tile `V` vectors, and its entry gives one step from the tile's running value `c`, `a` and `b`. A tile starts where its entry says, at the pair's unit for a tile of items, as `micro`'s does, so the first step drops a NaN term as the later steps do. -#[cfg(any( - target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") -))] -mod lanes { - #[cfg(target_arch = "aarch64")] - use core::arch::aarch64::*; - #[cfg(target_arch = "wasm32")] - use core::arch::wasm32::*; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::*; - macro_rules! kernels { - ($($name:ident: $t:ty => $out:ty, $lanes:literal, $splat:ident, $load:ident, $start:expr, $load_out:ident, $store_out:ident, |$a:ident, $b:ident, $c:ident| $step:expr;)+) => {$( - #[inline(always)] - pub(super) fn $name(a: &[$t], b: &[$t], t: &mut [[$out; NR]; MR], first: bool) { - assert!(NR == $lanes * V && a.len() / MR == b.len() / NR); - let steps = a.len() / MR; - // SAFETY: this module compiles only for targets with these instructions. Each load and store moves `$lanes` items inside `a`, `b` or a row of `t`. - unsafe { - let (ap, bp) = (a.as_ptr(), b.as_ptr()); - let mut acc = [[$start; V]; MR]; - if !first { for r in 0..MR { for v in 0..V { acc[r][v] = $load_out(t[r].as_ptr().add($lanes * v).cast()) } } } - for p in 0..steps { - let y: [_; V] = core::array::from_fn(|v| $load(bp.add(p * NR + $lanes * v).cast())); - for r in 0..MR { - let $a = $splat(*ap.add(p * MR + r) as _); - for v in 0..V { acc[r][v] = { let ($c, $b) = (acc[r][v], y[v]); $step } } - } - } - for r in 0..MR { for v in 0..V { $store_out(t[r].as_mut_ptr().add($lanes * v).cast(), acc[r][v]) } } - } - } - )+}; - } - /// `kernels!` for tiles of items, which start at the entry's pair's unit. - macro_rules! lanes { - ($($name:ident: $t:ty, $pair:ident, $lanes:literal, $splat:ident, $load:ident, $store:ident, |$a:ident, $b:ident, $c:ident| $step:expr;)+) => { - kernels! { $($name: $t => $t, $lanes, $splat, $load, $splat(>::UNIT), $load, $store, |$a, $b, $c| $step;)+ } - }; - } - // rustymath's own plus-times products, which macOS compiles only with the `native` feature. - #[cfg(all(target_arch = "aarch64", native))] - lanes! { - times64: f64, PlusTimes, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vfmaq_f64(c, a, b); - times32: f32, PlusTimes, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vfmaq_f32(c, a, b); - } - #[cfg(target_arch = "aarch64")] - lanes! { - // Integer plus-times products have no Accelerate form, so every target compiles them. - times_i32: i32, PlusTimes, 4, vdupq_n_s32, vld1q_s32, vst1q_s32, |a, b, c| vmlaq_s32(c, a, b); - max_plus64: f64, MaxPlus, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vmaxnmq_f64(c, vaddq_f64(a, b)); - max_plus32: f32, MaxPlus, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vmaxnmq_f32(c, vaddq_f32(a, b)); - min_plus64: f64, MinPlus, 2, vdupq_n_f64, vld1q_f64, vst1q_f64, |a, b, c| vminnmq_f64(c, vaddq_f64(a, b)); - min_plus32: f32, MinPlus, 4, vdupq_n_f32, vld1q_f32, vst1q_f32, |a, b, c| vminnmq_f32(c, vaddq_f32(a, b)); - } - // Baseline wasm has no fused multiply-add. - #[cfg(target_arch = "wasm32")] - lanes! { - times64: f64, PlusTimes, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_add(c, f64x2_mul(a, b)); - times32: f32, PlusTimes, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_add(c, f32x4_mul(a, b)); - times_i32: i32, PlusTimes, 4, i32x4_splat, v128_load, v128_store, |a, b, c| i32x4_add(c, i32x4_mul(a, b)); - max_plus64: f64, MaxPlus, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_pmax(c, f64x2_add(a, b)); - max_plus32: f32, MaxPlus, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_pmax(c, f32x4_add(a, b)); - min_plus64: f64, MinPlus, 2, f64x2_splat, v128_load, v128_store, |a, b, c| f64x2_pmin(c, f64x2_add(a, b)); - min_plus32: f32, MinPlus, 4, f32x4_splat, v128_load, v128_store, |a, b, c| f32x4_pmin(c, f32x4_add(a, b)); - } - // rustymath's own plus-times products, which macOS compiles only with the `native` feature. - #[cfg(all(target_arch = "x86_64", native))] - lanes! { - times64: f64, PlusTimes, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_fmadd_pd(a, b, c); - times32: f32, PlusTimes, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_fmadd_ps(a, b, c); - } - #[cfg(target_arch = "x86_64")] - lanes! { - // Integer plus-times products have no Accelerate form, so every target compiles them. AVX2 has no multiply-add on 32-bit integers. - times_i32: i32, PlusTimes, 8, _mm256_set1_epi32, _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| _mm256_add_epi32(c, _mm256_mullo_epi32(a, b)); - // AVX2's `maxpd` and `minpd` give their second argument beside NaN: the running value `c`. - max_plus64: f64, MaxPlus, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_max_pd(_mm256_add_pd(a, b), c); - max_plus32: f32, MaxPlus, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_max_ps(_mm256_add_ps(a, b), c); - min_plus64: f64, MinPlus, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_storeu_pd, |a, b, c| _mm256_min_pd(_mm256_add_pd(a, b), c); - min_plus32: f32, MinPlus, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_storeu_ps, |a, b, c| _mm256_min_ps(_mm256_add_ps(a, b), c); - } - // `PlusEqual`'s tiles count equal pairs. Where items are equal, a comparison gives a lane of all ones, which is -1 as an integer, so subtracting it adds 1 to the count. - #[cfg(target_arch = "aarch64")] - kernels! { - equal64: f64 => i64, 2, vdupq_n_f64, vld1q_f64, vdupq_n_s64(0), vld1q_s64, vst1q_s64, |a, b, c| vsubq_s64(c, vreinterpretq_s64_u64(vceqq_f64(a, b))); - equal32: f32 => i32, 4, vdupq_n_f32, vld1q_f32, vdupq_n_s32(0), vld1q_s32, vst1q_s32, |a, b, c| vsubq_s32(c, vreinterpretq_s32_u32(vceqq_f32(a, b))); - equal_i64: i64 => i64, 2, vdupq_n_s64, vld1q_s64, vdupq_n_s64(0), vld1q_s64, vst1q_s64, |a, b, c| vsubq_s64(c, vreinterpretq_s64_u64(vceqq_s64(a, b))); - equal_i32: i32 => i32, 4, vdupq_n_s32, vld1q_s32, vdupq_n_s32(0), vld1q_s32, vst1q_s32, |a, b, c| vsubq_s32(c, vreinterpretq_s32_u32(vceqq_s32(a, b))); - equal_i16: i16 => i16, 8, vdupq_n_s16, vld1q_s16, vdupq_n_s16(0), vld1q_s16, vst1q_s16, |a, b, c| vsubq_s16(c, vreinterpretq_s16_u16(vceqq_s16(a, b))); - } - #[cfg(target_arch = "wasm32")] - kernels! { - equal64: f64 => i64, 2, f64x2_splat, v128_load, i64x2_splat(0), v128_load, v128_store, |a, b, c| i64x2_sub(c, f64x2_eq(a, b)); - equal32: f32 => i32, 4, f32x4_splat, v128_load, i32x4_splat(0), v128_load, v128_store, |a, b, c| i32x4_sub(c, f32x4_eq(a, b)); - equal_i64: i64 => i64, 2, i64x2_splat, v128_load, i64x2_splat(0), v128_load, v128_store, |a, b, c| i64x2_sub(c, i64x2_eq(a, b)); - equal_i32: i32 => i32, 4, i32x4_splat, v128_load, i32x4_splat(0), v128_load, v128_store, |a, b, c| i32x4_sub(c, i32x4_eq(a, b)); - equal_i16: i16 => i16, 8, i16x8_splat, v128_load, i16x8_splat(0), v128_load, v128_store, |a, b, c| i16x8_sub(c, i16x8_eq(a, b)); - } - #[cfg(target_arch = "x86_64")] - kernels! { - equal64: f64 => i64, 4, _mm256_set1_pd, _mm256_loadu_pd, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { - _mm256_sub_epi64(c, _mm256_castpd_si256(_mm256_cmp_pd::<_CMP_EQ_OQ>(a, b))) - }; - equal32: f32 => i32, 8, _mm256_set1_ps, _mm256_loadu_ps, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { - _mm256_sub_epi32(c, _mm256_castps_si256(_mm256_cmp_ps::<_CMP_EQ_OQ>(a, b))) - }; - equal_i64: i64 => i64, 4, _mm256_set1_epi64x, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { - _mm256_sub_epi64(c, _mm256_cmpeq_epi64(a, b)) - }; - equal_i32: i32 => i32, 8, _mm256_set1_epi32, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { - _mm256_sub_epi32(c, _mm256_cmpeq_epi32(a, b)) - }; - equal_i16: i16 => i16, 16, _mm256_set1_epi16, _mm256_loadu_si256, _mm256_setzero_si256(), _mm256_loadu_si256, _mm256_storeu_si256, |a, b, c| { - _mm256_sub_epi16(c, _mm256_cmpeq_epi16(a, b)) - }; - } -} -#[cfg(any( - target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"), - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "fma") -))] -use lanes::*; - -/// The multiply-adds in each band of rows that `product` hands to a thread. On an M5 Max, 2^19 made 512×512 QR and inverses 5 to 13% faster on 4 threads than 2^20 did. 2^18 made `+.×`, Cholesky and QR 10 to 40% slower on 18 threads than 2^19. -const BAND_WORK: usize = 1 << 19; - -/// The multiply-adds that waking one more thread costs a product. A panel of `w` multiply-adds splits across at most √(w / `THREAD_WORK`) threads, the count that minimises w / t + t × `THREAD_WORK`. On an M5 Max, `+.×` of two 256×256 matrices took 21.6 copies on 8 threads and 28.8 on 18, and larger products were fastest on all 18. -const THREAD_WORK: usize = 1 << 17; - -/// The most items of `b` that `product` packs into one panel. It splits each panel's bands across threads in one call, however many steps of `KC` the panel holds. -const PANEL: usize = 1 << 22; -/// The most bytes of `b` that `product` packs into one panel when this thread's calls run serially, as on a pool thread. Larger panels outgrow the cache: on an M5 Max, a one-thread Gram product of a 16384×256 f64 matrix took 3% longer with 8 MB panels and 10% longer with 32 MB ones, and a threaded tall QR whose threads each ran one such product was 2% slower with 4 MB panels than with 2 MB ones. -const SERIAL_PANEL: usize = 2 << 20; -/// The steps of `b` that `product` packs across all of a panel's strips before packing the next steps. Packing a strip reads `NR` items from each row of a row-major `b`. Unless those items fill whole cache lines, the next strip reads the rest of each line. When `b`'s rows were a power of two apart, packing one strip at a time had already evicted those lines: on an M5 Max, a 64×256×16384 f64 product took 15 to 28% longer than with 16 steps at a time. -const ROWS: usize = 16; -/// The columns of the result that `product` fills one row of tiles at a time, before the next row of tiles. Filling each row of tiles in turn writes each row of `c` in order, whatever `ldc` is. Sweeping the whole width instead wrote the rows of a column of tiles together, and a power-of-two `ldc` maps those rows to the same cache sets: on an M5 Max, a 64×32×16384 f64 product took 2.3 times as long as a 64×32×16000 one. 128 columns of a packed `KC`-step f64 block of `b` hold 256 KB, which stays in a Zen 4 core's 1 MB second-level cache while each row of tiles rereads it. -const SWEEP: usize = 128; -/// Folds the inner product of `P` into `c`, an `m`×`n` matrix whose rows start `ldc` items apart, with `kernel` folding each `MR`×`NR` tile as `micro` does. `a(rows, p0, steps)` writes a strip of the `m`×`k` matrix `a`, and `b(columns, p0, steps)` a strip of the transpose of the `k`×`n` matrix `b`, as `Matrix::strip` does. With `start`, the product replaces `c`, which must then hold `f`'s identity where `k` is 0. Otherwise each result folds into `c`'s item with `f`. Only tiles that hold items of `part` fold. Items outside `part` in those tiles fold too. Each block of `NC` columns packs `b` in panels of at most [`PANEL`] items, or [`SERIAL_PANEL`] bytes on one thread, [`ROWS`] steps at a time across the panel's strips, across threads as [`parallel::split`] chooses. Then it runs bands of rows across at most √(w / [`THREAD_WORK`]) of this thread's [`parallel::threads`], for a panel of `w` multiply-adds: about one band for each [`BAND_WORK`] multiply-adds, up to four bands for each thread, each at most `MC` rows. Each band runs the panel's steps of `KC` in turn, packing its own block of `a` for each. Within a block of `a`, it fills one row of tiles at a time across [`SWEEP`] columns of the result. -#[inline(always)] -fn product, const MR: usize, const NR: usize>( - c: &mut [P::Out], ldc: usize, a: impl Fn([usize; 2], usize, &mut [[MaybeUninit; MR]]) + Sync, - b: impl Fn([usize; 2], usize, &mut [[MaybeUninit; NR]]) + Sync, [m, k, n]: [usize; 3], start: bool, part: Part, - kernel: impl Fn(&[T], &[T], &mut [[P::Out; NR]; MR], bool) + Sync) { - if m == 0 || k == 0 || n == 0 { return; } - assert!(ldc >= n && c.len() >= (m - 1) * ldc + n, "`c` must hold `m` rows of `n` items, `ldc` apart"); - let c = &mut c[..(m - 1) * ldc + n]; - let width = NC.min(n).next_multiple_of(NR); - let panel = if parallel::threads() == 1 { SERIAL_PANEL / size_of::() } else { PANEL }; - let depth = (panel / (width * KC)).clamp(1, k.div_ceil(KC)) * KC; - let mut bp = Vec::with_capacity(width * depth.min(k)); - for jc in (0..n).step_by(NC) { - let nc = NC.min(n - jc); - for pp in (0..k).step_by(depth) { - let kp = depth.min(k - pp); - let unit = NR * kp; - let work = m * kp * nc; - let threads = if work < 2 * BAND_WORK { 1 } else { parallel::threads().min((work / THREAD_WORK).isqrt()) }; - let band_rows = if threads == 1 { m } else { m.div_ceil((work / BAND_WORK).min(4 * threads)).next_multiple_of(MR).min(MC) }; - let pack_b = |at: usize, units: &mut [MaybeUninit]| { - for p0 in (0..kp).step_by(ROWS) { - let rows = ROWS.min(kp - p0); - for (u, strip) in units.chunks_exact_mut(unit).enumerate() { - let s = at / unit + u; - let (j, lines) = (jc + s * NR, NR.min(nc - s * NR)); - pack::<_, NR>(&mut strip[p0 * NR..][..rows * NR], rows, lines, |first, ln, steps| b([j + first, ln], pp + p0, steps)); - } - } - true - }; - bp.clear(); - slice::appended(&mut bp, nc.div_ceil(NR) * unit, |slots| if threads == 1 { pack_b(0, slots) } else { parallel::split(slots, unit, &pack_b) }); - let (a, bp, kernel) = (&a, &bp[..], &kernel); - let lower = matches!(part, Part::Lower); - parallel::pieces_of(c, band_rows * ldc, threads, &move |first, band| { - let (i0, i1) = (first / ldc, (first / ldc + band_rows).min(m)); - let mut slots = Box::<[T]>::new_uninit_slice(MC.min(i1 - i0).next_multiple_of(MR) * KC.min(kp)); - for step in 0..kp.div_ceil(KC) { - let pc = pp + step * KC; - let (kc, load) = (KC.min(k - pc), pc > 0 || !start); - for ic in (i0..i1).step_by(MC) { - let mc = MC.min(i1 - ic); - if lower && ic + mc <= jc { continue; } - let ap = pack::<_, MR>(&mut slots, kc, mc, |first, lines, steps| a([ic + first, lines], pc, steps)); - for jb in (0..nc).step_by(SWEEP) { - for ir in (0..mc).step_by(MR) { - for jr in (jb..nc.min(jb + SWEEP)).step_by(NR) { - let bj = &bp[jr / NR * unit + step * NR * KC..][..NR * kc]; - if lower && ic + ir + MR.min(mc - ir) <= jc + jr { continue; } - let (rows, cols) = (MR.min(mc - ir), NR.min(nc - jr)); - let at = |r: usize| (ic - i0 + ir + r) * ldc + jc + jr; - let mut t = [[P::UNIT; NR]; MR]; - // A whole row moves as one run of `NR` items, which compiles to vector moves. Item by item, each move compiles to a scalar load, a store and a bounds check. - if load { - for r in 0..rows { - if cols == NR { t[r].copy_from_slice(&band[at(r)..][..NR]) } else { for s in 0..cols { t[r][s] = band[at(r) + s] } } - } - } - kernel(&ap[ir * kc..(ir + MR) * kc], bj, &mut t, !load); - for r in 0..rows { - if cols == NR { band[at(r)..][..NR].copy_from_slice(&t[r]) } else { for s in 0..cols { band[at(r) + s] = t[r][s] } } - } - } - } - } - } - } - true - }); - } - } -} - -/// Packs `len` lines of `kc` steps into strips of `W` lines at the start of `buffer`, and gives the items it wrote. Each strip holds its `W` items for step 0, then those for step 1, and so on. `strip(first, lines, steps)` writes all `W` items of each of the strip's `kc` steps: `lines` items for lines `first` on, then the default. The results computed from the defaults are never stored. -fn pack( - buffer: &mut [MaybeUninit], kc: usize, len: usize, strip: impl Fn(usize, usize, &mut [[MaybeUninit; W]])) -> &[T] { - let strips = len.div_ceil(W); - for (s, chunk) in buffer.chunks_exact_mut(W * kc).take(strips).enumerate() { strip(s * W, W.min(len - s * W), chunk.as_chunks_mut::().0); } - slice::written(&mut buffer[..strips * W * kc]) -} - -/// Folds the steps of the strips `a` and `b`, which `pack` wrote, into the tile `t`. With `first`, the tile starts at the pair's unit, as `direct`'s sums do. -#[inline(always)] -fn micro, const MR: usize, const NR: usize>(a: &[T], b: &[T], t: &mut [[P::Out; NR]; MR], first: bool) { - let mut acc = if first { [[P::UNIT; NR]; MR] } else { *t }; - for (a, b) in a.as_chunks::().0.iter().zip(b.as_chunks::().0) { for r in 0..MR { for s in 0..NR { acc[r][s] = P::step(acc[r][s], a[r], b[s]) } } } - *t = acc; -} diff --git a/src/operation/fold.rs b/src/operation/fold.rs deleted file mode 100644 index 958e67a..0000000 --- a/src/operation/fold.rs +++ /dev/null @@ -1,65 +0,0 @@ -use super::*; - -fn piece_values(range: std::ops::Range, checked: bool, pieces: &Pieces<'_>, out: &mut Vec) -> Result<(), Error> { - out.clear(); - let (mut next, mut failure) = (range.start, None); - pieces(range.clone(), &mut |at, piece| { - if failure.is_some() { return; } - let Piece::Items(view) = piece else { failure = Some(Error::new(Kind::Length, "ordered fold received a precomputed total")); return; }; - if at != next || at + view.len() > range.end { failure = Some(Error::new(Kind::Length, "fold pieces do not cover the requested range")); return; } - if let Err(error) = T::extend_cast(view, view.len(), checked, out) { failure = Some(error); return; } - next += view.len(); - })?; - if let Some(error) = failure { return Err(error); } - if next != range.end { return Err(Error::new(Kind::Length, "fold pieces left an unread range")); } - Ok(()) -} - -/// The ordered folds of each lane of `[outer, len, inner]`, read from `pieces` a chunk of rows at a time. Each chunk folds from the right onto the fold so far: by `lane`, one call for each lane, along the last axis, and by `block`, one call for each row, along any other axis. -pub(super) fn ordered_fold( - [outer, len, inner]: [usize; 3], seed: Seed<'_>, direction: Direction, checked: bool, identity: Option, pieces: &Pieces<'_>, - lane: &dyn Fn(&[T], T) -> (T, Option), block: &dyn Fn(&[T], &[T], &mut [std::mem::MaybeUninit]) -> Result<(), Kind>, output: &mut Buffer -) -> Result<(), Error> { - let (mut result, mut items, mut seed_scratch) = (Vec::with_capacity(outer * inner), Vec::new(), Vec::new()); - if inner == 0 { *output = T::buffer(result); return Ok(()); } - let right = direction == Direction::Right; - let fold = |items: &[T], carry: Option<&[T]>| { - let (rows, tail) = match carry { Some(tail) => (items, tail), None => items.split_at(items.len() - inner) }; - let (mut total, mut next, mut failure) = (tail.to_vec(), Vec::with_capacity(inner), None); - if inner == 1 { (total[0], failure) = lane(rows, tail[0]); } else { - for row in rows.chunks_exact(inner).rev() { - next.clear(); - failure = failure.or(crate::slice::appended(&mut next, inner, |slots| block(&total, row, slots)).err()); - std::mem::swap(&mut total, &mut next); - } - } - failure.map_or(Ok(total), |kind| Err(Error::new(kind, "ordered fold is outside the function's domain"))) - }; - for cell in 0..outer { - let mut given = seed.read().map(|read| crate::reduce::seed_row(&read, cell, inner, checked, &mut seed_scratch)).transpose()?; - let upfront = len == 0 || match seed { Seed::Tail(_) => right, Seed::Initial(_) => !right, Seed::Identity => false }; - let mut carry = if upfront { given.take() } else { None }; - let rows = (crate::CHUNK / inner).max(1); - let mut remaining = len; - while remaining != 0 { - let n = rows.min(remaining); - let start = if right { remaining - n } else { len - remaining }; - let at = (cell * len + start) * inner; - piece_values(at..at + n * inner, checked, pieces, &mut items)?; - if !right { crate::reduce::reverse_rows(&mut items, [1, n, inner]); } - carry = Some(if carry.is_none() && n == 1 { items.clone() } else { fold(&items, carry.as_deref())? }); - remaining -= n; - } - let mut total = match carry { - Some(total) => total, - None => { - let value = identity.ok_or_else(|| Error::new(Kind::Length, "an empty fold needs an identity or a seed"))?; - vec![T::cast(View::Floats(crate::buffer::Floats::F64(&[value])), 0)?; inner] - } - }; - if let Some(seed) = given { total = fold(&seed, Some(&total))?; } - result.extend(total); - } - *output = T::buffer(result); - Ok(()) -} diff --git a/tests/accuracy.rs b/tests/accuracy.rs index 2d2e809..8a7210a 100644 --- a/tests/accuracy.rs +++ b/tests/accuracy.rs @@ -270,10 +270,10 @@ fn logarithms_accuracy() { /// `log2` gives k exactly at 2^k, and `log10` gives k at 10^k, or at the float nearest 10^k where 10^k isn't a float. Each does at every width, fused and unfused. So does `Logb` in base 2 and base 10: through its formula, and through its kernel with the base constant, the argument constant or neither. #[test] fn logarithms_exact() { - use rustymath::{Float, Logb, buffer::Element, domain::Variant, facts::{Formula, formula_at}}; + use rustymath::{Float, Logb, buffer::Element, domain::Variant, facts::formula_at, operation::Kernel}; use std::ops::RangeInclusive; fn exact(twos: RangeInclusive, tens: RangeInclusive) { - let kernel = Logb.resolve(Variant { checked: false, ..Variant::checked(F::DTYPE) }, [F::DTYPE; 2]).unwrap(); + let kernel = Kernel::resolve(Logb, Variant { checked: false, ..Variant::checked(F::DTYPE) }, [F::DTYPE; 2]).unwrap(); // The kernel's first result on `bases` and `xs`, where a one-item argument is a constant. let first = |bases: &[F], xs: &[F]| { let mut out = F::buffer(vec![]); diff --git a/tests/linalg.rs b/tests/linalg.rs index f8dd681..aca0586 100644 --- a/tests/linalg.rs +++ b/tests/linalg.rs @@ -15,8 +15,8 @@ mod products { #[test] fn native_inner_pairs_keep_right_folds_and_bounded_terms() { use rustymath::{ - agreement::{Binding, Missing, Read, ReadPlan}, buffer::{Buffer, Element, View}, domain::Variant, error::{self, Error}, facts::Formula, - operation::{Direction, Kernel, Seed, Settings}, tolerant::{Equal, Greater, Tolerance} + agreement::{Binding, Missing, Read, ReadPlan}, buffer::{Buffer, Element, View}, domain::Variant, error::{self, Error}, + operation::{Kernel, Settings}, reduce::{Direction, Seed}, tolerant::{Equal, Greater, Tolerance} }; fn fold(kernel: &Kernel, data: View<'_>, dims: [usize; 3], tail: Option>) -> Result { let mut sources = vec![Binding::full(data)]; @@ -35,8 +35,8 @@ mod products { ($f:expr, $g:expr, $a:expr, $b:expr, $size:expr) => {{ let (f, g, a, b) = ($f, $g, $a, $b); let dtype = a.dtype().common(b.dtype()).unwrap(); - let gk = g.resolve(Variant::checked(dtype), [a.dtype(), b.dtype()]).unwrap(); - let fk = f.resolve(Variant::checked(gk.result_dtype()), [gk.result_dtype(); 2]).unwrap(); + let gk = Kernel::resolve(g, Variant::checked(dtype), [a.dtype(), b.dtype()]).unwrap(); + let fk = Kernel::resolve(f, Variant::checked(gk.result_dtype()), [gk.result_dtype(); 2]).unwrap(); inner_with( a, b, $size, |x, y, agreement| { assert!(agreement.len <= rustymath::CHUNK); @@ -78,7 +78,7 @@ mod products { assert!(inner!(rustymath::Pow, rustymath::Add, f64::view(&[-1., 0.5]), f64::view(&[0., 2., 0., 1.5]), [1, 2, 2]).is_err()); assert!(inner!(rustymath::Add, rustymath::Multiply, i64::view(&[i64::MAX, 1]), i64::view(&[2, 2]), [1, 2, 1]).is_err()); assert!(inner!(rustymath::Add, rustymath::Multiply, i64::view(&[]), i64::view(&[2, 2]), [0, 2, 1]).unwrap().is_empty()); - let sum = rustymath::Add.resolve(Variant::checked(i64::DTYPE), [i64::DTYPE; 2]).unwrap(); + let sum = Kernel::resolve(rustymath::Add, Variant::checked(i64::DTYPE), [i64::DTYPE; 2]).unwrap(); assert_eq!(i64::vec(fold(&sum, bool::view(&[true, false]), [1, 2, 1], Some(i64::view(&[7]))).unwrap()).unwrap(), [8]); assert!(fold(&sum, i64::view(&[i64::MAX]), [1, 1, 1], Some(i64::view(&[1]))).is_err()); } @@ -145,9 +145,10 @@ mod products { let stored = expanded_items(&Walked::from(&mapping), data, 0, m * k, &mut buf).to_vec(); let a = if o == Orientation::Normal { stored } else { transpose(&stored, k, m) }; let b: Vec = (0..k).map(|i| (i * 5 % 13) as f64 - 6.0).collect(); - let got = PlusTimes.inner_mapped( + let mut got = vec![f64::NAN; m]; + PlusTimes.inner_mapped_into( Mapped { data, mapping: &Walked::from(&mapping) }, Mapped { data: &b, mapping: &Walked::from(&Mapping::Linear(1)) }, [o, Orientation::Normal], - [m, k, 1]); + [m, k, 1], &mut got); assert_eq!(got, PlusTimes.inner(&a, &b, [m, k, 1])); }; let x: Vec = (0..300).map(|i| (i * 7 % 17) as f64 - 8.0).collect(); @@ -167,6 +168,11 @@ mod products { let mut into = vec![1.0; m * n]; PlusTimes.inner_into(&a64, &b64, size, &mut into); assert_eq!(into, expected); + into.fill(f64::NAN); + let linear = rustymath::agreement::Walked::from(&rustymath::agreement::Mapping::Linear(1)); + PlusTimes.inner_mapped_into( + Mapped { data: &a64, mapping: &linear }, Mapped { data: &b64, mapping: &linear }, [Orientation::Normal; 2], size, &mut into); + assert_eq!(into, expected); assert_eq!(PlusTimes.inner(&a64, &b64, size), expected); assert_eq!(PlusTimes.inner(&a32, &b32, size), folded(&a32, &b32, size, 0.0, |s, x, y| s + x * y)); assert_eq!(MaxPlus.inner(&a64, &b64, size), folded(&a64, &b64, size, f64::NEG_INFINITY, |s, x, y| s.max(x + y))); @@ -617,11 +623,20 @@ mod factor { blocked::(4.0 * f32::EPSILON as f64); } - /// The `m`×`n` matrix U diag(s) Vᵀ, with orthonormal U and V from fixed seeds and the singular values `s`. - pub(super) fn with_spectrum(m: usize, n: usize, s: &[f64]) -> Vec { - let (u, v) = (qr(&random::(m, n, 3), [m, n]).0, qr(&random::(n, n, 5), [n, n]).0); - let us: Vec = (0..m * n).map(|k| u[k] * s[k % n]).collect(); - PlusTimes.inner(&us, &transpose(&v, n, n), [m, n, n]) + /// Orthonormal `m`×`n` U and `n`×`n` Vᵀ, from fixed seeds, for matrices of known singular values. A test that builds several matrices of one size factors them once. + pub(super) struct Bases { m: usize, n: usize, u: Vec, vt: Vec } + + impl Bases { + pub(super) fn new(m: usize, n: usize) -> Self { + let (u, v) = (qr(&random::(m, n, 3), [m, n]).0, qr(&random::(n, n, 5), [n, n]).0); + Self { m, n, u, vt: transpose(&v, n, n) } + } + + /// The `m`×`n` matrix U diag(s) Vᵀ, whose singular values are `s`. + pub(super) fn with_spectrum(&self, s: &[f64]) -> Vec { + let us: Vec = (0..self.m * self.n).map(|k| self.u[k] * s[k % self.n]).collect(); + PlusTimes.inner(&us, &self.vt, [self.m, self.n, self.n]) + } } /// QR, least squares and the SVD of tall matrices, with condition numbers from 1 to `worst`, which Cholesky QR factors, and `beyond`, which falls back to Householder QR. Each matrix's singular values run from 1 down to 1/condition. Its first item is positive, so R's first item is positive on the Cholesky path and negative on Householder's. @@ -630,10 +645,10 @@ mod factor { let quartic = T::narrow(2f64.powi((T::max_value().log2().wide() / 4.0) as i32 + 6)); let times = |x: &[T], c: T| x.iter().map(|&v| v * c).collect::>(); for (m, n) in [(4000, 20), (8000, 60)] { - let bound = tolerance * n as f64; + let (bound, bases) = (tolerance * n as f64, Bases::new(m, n)); for condition in [1.0, worst.sqrt(), worst, beyond] { let s: Vec = (0..n).map(|j| condition.powf(-(j as f64) / (n - 1) as f64)).collect(); - let mut aw = with_spectrum(m, n, &s); + let mut aw = bases.with_spectrum(&s); if aw[0] < 0.0 { aw[..n].iter_mut().for_each(|x| *x = -*x) } let a: Vec = aw.iter().map(|&x| T::narrow(x)).collect(); let (aw, at) = (wide(&a), format!("{m}×{n}, condition {condition:e}")); @@ -676,18 +691,18 @@ mod factor { } #[test] - fn tall_matrices() { - tall::(4.0 * f64::EPSILON, 1e7, 1e12); - tall::(4.0 * f32::EPSILON as f64, 1e2, 1e12); - } + fn tall_matrices_at_f64() { tall::(4.0 * f64::EPSILON, 1e7, 1e12); } + + #[test] + fn tall_matrices_at_f32() { tall::(4.0 * f32::EPSILON as f64, 1e2, 1e12); } - /// A tall matrix 520 columns wide takes the Cholesky path, with the same accuracy as the narrow ones in `tall_matrices`. + /// A tall matrix 520 columns wide takes the Cholesky path, with the same accuracy as the narrow ones in `tall_matrices_at_f64`. #[test] fn tall_wide_matrices() { let (m, n) = (8320, 520); let bound = 4.0 * f64::EPSILON * n as f64; let s: Vec = (0..n).map(|j| 1e3f64.powf(-(j as f64) / (n - 1) as f64)).collect(); - let mut a = with_spectrum(m, n, &s); + let mut a = Bases::new(m, n).with_spectrum(&s); if a[0] < 0.0 { a[..n].iter_mut().for_each(|x| *x = -*x) } let (q, r) = qr(&a, [m, n]); assert!(r[0] > 0.0, "wide tall qr takes the Cholesky path"); @@ -707,8 +722,9 @@ mod factor { let bound = tolerance * n as f64; let fast: Vec = (0..n).map(|j| 0.5f64.powi(j as i32)).collect(); let slow: Vec = (0..n).map(|j| 1.0 / (1 + j) as f64).collect(); + let bases = Bases::new(m, n); for (decay, s) in [("fast", fast), ("slow", slow)] { - let tall: Vec = with_spectrum(m, n, &s).iter().map(|&x| T::narrow(x)).collect(); + let tall: Vec = bases.with_spectrum(&s).iter().map(|&x| T::narrow(x)).collect(); // The wide case works on a view of the conjugate transpose. for (a, [m, n]) in [(transpose(&tall, m, n), [n, m]), (tall, [m, n])] { let f = truncated_svd(&a, [m, n], k); diff --git a/tests/numeric.rs b/tests/numeric.rs index b8d934b..cca2549 100644 --- a/tests/numeric.rs +++ b/tests/numeric.rs @@ -5,7 +5,7 @@ mod timing { /// The nesting rules for shared and exclusive holds, on a lock file of the test's own, which no build or timing holds. #[test] fn hold_modes_and_lifetimes() { - use rustymath::timing::hold_at; + use rustymath_timing::hold_at; use std::io::ErrorKind::InvalidInput; let path = std::env::temp_dir().join(format!("rustymath-timing-{}.lock", std::process::id())); let lock = path.to_str().unwrap(); @@ -23,7 +23,7 @@ mod timing { /// A budget is the expected cost times its multiplier, rounded up, and a `held` or `busy` line raises one column's budget to the budget for the cost it gives, or to twice a cost marked as an estimate. Only a gap's hold goes stale, once a cost is within 90% of the rule budget. #[test] fn budgets_follow_the_rule_and_holds() { - use rustymath::timing; + use rustymath_timing as timing; assert_eq!([0.2, 0.27, 15.6, 142.0, 12682.0].map(|c| timing::budget("f64", c, false)), [0.5, 0.54, 32.0, 220.0, 17000.0]); assert_eq!((timing::budget("f64", 60.0, true), timing::budget("widths", 0.72, false)), (120.0, 0.97)); let text = "[f64]\n# 2 copies: one read and one write.\n# held linux 15.6: the cause isn't known yet\nExp\t4\t4\n# busy macos 2.7: a busy host\nLog\t4\t4\n# held linux ~60: no run has measured it\nSin\t4\t4\n"; @@ -37,18 +37,33 @@ mod timing { assert!(timing::cases(&format!("[f64]\n{bad}\nExp\t4\t4\n")).is_err(), "{bad}"); } } + + /// A `-` budget is no budget in its column. Counts follow the budgets, one field for each measure, and a count fails above its margin times the count recorded, with one more byte for each item for bytes allocated. + #[test] + fn counts_follow_budgets() { + use rustymath_timing::{self as timing, COUNT_MARGIN}; + let cases = timing::cases("[f64]\nExp\t4\t-\tinstructions linux 2.5 macos 2\tallocated linux 0\n").unwrap(); + let case = &cases[0]; + assert_eq!(timing::Budget::of(case.section, &case.budgets, &case.holds, Some(1)), None); + let instructions = case.count("instructions", Some(1)).unwrap(); + assert!(!instructions.over(3.1, COUNT_MARGIN) && instructions.over(3.2, COUNT_MARGIN) && instructions.over(2.6, 1.0)); + assert!(!case.count("allocated", Some(1)).unwrap().over(1.0, COUNT_MARGIN) && case.count("allocated", Some(0)).is_none()); + for bad in ["Exp\t4\tinstructions linux", "Exp\tinstructions linux 2\t4", "Exp\t4\tinstructions linux 1 linux 2", "Exp\t4\tinstructions mars 2"] { + assert!(timing::cases(&format!("[f64]\n{bad}\n")).is_err(), "{bad}"); + } + } } mod arith { use crate::support::mapped; - use rustymath::{Abs, Add, Multiply, Sign, Subtract, buffer::Element, domain::Variant, facts::Formula, operation::Kernel}; + use rustymath::{Abs, Add, Multiply, Sign, Subtract, buffer::Element, domain::Variant, operation::Kernel}; /// The checked integer kernels of the arithmetic objects at type `$t`, on every pair of edge values, against the exact result in `i128`. Each fails where that doesn't fit `$t`. macro_rules! check { ($($t:ty),+) => {$({ let variant = Variant::checked(<$t>::DTYPE); - let pairs = [Add.resolve(variant, [variant.dtype; 2]), Subtract.resolve(variant, [variant.dtype; 2]), Multiply.resolve(variant, [variant.dtype; 2])].map(Result::unwrap); - let [abs, sign] = [Abs.resolve(variant, [variant.dtype]), Sign.resolve(variant, [variant.dtype])].map(Result::unwrap); + let pairs = [Kernel::resolve(Add, variant, [variant.dtype; 2]), Kernel::resolve(Subtract, variant, [variant.dtype; 2]), Kernel::resolve(Multiply, variant, [variant.dtype; 2])].map(Result::unwrap); + let [abs, sign] = [Kernel::resolve(Abs, variant, [variant.dtype]), Kernel::resolve(Sign, variant, [variant.dtype])].map(Result::unwrap); let one = |k: &Kernel, x: $t| mapped(k, [&[x][..]], 1).map(|r: Vec<$t>| r[0]); let edges: Vec<$t> = [<$t>::MIN, <$t>::MAX, 0, 1, 2, 3, 7].into_iter().chain([-1i128, -2, -3].map(|v| v as $t)) .chain([<$t>::MIN / 2, <$t>::MAX / 2, <$t>::MAX - 1, <$t>::MIN.wrapping_add(1)]).collect(); @@ -231,7 +246,7 @@ mod integer { use num_bigint::BigInt; use rustymath::integer::*; use rustymath::{ - Abs, Arg, Binary, Binom, Pow, buffer::{DType, Element}, domain::Variant, facts::Formula, + Abs, Arg, Binary, Binom, Pow, buffer::{DType, Element}, domain::Variant, operation::Kernel, tolerant::{Ceil, Equal, Floor, FloorDivide, Gcd, Greater, GreaterEqual, Lcm, LessEqual, NotEqual, Residue} }; @@ -401,7 +416,7 @@ mod integer { } } // Floor division of every `u8` dividend by every nonzero `u8` divisor, through each width's checked kernel. - let divide = |dtype| FloorDivide(t).resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); + let divide = |dtype| Kernel::resolve(FloorDivide(t), Variant::checked(dtype), [dtype; 2]).unwrap(); let floor = |x: i64, y: i64| { let q = x / y; if x % y != 0 && (x < 0) != (y < 0) { q - 1 } else { q } }; let (xs8, ds8): (Vec, Vec) = (0..=u8::MAX).flat_map(|x| (1..=u8::MAX).map(move |d| (x, d))).unzip(); let want: Vec = xs8.iter().zip(&ds8).map(|(&x, &d)| floor(x.into(), d.into()) as u8).collect(); @@ -436,16 +451,16 @@ mod integer { let (n, m) = (1_000_000_000_000_000i64, 1_000_000_000_000_001i64); let (v, ints) = (Variant::checked(i64::DTYPE), [i64::DTYPE; 2]); let compare = [ - Equal(t).resolve(v, ints), NotEqual(t).resolve(v, ints), LessEqual(t).resolve(v, ints), GreaterEqual(t).resolve(v, ints), - Greater(t).resolve(v, ints), + Kernel::resolve(Equal(t), v, ints), Kernel::resolve(NotEqual(t), v, ints), Kernel::resolve(LessEqual(t), v, ints), + Kernel::resolve(GreaterEqual(t), v, ints), Kernel::resolve(Greater(t), v, ints), ]; let got = compare.map(|k| mapped(&k.unwrap(), [&[m][..], &[n][..]], 1).map(|r: Vec| r[0])); assert_eq!(got, [false, true, false, true, true].map(Some)); - let [floor, ceil] = [Floor(t).resolve(v, [i64::DTYPE]), Ceil(t).resolve(v, [i64::DTYPE])].map(Result::unwrap); + let [floor, ceil] = [Kernel::resolve(Floor(t), v, [i64::DTYPE]), Kernel::resolve(Ceil(t), v, [i64::DTYPE])].map(Result::unwrap); assert_eq!((mapped(&floor, [&[m][..]], 1), mapped(&ceil, [&[-m][..]], 1)), (Some(vec![m]), Some(vec![-m]))); // `Floor` and `Abs` keep Booleans as they are. let booleans = Variant::checked(DType::Boolean); - for kernel in [Floor(t).resolve(booleans, [DType::Boolean]), Abs.resolve(booleans, [DType::Boolean])].map(Result::unwrap) { + for kernel in [Kernel::resolve(Floor(t), booleans, [DType::Boolean]), Kernel::resolve(Abs, booleans, [DType::Boolean])].map(Result::unwrap) { assert_eq!(mapped(&kernel, [&[false, true][..]], 2), Some(vec![false, true])); } } @@ -542,7 +557,8 @@ mod number_theory { mod operation { use rustymath::{ agreement::{Agreement, Binding, Mapping, Missing, Read, ReadPlan}, array::{Array, ArrayView}, buffer::{Buffer, DType, Element, Width}, - domain::{Range, Variant}, error::{self, Error, ErrorSlot}, facts::Formula, gather::Steps, layout::Layout, operation::{Direction, Kernel, Seed, Settings} + domain::{Range, Variant}, error::{self, Error, ErrorSlot}, facts::Formula, gather::Steps, layout::Layout, operation::{Kernel, Settings}, + reduce::{Direction, Seed} }; fn view<'a>(layout: &'a Layout, data: rustymath::buffer::View<'a>, mapping: &Mapping) -> ArrayView<'a> { @@ -558,41 +574,75 @@ mod operation { Ok(out) } + #[test] + fn ready_maps_append_direct_converted_and_failed_results() { + use rustymath::{Add, Sqrt, operation::MapOutput, parallel::Slots, tolerant::{Less, Tolerance}}; + let settings = Settings { tolerance: Tolerance::EXACT }; + let float = f64::DTYPE; + let add = Kernel::resolve(Add, Variant::checked(float), [float; 2]).unwrap(); + let mut values = vec![10.]; + Slots::append(&mut values, 3, |slots| add.map_ready(&[f64::view(&[1.]), f64::view(&[2., 3., 4.])], 3, settings, MapOutput::F64(slots))).unwrap(); + assert_eq!(values, [10., 3., 4., 5.]); + let less = Kernel::resolve(Less(Tolerance::EXACT), Variant::checked(float), [float; 2]).unwrap(); + Slots::append(&mut values, 3, |slots| less.map_ready(&[f64::view(&[3.]), f64::view(&[2., 3., 4.])], 3, settings, MapOutput::F64(slots))).unwrap(); + assert_eq!(values, [10., 3., 4., 5., 0., 0., 1.]); + let complex = num_complex::Complex64::DTYPE; + let sqrt = Kernel::resolve(Sqrt, Variant::checked(complex), [complex]).unwrap(); + let result = Slots::append(&mut values, 1, |slots| { + sqrt.map_ready(&[num_complex::Complex64::view(&[num_complex::Complex64::new(-1., 0.)])], 1, settings, MapOutput::F64(slots)) + }); + assert!(result.is_err()); + assert_eq!(values.len(), 8); + let ints = Kernel::resolve(Add, Variant::checked(i64::DTYPE), [float; 2]).unwrap(); + let inputs = [f64::view(&[f64::NAN]), f64::view(&[1.])]; + Slots::append(&mut values, 0, |slots| ints.map_ready(&inputs, 0, settings, MapOutput::F64(slots))).unwrap(); + assert!(Slots::append(&mut values, 1, |slots| ints.map_ready(&inputs, 1, settings, MapOutput::F64(slots))).is_err()); + assert_eq!(values.len(), 9); + } + #[test] fn native_operations_use_input_kinds_and_run_settings() { use rustymath::{Add, Binary, Binom, Bitwise, Exp, Subtract, tolerant::{Residue, Tolerance}}; let (small, wide) = (DType::Integer(Width::I16), DType::Integer(Width::I64)); - let exp = Exp.resolve(Variant::checked(wide), [wide]).unwrap(); + let exp = Kernel::resolve(Exp, Variant::checked(wide), [wide]).unwrap(); let mut out = Buffer::with_capacity(exp.result_dtype(), 2); crate::support::map(&exp, [i64::view(&[0, 1])], 2, &mut out).unwrap(); let r = f64::vec(out).unwrap(); assert!(r[0] == 1. && (r[1] - std::f64::consts::E).abs() < 1e-15); - let add = Add.resolve(Variant::checked(DType::Boolean), [DType::Boolean; 2]).unwrap(); + let add = Kernel::resolve(Add, Variant::checked(DType::Boolean), [DType::Boolean; 2]).unwrap(); let mut out = Buffer::with_capacity(add.result_dtype(), 3); crate::support::map(&add, [bool::view(&[true]), bool::view(&[true, false, true])], 3, &mut out).unwrap(); assert_eq!(i64::vec(out).unwrap(), [2, 1, 2]); - let checked = Add.resolve(Variant::checked(wide), [wide; 2]).unwrap(); + let checked = Kernel::resolve(Add, Variant::checked(wide), [wide; 2]).unwrap(); let slot = ErrorSlot::new(); let mut out = Buffer::with_capacity(wide, 1); slot.within(|| crate::support::map(&checked, [i64::view(&[i64::MAX]), i64::view(&[1])], 1, &mut out)).unwrap(); assert!(slot.pending()); - let wrapped = Add.resolve(Variant { wrapping: true, ..Variant::checked(wide) }, [wide; 2]).unwrap(); + let wrapped = Kernel::resolve(Add, Variant { wrapping: true, ..Variant::checked(wide) }, [wide; 2]).unwrap(); let mut out = Buffer::with_capacity(wide, 1); error::checked(|| crate::support::map(&wrapped, [i64::view(&[i64::MAX]), i64::view(&[1])], 1, &mut out)).unwrap(); assert_eq!(i64::vec(out).unwrap(), [i64::MIN]); let range = wrapped.range_on(&[Range::of(Some(i64::MAX as f64)), Range::of(Some(1.))], false); assert!(Range::of(Some(i64::MIN as f64)).within(range.lo, range.hi)); - let binom = Binom.resolve(Variant { wrapping: true, ..Variant::checked(wide) }, [wide; 2]).unwrap(); + let binom = Kernel::resolve(Binom, Variant { wrapping: true, ..Variant::checked(wide) }, [wide; 2]).unwrap(); let mut out = Buffer::with_capacity(wide, 3); error::checked(|| crate::support::map(&binom, [i64::view(&[200, -200, 67]), i64::view(&[100, 101, 33])], 3, &mut out)).unwrap(); assert_eq!(i64::vec(out).unwrap(), [3674307795577560168, -7960601317545944800, -4220223336089263246]); let input = [Binding::full(i16::view(&[10, 2, 3]))]; let plan = ReadPlan::input(0, 3); let read = Read { sources: &input, plan: &plan }; - let subtract = Subtract.resolve(Variant::checked(small), [small; 2]).unwrap(); + let subtract = Kernel::resolve(Subtract, Variant::checked(small), [small; 2]).unwrap(); let mut out = Buffer::with_capacity(small, 3); subtract.scan([1, 3, 1], Seed::Identity, Direction::Left, Settings { tolerance: rustymath::tolerant::Tolerance::DEFAULT }, read, &mut out).unwrap(); assert_eq!(i16::vec(out).unwrap(), [10, 8, 5]); + let (matrix, row) = (Layout::from([3, 2]), Layout::from([2])); + let values = ArrayView::compact(&matrix, i16::view(&[1, 2, 3, 4, 5, 6])); + let starts = ArrayView::compact(&row, i16::view(&[100, 200])); + for (seed, expected) in [(Seed::Identity, [1, 2, -2, -2, -7, -8]), (Seed::Initial(starts.read()), [99, 198, 96, 194, 91, 188])] { + let mut out = Buffer::with_capacity(small, 6); + subtract.scan([1, 3, 2], seed, Direction::Left, Settings { tolerance: Tolerance::EXACT }, values.read(), &mut out).unwrap(); + assert_eq!(i16::vec(out).unwrap(), expected); + } let seed = [Binding::full(i64::view(&[100]))]; let scalar = ReadPlan::input(0, 1); let seed = Read { sources: &seed, plan: &scalar }; @@ -601,7 +651,7 @@ mod operation { .unwrap(); assert_eq!(i16::vec(out).unwrap(), [-89]); let dtype = f64::DTYPE; - let residue = Residue(Tolerance::EXACT).resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); + let residue = Kernel::resolve(Residue(Tolerance::EXACT), Variant::checked(dtype), [dtype; 2]).unwrap(); let values = [[3.], [3. - 1e-14]]; let bindings = values.each_ref().map(|x| Binding::full(f64::view(x))); let reads = bindings.each_ref().map(|x| Read { sources: std::slice::from_ref(x), plan: &scalar }); @@ -612,7 +662,7 @@ mod operation { assert_eq!(f64::vec(out).unwrap(), [Residue(tolerance).one(3., values[1][0])]); } for table in 0..16 { - let kernel = Bitwise(table).resolve(Variant::checked(DType::Boolean), [DType::Boolean; 2]).unwrap(); + let kernel = Kernel::resolve(Bitwise(table), Variant::checked(DType::Boolean), [DType::Boolean; 2]).unwrap(); let mut out = Buffer::with_capacity(DType::Boolean, 4); crate::support::map(&kernel, [bool::view(&[false, false, true, true]), bool::view(&[false, true, false, true])], 4, &mut out).unwrap(); assert_eq!(bool::vec(out).unwrap(), (0..4).map(|i| table >> i & 1 != 0).collect::>()); @@ -621,11 +671,11 @@ mod operation { #[test] fn inverse_folds_keep_logical_order() { - use rustymath::{Divide, MathishDivide, Subtract, operation::{Piece, Total}}; + use rustymath::{Divide, MathishDivide, Subtract, reduce::{Piece, Total}}; let dtype = DType::Float(rustymath::buffer::FloatWidth::F64); - let subtract = Subtract.resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); - let divide = Divide.resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); - let mathish = MathishDivide.resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); + let subtract = Kernel::resolve(Subtract, Variant::checked(dtype), [dtype; 2]).unwrap(); + let divide = Kernel::resolve(Divide, Variant::checked(dtype), [dtype; 2]).unwrap(); + let mathish = Kernel::resolve(MathishDivide, Variant::checked(dtype), [dtype; 2]).unwrap(); let fold = |kernel: &Kernel, shape: &[usize], axis, mapping: Mapping, data: &Buffer| { let layout = Layout::from(shape.to_vec()); let input = view(&layout, data.view(), &mapping); @@ -645,7 +695,7 @@ mod operation { assert_eq!(f64::vec(fold(&subtract, &[2], 0, reverse, &f64::buffer(vec![1., 2.]))).unwrap(), [1.]); let windows = Mapping::from(vec![Steps::along(2, 1), Steps::along(3, 1)]); assert_eq!(f64::vec(fold(&subtract, &[2, 3], 1, windows, &f64::buffer(vec![1., 2., 3., 4.]))).unwrap(), [2., 3.]); - let add = rustymath::Add.resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); + let add = Kernel::resolve(rustymath::Add, Variant::checked(dtype), [dtype; 2]).unwrap(); let windows = Mapping::from(vec![Steps::along(2, 6), Steps::along(4, 1), Steps::along(3, 6), Steps::along(3, 1)]); let data = f64::buffer((0..24).map(|n| 0.5 * f64::from(n)).collect()); let rows = fold(&add, &[2, 4, 3, 3], 3, windows, &data); @@ -675,21 +725,21 @@ mod operation { .is_err() ); let byte = DType::Integer(Width::U8); - let wrapped = Subtract.resolve(Variant { wrapping: true, ..Variant::checked(byte) }, [byte; 2]).unwrap(); + let wrapped = Kernel::resolve(Subtract, Variant { wrapping: true, ..Variant::checked(byte) }, [byte; 2]).unwrap(); assert_eq!(u8::vec(fold(&wrapped, &[2], 0, Mapping::Linear(1), &u8::buffer(vec![0, 255]))).unwrap(), [1]); let range = wrapped.range_on(&[Range::of(Some(0.)), Range::of(Some(255.))], false); assert!(Range::of(Some(1.)).within(range.lo, range.hi)); let complex = DType::Complex(rustymath::buffer::ComplexWidth::F64); - let quotient = Divide.resolve(Variant::checked(complex), [complex; 2]).unwrap(); + let quotient = Kernel::resolve(Divide, Variant::checked(complex), [complex; 2]).unwrap(); let data = num_complex::Complex64::buffer(vec![num_complex::Complex64::new(0., 2.), num_complex::Complex64::new(0., 1.)]); assert_eq!(num_complex::Complex64::vec(fold("ient, &[2], 0, Mapping::Linear(1), &data)).unwrap(), [num_complex::Complex64::new(2., 0.)]); } #[test] fn folds_keep_uncalled_values_and_scans_keep_their_carry() { - use rustymath::{Add, tolerant::{Less, Tolerance}}; + use rustymath::{Add, Binary, tolerant::{Less, Tolerance}}; let dtype = DType::Integer(Width::I64); - let less = Less(Tolerance::EXACT).resolve(Variant::checked(dtype), [dtype; 2]).unwrap(); + let less = Kernel::resolve(Less(Tolerance::EXACT), Variant::checked(dtype), [dtype; 2]).unwrap(); let input = [Binding::full(i64::view(&[3, 2, 1]))]; let seed = [Binding::full(i64::view(&[7]))]; let scalar = ReadPlan::input(0, 1); @@ -703,36 +753,59 @@ mod operation { assert_eq!(out.dtype(), if len < 2 { dtype } else { DType::Boolean }); assert_eq!(i64::vec(Buffer::cast_results(out.view(), dtype, true).unwrap()).unwrap(), [expected]); } + for inner in [1, 2] { + let outer = rustymath::CHUNK / 3 + 1; + let data: Vec = (0..outer * 3 * inner).map(|i| [3, 2, 1][i / inner % 3]).collect(); + let input = [Binding::full(i64::view(&data))]; + let plan = ReadPlan::input(0, data.len()); + for (direction, expected) in [(Direction::Right, false), (Direction::Left, true)] { + let mut out = Buffer::with_capacity(DType::Boolean, outer * inner); + less.fold_read( + &[outer, 3, inner], Some(1), Seed::Identity, direction, Settings { tolerance: Tolerance::EXACT }, Read { sources: &input, plan: &plan }, + &mut out).unwrap(); + assert_eq!(bool::of(out.view()).unwrap(), vec![expected; outer * inner]); + } + } let plan = ReadPlan::input(0, 3); let read = Read { sources: &input, plan: &plan }; let mut out = Buffer::with_capacity(less.scan_dtype(false), 3); less.scan([1, 3, 1], Seed::Identity, Direction::Left, Settings { tolerance: rustymath::tolerant::Tolerance::EXACT }, read, &mut out).unwrap(); assert_eq!(i64::vec(out).unwrap(), [3, 0, 1]); // Packed comparisons keep raw seeds and numeric heads across word and read-chunk boundaries. - for len in [63, 64, 65, 5000] { - let data: Vec = (0..len).map(|i| (i % 5) as i64 - 2).collect(); - let input = [Binding::full(i64::view(&data))]; - let plan = ReadPlan::input(0, len); - for initial in [None, Some(7)] { - for direction in [Direction::Left, Direction::Right] { - let mut ordered = data.clone(); - if direction == Direction::Right { ordered.reverse(); } - let (mut carry, mut expected) = (initial, Vec::new()); - for x in ordered { - let value = carry.map_or(x, |c| i64::from(if direction == Direction::Left { c < x } else { x < c })); - expected.push(value); - carry = Some(value); + for (dtype, tolerance) in + [i16::DTYPE, i64::DTYPE, f32::DTYPE, f64::DTYPE].into_iter().flat_map(|dtype| [Tolerance::EXACT, Tolerance::DEFAULT].map(|t| (dtype, t))) { + let less = Kernel::resolve(Less(tolerance), Variant::checked(dtype), [dtype; 2]).unwrap(); + let raw = Buffer::cast_results(i64::view(&[7]), dtype, true).unwrap(); + let seed = [Binding::full(raw.view())]; + for len in [0, 1, 63, 64, 65, 5000] { + let data: Vec = (0..len).map(|i| [-2., -1e-20, 0., 1., 1. + 1e-15, 1. + 1e-13, 2.][i % 7]).collect(); + let typed = Buffer::cast_results(f64::view(&data), dtype, true).unwrap(); + let data = f64::vec(Buffer::cast_results(typed.view(), f64::DTYPE, true).unwrap()).unwrap(); + let input = [Binding::full(typed.view())]; + let plan = ReadPlan::input(0, len); + for initial in [None, Some(7.)] { + for direction in [Direction::Left, Direction::Right] { + let mut ordered = data.clone(); + if direction == Direction::Right { ordered.reverse(); } + let (mut carry, mut expected) = (initial, Vec::new()); + for x in ordered { + let value = carry.map_or(x, |c| { + u8::from(if direction == Direction::Left { Less(tolerance).one(c, x) } else { Less(tolerance).one(x, c) }) as f64 + }); + expected.push(value); + carry = Some(value); + } + if direction == Direction::Right { expected.reverse(); } + let seed = if initial.is_some() { Seed::Initial(Read { sources: &seed, plan: &scalar }) } else { Seed::Identity }; + let mut out = Buffer::with_capacity(less.scan_dtype(initial.is_some()), len); + less.scan([1, len, 1], seed, direction, Settings { tolerance }, Read { sources: &input, plan: &plan }, &mut out).unwrap(); + assert_eq!(out.dtype(), if initial.is_some() { DType::Boolean } else { dtype }); + assert_eq!(f64::vec(Buffer::cast_results(out.view(), f64::DTYPE, true).unwrap()).unwrap(), expected); + } } - if direction == Direction::Right { expected.reverse(); } - let seed = if initial.is_some() { Seed::Initial(Read { sources: &seed, plan: &scalar }) } else { Seed::Identity }; - let mut out = Buffer::with_capacity(less.scan_dtype(initial.is_some()), len); - less.scan([1, len, 1], seed, direction, Settings { tolerance: Tolerance::EXACT }, Read { sources: &input, plan: &plan }, &mut out).unwrap(); - assert_eq!(out.dtype(), if initial.is_some() { DType::Boolean } else { dtype }); - assert_eq!(i64::vec(Buffer::cast_results(out.view(), dtype, true).unwrap()).unwrap(), expected); } } - } - let sum = Add.resolve(Variant::checked(DType::Integer(Width::I16)), [DType::Integer(Width::I16); 2]).unwrap(); + let sum = Kernel::resolve(Add, Variant::checked(DType::Integer(Width::I16)), [DType::Integer(Width::I16); 2]).unwrap(); let narrow = [Binding::full(i16::view(&[3]))]; let mut out = Buffer::with_capacity(sum.fold_dtype(DType::Integer(Width::I16), 1, None), 1); sum.fold_read( @@ -751,10 +824,10 @@ mod operation { let reversed = Mapping::Steps { base: 0, steps: vec![Steps::Stride { start: 4, len: 3, step: -2 }, Steps::along(2, 1)].into() }; let composed = transpose.compose(&reversed, shape.shape(), shape.shape()).unwrap(); assert_eq!(view(&shape, f64::view(&data), &composed).items::().unwrap().as_ref(), &[2., 5., 1., 4., 0., 3.]); - let add = rustymath::Add.resolve(Variant::checked(f64::DTYPE), [f64::DTYPE; 2]).unwrap(); + let add = Kernel::resolve(rustymath::Add, Variant::checked(f64::DTYPE), [f64::DTYPE; 2]).unwrap(); let compact = || ArrayView::compact(&shape, f64::view(&data)); assert_eq!(f64::vec(apply(&add, [x, compact()]).unwrap()).unwrap(), [0., 4., 3., 7., 6., 10.]); - let clip = rustymath::Clip.resolve(Variant::checked(f64::DTYPE), [f64::DTYPE; 3]).unwrap(); + let clip = Kernel::resolve(rustymath::Clip, Variant::checked(f64::DTYPE), [f64::DTYPE; 3]).unwrap(); let args = [ view(&shape, f64::view(&data), &transpose), ArrayView::compact(&Layout::EMPTY, f64::view(&[1.])), ArrayView::compact(&Layout::EMPTY, f64::view(&[4.])), @@ -825,7 +898,7 @@ mod operation { fn characters>(k: K, x: &Buffer, y: &Buffer) -> Result { let (lx, ly) = (Layout::from([x.len()]), Layout::from([y.len()])); let dtype = x.dtype().common(y.dtype()).unwrap_or(DType::Integer(Width::I64)); - let kernel = k.resolve(Variant::checked(dtype), [x.dtype(), y.dtype()])?; + let kernel = Kernel::resolve(k, Variant::checked(dtype), [x.dtype(), y.dtype()])?; error::checked(|| apply(&kernel, [ArrayView::compact(&lx, x.view()), ArrayView::compact(&ly, y.view())])) } @@ -845,7 +918,7 @@ mod operation { assert_eq!(i64::vec(characters(Minimum, &ab, &ints).unwrap()), Some(vec![1, 2])); assert!(characters(Add, &text("\u{10FFFF}"), &i64::buffer(vec![1])).is_err()); let offsets = f64::buffer(vec![0.5, 1.000000000000001]); - let kernel = Add.resolve(Variant::checked(offsets.dtype()), [ab.dtype(), offsets.dtype()]).unwrap(); + let kernel = Kernel::resolve(Add, Variant::checked(offsets.dtype()), [ab.dtype(), offsets.dtype()]).unwrap(); let layout = Layout::from([2]); let (chars, offsets) = (ArrayView::compact(&layout, ab.view()), ArrayView::compact(&layout, offsets.view())); let mut output = Buffer::with_capacity(DType::Character, 2); @@ -863,7 +936,7 @@ mod operation { .is_err() ); let (empty, chosen) = (text(""), text("x")); - let kernel = Add.resolve(Variant::checked(ints.dtype()), [DType::Character, ints.dtype()]).unwrap(); + let kernel = Kernel::resolve(Add, Variant::checked(ints.dtype()), [DType::Character, ints.dtype()]).unwrap(); for (fill, want) in [(Missing::Default, vec!['!', '"']), (Missing::Scalar(1), vec!['y', 'z'])] { let plan = ReadPlan::copy(0, 0, Mapping::Tiled(0), 2, fill); let chars = ArrayView::new(&layout, vec![Binding::full(empty.view()), Binding::full(chosen.view())], plan).unwrap(); diff --git a/tests/slices.rs b/tests/slices.rs index 0ff3481..a8a223d 100644 --- a/tests/slices.rs +++ b/tests/slices.rs @@ -50,7 +50,7 @@ mod parallel { mod forms { use half::f16; - use rustymath::{reduce::{Maximum, Minimum}, *}; + use rustymath::{operation::Kernel, reduce::{Maximum, Minimum}, *}; use std::fmt::Debug; /// Whether `a` and `b` agree: both NaN, the same value, or finite and nonzero within 8 units in the last place of each other. A scalar form and its slice form can round differently, as `Pow`'s do at f32, where only the scalar form reads tables. Infinities and signs must match exactly, apart from a zero's sign, which can differ between routes: a vector maximum orders `-0` below `0`, where the scalar form gives its second argument on a tie. @@ -68,7 +68,7 @@ mod forms { } fn check, const A: usize>(k: &K, name: &str, args: [Arg<'_, F>; A], n: usize) { - let kernel = k.resolve(domain::Variant { checked: false, ..domain::Variant::checked(F::DTYPE) }, [F::DTYPE; A]).unwrap(); + let kernel = Kernel::resolve((*k).clone(), domain::Variant { checked: false, ..domain::Variant::checked(F::DTYPE) }, [F::DTYPE; A]).unwrap(); let views = args.each_ref().map(|a| { F::view(match a { Arg::One(v) => std::slice::from_ref(v), Arg::Many(s) => s }) }); @@ -129,7 +129,7 @@ mod forms { each_width!( unary: Exp, Expm1, Ln, Log1p, Log2, Log10, Sin, Cos, Tan, Asin, Acos, Atan, Sinh, Cosh, Tanh, Asinh, Acosh, Atanh, Cbrt, Gamma, Lgamma, Erf, Erfc, Erfinv, Erfcinv, LambertW, Sqrt, Sqrt1mx2, Sqrt1px2, Sqrtx2m1, Abs, Sign, Angle; - binary: Pow, Root, Hypot, Atan2, Gammainc, Gammaincc, Gammaincinv, Binom, Logb, Maximum, Minimum; ternary: Betainc, Betaincinv + binary: Pow, Root, Hypot, Atan2, Gammainc, Gammaincc, Gammaincinv, Binom, Logb, MathishLogb, Maximum, Minimum; ternary: Betainc, Betaincinv ); // `Erfcinv`'s f64 slice form places each block's items by region. Several blocks of uniform items, in random order, reach every region. use rand::RngExt; @@ -157,6 +157,27 @@ mod forms { for b in [2.0, 10.0] { check(&Logb, "Logb", [Arg::One(b), Arg::Many(&x)], x.len()); } } + #[test] + fn logarithm_slices_keep_permitted_nan_results() { + fn check() { + let items = [F::zero(), F::one(), F::infinity(), F::nan()]; + for mathish in [false, true] { + let variant = domain::Variant::checked(F::DTYPE); + let kernel = if mathish { Kernel::new(MathishLogb, variant) } else { Kernel::new(Logb, variant) }; + for args in [[&items[..], &items[..]], [&items[1..2], &items[..]], [&items[..], &items[1..2]]] { + let got = crate::support::mapped::(&kernel, args, items.len()).unwrap(); + for (i, &r) in got.iter().enumerate() { + let x = args.map(|s| s[if s.len() == 1 { 0 } else { i }]); + let want = if mathish { facts::formula_at(&MathishLogb, x) } else { facts::formula_at(&Logb, x) }; + assert!(same(r, want), "{x:?}: {r:?} != {want:?}"); + } + } + } + } + check::(); + check::(); + } + /// An output type other than the item type, at each float type, f16 included. A `bool` output is never NaN, so it never fails. #[test] fn outputs_can_differ_from_items() { @@ -176,6 +197,7 @@ mod reductions { use half::f16; use num_complex::Complex; use rustymath::reduce::*; + use std::fmt::Display; /// Each reduction and scan against the same taken item by item, in order: sums and running sums of `sums`, and the product of `prods`. Their items keep every sum and product exact at every width, so any grouping gives the same result. fn exact( @@ -249,7 +271,7 @@ mod reductions { }; let unit = |greatest: bool| if greatest { f64::NEG_INFINITY } else { f64::INFINITY }; let want = |items: &mut dyn Iterator, greatest: bool| items.fold(unit(greatest), |a, v| step(a, v.into(), greatest)); - let check = |got: T, want: f64, case: &str| { + let check = |got: T, want: f64, case: &dyn Display| { let got: f64 = got.into(); assert!(got.is_nan() || got == want, "{case}: {got}, where {want} or NaN"); }; @@ -275,7 +297,7 @@ mod reductions { let mut seen = start.map_or(unit(greatest), |s| step(unit(greatest), s.into(), greatest)); for (i, &got) in out.iter().enumerate() { seen = step(seen, x[i].into(), greatest); - check(got, seen, &format!("{case}, scan from {:?}, item {i}", start.map(Into::::into))); + check(got, seen, &format_args!("{case}, scan from {:?}, item {i}", start.map(Into::::into))); } } for inner in [1, 3, 7] { @@ -283,7 +305,7 @@ mod reductions { let data = &x[..rows * inner]; let column = |j: usize| (0..rows).map(move |r| data[r * inner + j]); let folds = if greatest { along::(data, [1, rows, inner]).0 } else { along::(data, [1, rows, inner]).0 }; - for j in 0..inner { check(folds[j], want(&mut column(j), greatest), &format!("{case}, column {j} of {inner}")); } + for j in 0..inner { check(folds[j], want(&mut column(j), greatest), &format_args!("{case}, column {j} of {inner}")); } for width in [3, 50].into_iter().filter(|&w| w <= rows && len <= 200) { let windows = if greatest { sliding::(data, [1, rows, inner], width).0 } else { sliding::(data, [1, rows, inner], width).0 }; @@ -291,7 +313,7 @@ mod reductions { for j in 0..inner { check( windows[w * inner + j], want(&mut (w..w + width).map(|r| data[r * inner + j]), greatest), - &format!("{case}, window {w} of {width} rows of {inner}")); + &format_args!("{case}, window {w} of {width} rows of {inner}")); } } } @@ -826,7 +848,7 @@ mod tolerant { #[test] fn residue_and_gcd() { - use rustymath::{buffer::{DType, FloatWidth}, domain::Variant, facts::Formula}; + use rustymath::{buffer::{DType, FloatWidth}, domain::Variant, operation::Kernel}; assert_eq!([(3.0, -5.0), (3.0, 5.0), (-3.0, -4.0), (-3.0, 4.0)].map(|(x, y)| T.residue(x, y)), [1.0, 2.0, -1.0, -2.0]); let r = [3.12, -1.0, -0.6].map(|y| T.residue(0.5, y)); assert!((r[0] - 0.12).abs() < 1e-15 && r[1] == 0.0 && (r[2] - 0.4).abs() < 1e-15); @@ -837,7 +859,7 @@ mod tolerant { assert!((Exactly.residue(0.1, 0.3) - 0.1).abs() < 1e-15); let (x, y) = (2.2500000000000018, 0.7500000000000007); let f64s = DType::Float(FloatWidth::F64); - let quotient = FloorDivide(Tolerance::EXACT).resolve(Variant::checked(f64s), [f64s; 2]).unwrap(); + let quotient = Kernel::resolve(FloorDivide(Tolerance::EXACT), Variant::checked(f64s), [f64s; 2]).unwrap(); assert_eq!( (quotient.real(&[x, y], rustymath::operation::Settings { tolerance: Tolerance::EXACT }).unwrap(), Residue(Tolerance::EXACT).one(y, x)), (2.0, x - 2.0 * y) diff --git a/tests/speed.rs b/tests/speed.rs index 35faf7e..c212e97 100644 --- a/tests/speed.rs +++ b/tests/speed.rs @@ -1,11 +1,11 @@ -//! How fast each function object's kernel runs on 65,536 inputs: the loop that its formula resolves to, which callers such as basedpl and the Python package run. `speed.txt` gives each function a budget at f64 and at f32, in copies of the same number of items: the most time it may take, as a multiple of the time to copy those items. Each copy's time is this machine's stored copy time, or one measured once at the start, as `timing::copy_time` gives it. A row whose footprint, `N` items of its type and the most bytes it holds at once, passes one core's cache adds that footprint and the time of a copy of it, measured beside the row, to its line, as `timing::footprint` describes. A loop that stops vectorizing takes several times as long and goes over budget. Where the standard library has the function, its time item by item is measured too, for the comparison table in `src/lib.md`, but sets no limit. The rows `Pow.one` and `Root.one` time the scalar formulas pair by pair, as a caller with one value at a time runs them. The inner products multiply 256×256 matrices, whose results hold the same number of items. The decompositions factor 256×256 real matrices and 128×128 complex ones, and `lstsq` solves a 4096×16 real one. Sorts, partitions, medians and variances run along the rows of a 256×256 matrix. One sort runs along two lanes of 32,768. A weighted average reduces the first and last axes of a 16×64×64 array. Each of these counts copies of as many items as its matrix holds. A row over its budget fails only if it stays over after each pause in `timing::RETRY_PAUSES`, when every row is measured again. Each retry prints the row's earlier and new costs. The test fails listing every cost still over its budget after the last retry, as its width, name, cost and budget, in the form that `basedpl.bpltests.logged_costs` reads. It fails on budgets only on the CPU they were set for, as `timing::checks_budgets` decides. On any other CPU, it writes what `RUSTYMATH_COSTS` writes, then each cost still over its budget, marked `over`, and fails on none. With `RUSTYMATH_COSTS` set, as in `RUSTYMATH_COSTS=1 cargo test --test speed`, the test writes that table, then every cost in that form, straight to stderr. The test harness captures `println!` output from a passing test, but not these lines. `DEV.md` lists the manual commands for collecting costs on the authoritative hosts. The test holds the machine's timing lock while it runs, as `rustymath::timing::exclusive` describes. Array guards time f32 differences on length-2 lanes and on one long vector, and a compact broadcast repeating each input 64 times. Safe i32-to-i64 widening counts i64 copies. These rows use only the demonstrated dtypes. Like `bpl` and rustymath's Python extension, this test registers the memory guard as its global allocator. Each row then allocates as a caller's process does, including the guard's request for huge pages on large blocks on Linux. +//! How fast each function object's kernel runs on 65,536 inputs: the loop that its formula resolves to, which callers such as basedpl and the Python package run. `speed.txt` gives each function a budget at f64 and at f32, in copies of the same number of items: the most time it may take, as a multiple of the time to copy those items. Each copy's time is this machine's stored copy time, or one measured once at the start, as `timing::copy_time` gives it. A row whose footprint, `N` items of its type and the most bytes it holds at once, passes one core's cache adds that footprint and the time of a copy of it, measured beside the row, to its line, as `timing::footprint` describes. A loop that stops vectorizing takes several times as long and goes over budget. Each row's first timed call also counts, for each item, the instructions that it retires and the bytes that it allocates, as `timing::instructions` and `timing::bytes` count them, and the row's line ends with those counts. Counts don't depend on what else the machine runs. Where `speed.txt` records this platform's counts for a row, a count more than `timing::COUNT_MARGIN` times its record fails, and a count below its record divided by that margin is marked `fewer than recorded`. The rows `Pow.one` and `Root.one` time the scalar formulas pair by pair, as a caller with one value at a time runs them. The inner products multiply 256×256 matrices, whose results hold the same number of items. The decompositions factor 256×256 real matrices and 128×128 complex ones, and `lstsq` solves a 4096×16 real one. Sorts, partitions, medians and variances run along the rows of a 256×256 matrix. One sort runs along two lanes of 32,768. A weighted average reduces the first and last axes of a 16×64×64 array. Each of these counts copies of as many items as its matrix holds. An ordinary run holds the machine's timing lock shared, beside builds and other tests, and runs every call on one thread (`timing::Hold::on_one_thread`). Where rows are over their budgets, it then takes the lock exclusively if no one else holds it (`timing::try_exclusive`), times every row again and fails each row still over. Where someone else holds the lock, it marks each of those rows `unconfirmed` and doesn't fail. With `RUSTYMATH_COSTS` set, as in `RUSTYMATH_COSTS=1 cargo test --test speed`, the test is a calibration run. It holds the lock exclusively throughout and times the standard library's version of each function that has one, for the comparison table in `src/lib.md`. It writes that table, then every cost, straight to stderr. While any row is over its budget, it times every row again after each pause in `timing::RETRY_PAUSES`, printing each such row's earlier and new costs. The test fails listing every row still over its budget, as its width, name, cost and budget, in the form that `basedpl.bpltests.logged_costs` reads. It fails on budgets and counts only on the CPU they were set for, as `timing::checks_budgets` decides. On any other CPU, it writes every cost, then each cost still over its budget, marked `over`, and fails on none. The test harness captures `println!` output from a passing test, but not these lines. `DEV.md` lists the manual commands for collecting costs on the authoritative hosts. Array guards time f32 differences on length-2 lanes and on one long vector, and a compact broadcast repeating each input 64 times. Safe i32-to-i64 widening counts i64 copies. These rows use only the demonstrated dtypes. Like `bpl` and rustymath's Python extension, this test registers the memory guard as its global allocator, here inside `timing::Counting`, which counts each thread's bytes. Each row then allocates as a caller's process does, including the guard's request for huge pages on large blocks on Linux. use rand::seq::SliceRandom; -use rustymath::timing::RUNS; -use rustymath::{reduce::Reduction, *}; +use rustymath::{operation::Kernel, reduce::Reduction, *}; +use rustymath_timing::{self as timing, RUNS}; use std::{collections::HashMap, hint::black_box, io::Write, marker::PhantomData, ops::Neg}; #[global_allocator] -static GUARD: memory::Guard = memory::Guard::new(); +static GUARD: timing::Counting = timing::Counting(memory::Guard::new()); pub mod support; const N: usize = 65_536; @@ -13,13 +13,29 @@ const N: usize = 65_536; type Compared = (&'static str, [f64; 4], [Timed; 2]); /// A function's name and its costs, at f64 and at f32. type Costed = (&'static str, [Timed; 2]); -/// A cost in copies, the time per item in ns that gives it, and the case's footprint where it passes one core's cache, with a copy of it measured beside the case (`timing::footprint`). -type Timed = (f64, f64, Option); +/// A cost in copies, the time per item in ns that gives it, the case's footprint where it passes one core's cache, with a copy of it measured beside the case (`timing::footprint`), and its counts for each item, as `Measured` gives them. +type Timed = (f64, f64, Option, [Option; 2]); +/// A row's item type, its function's name and its `Timed` cost. +type Row = (&'static str, &'static str, Timed); -/// The fastest of `RUNS` runs of `run`, as `timing::fastest` times them, in ns for each of `n` items. First it reads one byte in every 64 of each of `inputs`, which hold the sets that `set` gives the runs. That brings every set into cache, whatever ran since the sets were built, without calling the function on them, so its branch predictor still meets each set fresh. Runs that read the same inputs each time pass none: their first run brings them into cache. -fn fastest(n: usize, inputs: &[&[u8]], run: impl FnMut(usize)) -> f64 { +/// Whether this is a calibration run, which `RUSTYMATH_COSTS` asks for, as the module's docs describe. +fn calibrating() -> bool { std::env::var_os("RUSTYMATH_COSTS").is_some() } + +/// What one call of `fastest` measures: the time of the fastest call in ns for each item, and the counts of the first call for each item, the instructions that it retired and the bytes that it allocated, in the order of `timing::MEASURES`. +#[derive(Clone, Copy)] +struct Measured { ns: f64, counts: [Option; 2] } + +/// The fastest of `run`'s calls, as `timing::fastest` makes and times them, in ns for each of `n` items, with the counts of its first call. First it reads one byte in every 64 of each of `inputs`, which hold the sets that `set` gives the runs. That brings every set into cache, whatever ran since the sets were built, without calling the function on them, so its branch predictor still meets each set fresh. Runs that read the same inputs each time pass none: their first run brings them into cache. +fn fastest(n: usize, inputs: &[&[u8]], mut run: impl FnMut(usize)) -> Measured { for x in inputs { for &b in x.iter().step_by(64) { black_box(b); } } - timing::fastest(run) * 1e9 / n as f64 + let per_item = |count: f64| count / n as f64; + let mut counts = [None; 2]; + let seconds = timing::fastest(std::time::Duration::ZERO, |k| { + if k > 0 { return run(k); } + let (((), instructions), bytes) = timing::bytes(|| timing::instructions(|| run(0))); + counts = [instructions.map(|i| per_item(i as f64)), Some(per_item(bytes.allocated as f64))]; + }); + Measured { ns: seconds * 1e9 / n as f64, counts } } /// Run `k`'s inputs, from `x`, which holds `RUNS` sets of them one after another. Each run reads inputs that no earlier run has read. A branch predictor learns the outcomes of up to tens of thousands of items it has seen. On repeated inputs, a function that branches on its data runs as fast as on sorted data. @@ -32,7 +48,7 @@ fn bytes(x: &[T]) -> &[u8] { } /// The fastest of `RUNS` runs of `f`, which writes `n` outputs into a slice, in ns per item. `inputs` are as `fastest` takes them. -fn time_slice(n: usize, inputs: &[&[u8]], f: impl Fn(usize, &mut [T])) -> f64 { +fn time_slice(n: usize, inputs: &[&[u8]], f: impl Fn(usize, &mut [T])) -> Measured { let mut out = vec![T::default(); n]; fastest(n, inputs, |k| { f(k, &mut out); @@ -41,7 +57,7 @@ fn time_slice(n: usize, inputs: &[&[u8]], f: impl Fn(usize, & } /// The fastest of `RUNS` runs of `f`, which appends `n` outputs to an empty vector, in ns per item. `inputs` are as `fastest` takes them. -fn time_appended(n: usize, inputs: &[&[u8]], f: impl Fn(usize, &mut Vec)) -> f64 { +fn time_appended(n: usize, inputs: &[&[u8]], f: impl Fn(usize, &mut Vec)) -> Measured { let mut out = Vec::with_capacity(n); fastest(n, inputs, |k| { out.clear(); @@ -51,15 +67,15 @@ fn time_appended(n: usize, inputs: &[&[u8]], f: impl Fn(usize, &mut Vec)) } /// The time per item of `f` on each of the inputs in `x`. -fn time(x: &[A], f: impl Fn(A) -> T) -> f64 { +fn time(x: &[A], f: impl Fn(A) -> T) -> Measured { time_slice(x.len() / RUNS, &[bytes(x)], |k, out| { for (o, &v) in out.iter_mut().zip(black_box(set(x, k))) { *o = f(v); } }) } -fn time_map, const A: usize>(args: [&[F]; A], k: &K) -> f64 { +fn time_map, const A: usize>(args: [&[F]; A], k: &K) -> Measured { let n = args[0].len() / RUNS; - let kernel = k.resolve(domain::Variant::checked(F::DTYPE), [F::DTYPE; A]).unwrap(); + let kernel = Kernel::resolve((*k).clone(), domain::Variant::checked(F::DTYPE), [F::DTYPE; A]).unwrap(); let mut out = buffer::Buffer::with_capacity(kernel.result_dtype(), n); let inputs = args.map(bytes); fastest(n, &inputs, |r| { @@ -89,12 +105,12 @@ fn widths(lo: f64, hi: f64, step: f64, len: usize) -> (Vec, Vec) { (x, narrowed) } -/// A row of the comparison table in `src/lib.md`, `name`: the times of its own kernel and of the standard library's version, at f64 and then f32, and its own costs in copies. +/// A row of the comparison table in `src/lib.md`, `name`: the times of its own kernel and of the standard library's version, at f64 and then f32, and its own costs in copies. Only a calibration run times the standard library's version. Other runs give NaN for its times. fn compare( - units: &Units, name: &'static str, ours64: impl FnOnce() -> f64, std64: impl FnOnce() -> f64, ours32: impl FnOnce() -> f64, - std32: impl FnOnce() -> f64) -> Compared { - let (own64, s64) = (units.0.cost(ours64), std64()); - let (own32, s32) = (units.1.cost(ours32), std32()); + units: &Units, name: &'static str, ours64: impl FnOnce() -> Measured, std64: impl FnOnce() -> Measured, ours32: impl FnOnce() -> Measured, + std32: impl FnOnce() -> Measured) -> Compared { + let (own64, s64) = (units.0.cost(ours64), if calibrating() { std64().ns } else { f64::NAN }); + let (own32, s32) = (units.1.cost(ours32), if calibrating() { std32().ns } else { f64::NAN }); (name, [own64.1, s64, own32.1, s32], [own64, own32]) } @@ -109,7 +125,7 @@ macro_rules! unary { } /// The time per item of `f` on each pair of `x` and `y`, called one pair at a time. -fn time_pairs(x: &[F], y: &[F], f: impl Fn(F, F) -> T) -> f64 { +fn time_pairs(x: &[F], y: &[F], f: impl Fn(F, F) -> T) -> Measured { let n = x.len() / RUNS; time_slice(n, &[bytes(x), bytes(y)], |r, out| { let (x, y) = (black_box(set(x, r)), set(y, r)); @@ -199,10 +215,10 @@ struct Unit(timing::CopyTime, PhantomData); impl> Unit { fn new() -> Self { Self(timing::copy_time::(N), PhantomData) } - /// The cost in copies, the time per item in ns, and the footprint of `run`, which gives a time per item in ns. The footprint counts `N` items of `T` as the case's input, with the most bytes that `run` held at once. - fn cost(&self, run: impl FnOnce() -> f64) -> Timed { - let (t, held) = memory::peak_held(run); - (t / (self.0.seconds * 1e9), t, timing::footprint::(N, held)) + /// The cost in copies, the time per item in ns, the footprint and the counts of `run`, which gives what one call of `fastest` measured. The footprint counts `N` items of `T` as the case's input, with the most bytes that `run` held at once. + fn cost(&self, run: impl FnOnce() -> Measured) -> Timed { + let (m, held) = memory::peak_held(run); + (m.ns / (self.0.seconds * 1e9), m.ns, timing::footprint::(N, held), m.counts) } } @@ -211,7 +227,7 @@ struct Units(Unit, Unit); impl, B: Copy + Default + Neg> Units { /// The costs of `run64` and `run32`, each a time per item, against the unit at its own width. - fn costs(&self, run64: impl FnOnce() -> f64, run32: impl FnOnce() -> f64) -> [Timed; 2] { [self.0.cost(run64), self.1.cost(run32)] } + fn costs(&self, run64: impl FnOnce() -> Measured, run32: impl FnOnce() -> Measured) -> [Timed; 2] { [self.0.cost(run64), self.1.cost(run32)] } } /// The functions that the standard library lacks, with their costs in copies of floats against `units`. The incomplete gamma and beta functions iterate, and run on 1,024 inputs. The others run on 16,384. @@ -370,7 +386,7 @@ fn orderings(units: &Units) -> [Costed; 6] { } /// Allocated array results at the demonstrated dtypes. Differences and widening count copies of `N` input items; broadcasting counts `N` output items. -fn arrays(unit: &Unit) -> [(&'static str, &'static str, Timed); 4] { +fn arrays(unit: &Unit) -> [Row; 4] { use array::{Array, ArrayView}; use buffer::{Buffer, DType, Floats, Ints, View, Width}; use layout::Layout; @@ -447,16 +463,15 @@ fn decompositions(units: &Units, complex_units: &Units HashMap<(&str, &str), Option> { - timing::budgeted(text, timing::NATIVE_COLUMN).expect("speed.txt is a budget file with this machine's budgets").into_iter().map(|(s, c, b)| ((s, c), b)) - .collect() +/// What `speed.txt` sets for each function at each item type, keyed by `(type, name)`: its budget, as `timing::budgeted` gives it for the column that `timing::NATIVE_COLUMN` picks, and the counts that it records in that column, in the order of `timing::MEASURES`. A machine with no column has `None` for each. +fn budgets(text: &str) -> HashMap<(&str, &str), (Option, [Option; 2])> { + let column = timing::NATIVE_COLUMN; + timing::budgeted(text, column).expect("speed.txt is a budget file with this machine's budgets").into_iter() + .map(|(case, budget)| ((case.section, case.code), (budget, timing::MEASURES.map(|m| case.count(m, column))))).collect() } /// Every row's comparison with the standard library, where there is one, and every row's cost in copies, as `(width, name, cost)`. -fn measured( - units: &Units, complex_units: &Units, num_complex::Complex> -) -> (Vec, Vec<(&'static str, &'static str, Timed)>) { +fn measured(units: &Units, complex_units: &Units, num_complex::Complex>) -> (Vec, Vec) { let mut compared = unary! { units; Exp vs exp on -20.0 to 20.0; @@ -507,41 +522,70 @@ fn measured( #[test] #[cfg_attr(debug_assertions, ignore = "debug assertions change the costs it times")] fn speed() { - let _lock = timing::exclusive().unwrap(); + let calibrating = calibrating(); + let lock = (if calibrating { timing::exclusive() } else { timing::shared().map(timing::Hold::on_one_thread) }).unwrap(); let units: Units = Units(Unit::new(), Unit::new()); let complex_units: Units, num_complex::Complex> = Units(Unit::new(), Unit::new()); let (compared, costed) = measured(&units, &complex_units); let checks = timing::checks_budgets(); - let print_all = !checks || std::env::var_os("RUSTYMATH_COSTS").is_some(); + let print_all = !checks || calibrating; let mut stderr = std::io::stderr(); if print_all { let copies = [units.0.0, units.1.0, complex_units.0.0, complex_units.1.0]; for (i, t) in copies.iter().enumerate() { if !copies[..i].iter().any(|u| (u.bytes, u.items) == (t.bytes, t.items)) { writeln!(stderr, "{}", timing::copy_line(t)).unwrap(); } } + } + if calibrating { writeln!(stderr, "| Function | f64 | std f64 | f32 | std f32 |\n|---|---|---|---|---|").unwrap(); for (name, t, _) in compared.iter() { writeln!(stderr, "| `{name}` | {:.2} | {:.2} | {:.2} | {:.2} |", t[0], t[1], t[2], t[3]).unwrap(); } writeln!(stderr).unwrap(); } let budgets = budgets(include_str!("speed.txt")); - let line = |&(width, name, (cost, ns, footprint)): &(&str, &str, Timed)| match budgets.get(&(width, name)) { - Some(budget) => timing::line(width, name, cost, &budget.map_or("-".into(), |b| b.to_string()), ns * 1e-9, footprint), + let line = |&(width, name, (cost, ns, footprint, counts)): &Row| match budgets.get(&(width, name)) { + Some((budget, _)) => timing::line(width, name, cost, &budget.map_or("-".into(), |b| b.to_string()), ns * 1e-9, footprint, counts), None => format!("{width} {name}: {cost:.2}, with no budget in speed.txt"), }; - let budget = |&(width, name, _): &(&str, &str, Timed)| budgets.get(&(width, name)).copied().flatten(); - let over = |row: &(&str, &str, Timed)| budget(row).is_some_and(|b| b.over(row.2.0)); - let mut failures = Vec::new(); + let budget = |&(width, name, _): &Row| budgets.get(&(width, name)).and_then(|b| b.0); + let over = |row: &Row| budget(row).is_some_and(|b| b.over(row.2.0)); + let (mut failures, mut counted_over) = (Vec::new(), Vec::new()); for row in &costed { if print_all { writeln!(stderr, "{}", line(row)).unwrap(); } - if !budgets.contains_key(&(row.0, row.1)) { failures.push(line(row)); } + let Some((_, recorded)) = budgets.get(&(row.0, row.1)) else { failures.push(line(row)); continue; }; + for ((measure, record), count) in timing::MEASURES.iter().zip(recorded).zip(row.2.3) { + let (Some(record), Some(count)) = (record, count) else { continue }; + if record.over(count, timing::COUNT_MARGIN) { + counted_over.push(format!("{}: {measure} over the {} recorded", line(row), record.value)); + } else if count * timing::COUNT_MARGIN < record.value { + writeln!(stderr, "{}: fewer {measure} than the {} recorded", line(row), record.value).unwrap(); + } + } + } + let retime = + |slow: &[Row]| -> Vec { measured(&units, &complex_units).1.into_iter().filter(|row| slow.iter().any(|s| (s.0, s.1) == (row.0, row.1))).collect() }; + let mut slow: Vec = costed.iter().copied().filter(over).collect(); + if calibrating { + slow = timing::retried(slow, |slow, pause| { + for row in &slow { writeln!(stderr, "{}: over, timed again after {pause} s", line(row)).unwrap(); } + retime(&slow).into_iter().inspect(|row| writeln!(stderr, "{}", line(row)).unwrap()).filter(over).collect() + }); + } else if !slow.is_empty() { + drop(lock); + if let Some(_quiet) = timing::try_exclusive().unwrap() { + timing::warm(); + for row in &slow { writeln!(stderr, "{}: over, timed again with the timing lock held exclusively", line(row)).unwrap(); } + slow = retime(&slow).into_iter().inspect(|row| writeln!(stderr, "{}", line(row)).unwrap()).filter(over).collect(); + } else { + for row in slow.drain(..) { writeln!(stderr, "{}: unconfirmed, because builds or tests hold the timing lock", line(&row)).unwrap(); } + } + } + if checks { + failures.extend(slow.iter().map(line)); + failures.append(&mut counted_over); + } else { + for row in &slow { writeln!(stderr, "{}: over", line(row)).unwrap(); } + for text in &counted_over { writeln!(stderr, "{text}").unwrap(); } } - let slow = timing::retried(costed.iter().copied().filter(over).collect(), |slow, pause| { - for row in &slow { writeln!(stderr, "{}: over, timed again after {pause} s", line(row)).unwrap(); } - let again = measured(&units, &complex_units).1.into_iter().filter(|row| slow.iter().any(|s| (s.0, s.1) == (row.0, row.1))); - again.inspect(|row| writeln!(stderr, "{}", line(row)).unwrap()).filter(over).collect() - }); - if checks { failures.extend(slow.iter().map(line)); } - else { for row in &slow { writeln!(stderr, "{}: over", line(row)).unwrap(); } } for row in costed.iter().filter(|row| budget(row).is_some_and(|b| b.stale(row.2.0))) { writeln!(stderr, "{}: stale", line(row)).unwrap(); } for &(width, name) in budgets.keys().filter(|&&(w, n)| !costed.iter().any(|&(width, name, _)| (width, name) == (w, n))) { failures.push(format!("{width} {name} has a budget in speed.txt, but the test doesn't time it")); diff --git a/tests/speed.txt b/tests/speed.txt index 7dd63f2..88d3d8a 100644 --- a/tests/speed.txt +++ b/tests/speed.txt @@ -1,7 +1,8 @@ -# The budgets that `tests/speed.rs` checks. Each section starts with an item type in brackets. Each line under it holds a function's name and its two native budgets at that type, separated by tabs: first for macOS on Apple silicon, where matrix products and decompositions run on Accelerate, then for Linux on CI's x86 runner, where they run on rustymath's own kernels. `rustymath::timing::NATIVE_COLUMN` picks the budget a machine checks. A budget counts copies of as many items as the function's inputs hold, or its output for broadcasting. -# A budget is the function's expected cost times the multiplier that `rustymath::timing::budget` holds, as basedpl's DEV.md describes, and at least 0.5. A line such as `# held linux 14.2: REASON` above a function's line holds that column at the budget for the cost it gives, until the gap it names is fixed, and `# busy` does the same for CI's busy hosts, as `rustymath::timing::hold` reads them. The expected cost starts from a first-principles analysis of what the function should cost. A later expected cost comes from a fuller understanding of the function than that analysis gives, such as a platform difference or a cost the analysis missed. A comment above each function, or above its group, gives the expected cost and its reason, as in "1 copy: one read and one write per item", and records each later change with its reason. +# The budgets that `tests/speed.rs` checks. Each section starts with an item type in brackets. Each line under it holds a function's name and its two native budgets at that type, separated by tabs: first for macOS on Apple silicon, where matrix products and decompositions run on Accelerate, then for Linux on CI's x86 runner, where they run on rustymath's own kernels. `rustymath_timing::NATIVE_COLUMN` picks the budget a machine checks. A budget counts copies of as many items as the function's inputs hold, or its output for broadcasting. +# A budget is the function's expected cost times the multiplier that `rustymath_timing::budget` holds, as basedpl's DEV.md describes, and at least 0.5. A line such as `# held linux 14.2: REASON` above a function's line holds that column at the budget for the cost it gives, until the gap it names is fixed, and `# busy` does the same for CI's busy hosts, as `rustymath_timing::hold` reads them. The expected cost starts from a first-principles analysis of what the function should cost. A later expected cost comes from a fuller understanding of the function than that analysis gives, such as a platform difference or a cost the analysis missed. A comment above each function, or above its group, gives the expected cost and its reason, as in "1 copy: one read and one write per item", and records each later change with its reason. # One expected cost serves both platforms at first. A platform's budget differs only where a specific difference on that platform is found, such as Accelerate's matrix products on macOS. # A function that costs more than its budget needs study, which finds either a speed bug to fix or something the analysis missed. Avoid raising a budget to match a run without understanding why the function costs what it does. +# After its budgets, a function's line can record its counts for each item in more tab-separated fields, as `rustymath_timing::count` reads them, such as `instructions linux 33.1 macos 20.2` and `allocated linux 0`. `tests/speed.rs` fails a count more than `rustymath_timing::COUNT_MARGIN` times its record. `basedpl.bpltests.set_counts` writes the counts from a calibration run's log. # CI's x86 runner is an AMD Ryzen 9 7950X3D 16-Core Processor. # The macOS budgets come from an Apple M5 Max. On 23 October 2026 it measured Accelerate's rates for four classes of product, in flops in the time of a copy of the same item type, with real flops in a complex copy's time. A case whose product runs far from its class's rate gives its own measured rate. # - Large square products: 58 at f64 and 117 at f32 for 1024×1024 matrices, and 110 and 189 for complex ones. 42 and 48 for 128×128 matrices, and 90 and 137 for complex ones. @@ -10,344 +11,346 @@ # - Tall thin Gram products, a tall matrix's transpose times the matrix: about 7 at f64 and 4 at f32 for a 4096×16 matrix. [f64] # 5.75 copies, for 23 vector ops: a test for each block of 64 items that every item lies within ±708, then reduction by ln 2, a degree-9 polynomial and one power of two. A block of items below -746, such as one that a causal mask fills with -∞, writes zeros. Any other block takes the clamp and two powers of two, which cost about 1.35 times as much on macOS. -Exp 12 14 +Exp 12 14 instructions linux 10.288 allocated linux 0.002 # 5.75 copies, for 23 vector ops: exp's test and reduction, a degree-10 polynomial and one power of two, with the guards only in a block that holds an item beyond ±708. -Expm1 12 17 +Expm1 12 17 instructions linux 11.037 allocated linux 0.002 # 6.5 copies, for 26 vector ops: a test for each block of 64 items that every item is positive, normal and finite, then the split into mantissa and exponent, one division, a series in s² and the combine. A block of 64 items that holds a zero, a negative, a subnormal, an infinity or NaN takes the full form with its special values, which costs about 1.9 times as much on macOS. -Ln 13 15 +Ln 13 15 instructions linux 16.226 allocated linux 0.002 # 10 copies, for 40 vector ops: ln's steps plus the correction for 1 + x, with the block test on 1 + x and on whether x is 0. A block of 64 items that holds a zero, a negative, a subnormal, an infinity or NaN takes the full form with its special values, which costs about 1.6 times as much on macOS. -Log1p 20 16 +Log1p 20 16 instructions linux 18.725 allocated linux 0.002 # 6.5 copies, for 26 vector ops: ln's steps with a scaled combine. -Log2 13 11 +Log2 13 11 instructions linux 9.539 allocated linux 0.002 # 6.5 copies, for 26 vector ops: ln's steps with a scaled combine. -Log10 13 11 +Log10 13 11 instructions linux 9.804 allocated linux 0.002 # 6.5 copies, for 26 vector ops: reduction by π/2, a polynomial, the quadrant. -Sin 13 12 +Sin 13 12 instructions linux 12.788 allocated linux 0.002 # 7 copies, for 28 vector ops: sin's steps with a shifted quadrant. -Cos 14 12 +Cos 14 12 instructions linux 13.413 allocated linux 0.002 # 8 copies, for 32 vector ops: reduction, a polynomial, a division. -Tan 16 17 +Tan 16 17 instructions linux 13.913 allocated linux 0.002 # 7.5 copies, for 30 vector ops: a square root past 1/2, a polynomial, selects. -Asin 15 15 +Asin 15 15 instructions linux 12.975 allocated linux 0.002 # 8 copies, for 32 vector ops: asin's steps and a reflection. -Acos 16 16 +Acos 16 16 instructions linux 14.225 allocated linux 0.002 # 7 copies, for 28 vector ops: a reduction by division, a polynomial, selects. -Atan 14 16 +Atan 14 16 instructions linux 11.537 allocated linux 0.002 # 10 copies, for 40 vector ops: expm1, a division, selects. -Sinh 20 20 +Sinh 20 20 instructions linux 13.788 allocated linux 0.002 # 8 copies, for 32 vector ops: exp and its reciprocal. -Cosh 16 14 +Cosh 16 14 instructions linux 10.788 allocated linux 0.002 # 7.5 copies, for 30 vector ops: expm1 without its guards, which |x| up to 20 never needs, a division and the sign. -Tanh 15 15 +Tanh 15 15 instructions linux 13.537 allocated linux 0.002 # 11.5 copies, for 46 vector ops: ln, a square root, selects. -Asinh 23 25 +Asinh 23 25 instructions linux 16.538 allocated linux 0.002 # 11.5 copies, for 46 vector ops: ln and a square root. -Acosh 23 27 +Acosh 23 27 instructions linux 17.975 allocated linux 0.002 # 10.5 copies, for 42 vector ops: log1p of a quotient. -Atanh 21 21 +Atanh 21 21 instructions linux 16.975 allocated linux 0.002 # 9 copies, for 36 vector ops: split, a polynomial, two Newton steps with divisions. -Cbrt 18 19 +Cbrt 18 19 instructions linux 16.787 allocated linux 0.002 # 27.5 copies, for 110 vector ops: double-word ln and exp, special cases. -Pow 55 34 +Pow 55 34 instructions linux 38.065 allocated linux 0.005 # 55 copies, for Pow's 110 ops as scalar code, pair by pair, at half a copy each. -Pow.one 110 91 +# held linux 103: each pair calls the outlined scalar formula and its repair step through `facts::formula_at`; the released build took 21.7 ns a pair, against about 42 now +Pow.one 110 91 instructions linux 189.002 allocated linux 0 # 4 copies, for 16 vector ops: scaling, a square root, a division; two reads and a write take 1.5. -Hypot 8 7.2 +Hypot 8 7.2 instructions linux 5.558 allocated linux 0.005 # 28 copies, for Pow's steps with a reciprocal power. -Root 56 57 +Root 56 57 instructions linux 76.81 allocated linux 0.005 # 55 copies, for Root as scalar code, pair by pair. -Root.one 110 94 +# held linux 103: each pair calls the outlined scalar formula and its repair step through `facts::formula_at`; the released build took 18.0 ns a pair, against about 42 now +Root.one 110 94 instructions linux 172.002 allocated linux 0 # 1.5 copies, for two reads and one write per item. -Maximum 3 4.4 +Maximum 3 4.4 instructions linux 0.995 allocated linux 0.005 # 0.5 copies, for one read per item. -reduce::sum 1 1.5 +reduce::sum 1 1.5 instructions linux 0.329 allocated linux 0 # 0.5 copies, for one read per item. -reduce::max 1 1.6 +reduce::max 1 1.6 instructions linux 0.579 allocated linux 0 # 1 copies, for one read and one write per item; the scan runs in vector steps. -reduce::Sum::scan 2 3.6 +reduce::Sum::scan 2 3.6 instructions linux 3.855 allocated linux 0 # 1 copies, for one read and one write per item; the scan runs in vector steps. -reduce::Max::scan 2 2.9 +reduce::Max::scan 2 2.9 instructions linux 2.381 allocated linux 0 # 2.75 copies, for a read and a write (1) and 7 FMAs (1.75). It was 3.75 while `evaluate` worked in place, which its callers paid for with a copy of the input. -polynomial::evaluate 5.5 2.7 +polynomial::evaluate 5.5 2.7 instructions linux 4.662 allocated linux 0 # 132 copies, for 530 vector ops: a double-word Lanczos sum and quotient, double-word ln of the exponent, exp_plus, sinpi for the reflection. -Gamma 270 150 +Gamma 270 150 instructions linux 159.587 allocated linux 0.01 # 54 copies, for 215 vector ops: a plain Lanczos sum, double-word ln, ln_split, sinpi. -Lgamma 110 110 +Lgamma 110 110 instructions linux 102.088 allocated linux 0.01 # 17.5 copies, for 70 vector ops: both of erf's formulas for every item, x + x a(x²) with 13 terms and 1 - e^-(|x| + |x| b(|x|)) with 14 terms and the full `exp`, then the selects. macOS measures about 25 copies, about 1.45 times the count, for a reason not yet found. -Erf 35 42 +Erf 35 42 instructions linux 29.148 allocated linux 0.01 # 32 copies, for 130 vector ops: rational pieces with divisions, a double-word -a², exp_plus. -Erfc 64 87 +Erfc 64 87 instructions linux 55.399 allocated linux 0.01 # 20 copies, for 80 vector ops: piecewise rationals, ln and a square root in the tails. -Erfinv 40 61 +Erfinv 40 61 instructions linux 65.183 allocated linux 0.01 # 21 copies, for 85 vector ops: as erfinv, from 1 - q. -Erfcinv 42 72 +Erfcinv 42 72 instructions linux 85.206 allocated linux 0.01 # 450 copies, for 900 scalar ops: a double-word prefactor (300) and about 50 series or continued-fraction steps of 12 ops, at half a copy each. -Gammainc 900 840 +Gammainc 900 840 instructions linux 962.909 allocated linux 0.312 # 450 copies, for Gammainc's 900 scalar ops. -Gammaincc 900 840 +Gammaincc 900 840 instructions linux 962.907 allocated linux 0.312 # 1800 copies, for about four evaluations of Gammainc and its derivative, 3,600 scalar ops. -Gammaincinv 3600 2800 +Gammaincinv 3600 2800 instructions linux 2728.07 allocated linux 0.312 # 1200 copies, for 2,400 scalar ops: a double-word prefactor of two ln and three lgamma (1,500) and about 60 continued-fraction steps of 15 ops. -Betainc 2400 2000 +Betainc 2400 2000 instructions linux 2354.683 allocated linux 0.469 # 4800 copies, for about four evaluations of Betainc. -Betaincinv 9600 6200 +Betaincinv 9600 6200 instructions linux 6788.477 allocated linux 0.469 # Times the per-item kernel through `time_map`: `complex::exp` on each item, exp of the real part and sin and cos of the imaginary part. The expected cost of 10 copies, for 78 f64 ops per complex item, was worked out for the split slice form that this row timed before that form was deleted. The per-item kernel's cost isn't studied yet, and its Linux budget stays held. -complex Exp 20 20 +complex Exp 20 20 instructions linux 71.034 allocated linux 56.01 # 12.5 copies, for 100 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: hypot, ln with care near |z| = 1, atan2. It costs about 30 copies on macOS and 32.4 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -complex Ln 63 28 +complex Ln 63 28 instructions linux 96.481 allocated linux 64.01 # 5.1 copies, for 41 instructions per vector of two complex items, at a quarter copy each and two floats per complex copy: about 18 for each block's test of 64 items, and 23 for the plain form, two square roots and a division with no scaling or special cases. A block holding a nonzero part below 2^-511 or above 2^511, a NaN or ∞ part, or a 0 with both parts 0 takes the full form instead, about 64 instructions beside the test's 18, with `hypot`'s scaling, the scaling of tiny items, the special-value selects and the NaN check. It costs about 4.8 copies on macOS on 24 October 2026. It costs about 6.2 copies on CI on 24 October 2026, where it cost 10.2 through the full form alone on 22 October. -complex Sqrt 11 6.5 +complex Sqrt 11 6.5 instructions linux 21.236 allocated linux 0.01 # 16 copies, for 126 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: sin, cos, sinh and cosh. -complex Sin 32 25 +complex Sin 32 25 instructions linux 65.556 allocated linux 88.01 # 17.5 copies, for 140 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: sin, cos, sinh, cosh and a division. -complex Tan 35 28 +complex Tan 35 28 instructions linux 76.563 allocated linux 104.01 # 22.5 copies, for 180 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex square root of 1 - z², a complex ln, products. It costs 42.5 copies on CI on 22 October 2026, about twice that, where a budget of twice it would fail by chance, so its Linux budget is unchanged until the cost is studied. -complex Asin 45 46 +complex Asin 45 46 instructions linux 124.535 allocated linux 120.01 # 16 copies, for 130 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex ln of a quotient. It costs 31.1 copies on CI on 22 October 2026, about twice that, where a budget of twice it would fail by chance, so its Linux budget is unchanged until the cost is studied. -complex Atanh 32 29 +complex Atanh 32 29 instructions linux 88.432 allocated linux 96.01 # 52 copies, for 420 f64 ops per complex item: a complex Lanczos sum with 13 complex divisions, a complex ln. It costs 167 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. -complex Lgamma 110 130 +complex Lgamma 110 130 instructions linux 418.675 allocated linux 0.01 # 62 copies, for 500 f64 ops: complex Lgamma and a complex exp. It costs about 134.3 copies on macOS and 217 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -complex Gamma 280 160 +complex Gamma 280 160 instructions linux 592.675 allocated linux 0.01 # 2 copies, for 16 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: hypot; reading two floats and writing one take 0.75. -complex Abs 4 2.9 +complex Abs 4 2.9 instructions linux 6.898 allocated linux 0.01 # Times the per-item kernel through `time_map`: `complex::sqrt1mx2` on each item, two complex square roots, then a product and a rebalance. The expected cost of 11.5 copies, for 92 f64 ops per complex item, was worked out for the split slice form that this row timed before that form was deleted. Both budgets stay unchanged until the per-item kernel's cost is studied. -complex Sqrt1mx2 53 30 +complex Sqrt1mx2 53 30 instructions linux 93.822 allocated linux 104.01 # 23 copies, for 184 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex ln, a complex product, a complex exp. It costs about 50.6 copies on macOS and 55.3 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -complex Pow 110 53 +complex Pow 110 53 instructions linux 210.34 allocated linux 136.02 # 64 copies, for 256 FMAs per result item, a quarter copy each, on rustymath's own kernels. On macOS, 8.8 copies, for the same 512 flops at Accelerate's rate for a 1024×1024 product. -linalg::PlusTimes 18 98 +linalg::PlusTimes 18 98 instructions linux 145.994 allocated linux 20.125 # 0.5 copies: one read of the matrix, about half a copy, on rustymath's own kernels, where its one FMA per item, a quarter copy, runs alongside. On macOS, 0.29 copies, for 2 flops per item at Accelerate's matrix-vector rate. -linalg::PlusTimes by a vector 0.6 1.4 +linalg::PlusTimes by a vector 0.6 1.4 instructions linux 2.437 allocated linux 0.031 # 0.5 copies, as `linalg::PlusTimes by a vector`. -linalg::PlusTimes of a vector 0.6 0.79 +linalg::PlusTimes of a vector 0.6 0.79 instructions linux 0.838 allocated linux 0.094 # 128 copies, for 256 steps of a max and an add per result item, a quarter copy per op. -linalg::MaxPlus 260 120 +linalg::MaxPlus 260 120 instructions linux 212.912 allocated linux 20 # 128 copies, for 256 steps of a min and an add per result item, a quarter copy per op. -linalg::MinPlus 260 120 +linalg::MinPlus 260 120 instructions linux 212.912 allocated linux 20 # 128 copies, for 256 steps of a comparison and an add per result item, a quarter copy per op. -linalg::PlusEqual 260 120 +linalg::PlusEqual 260 120 instructions linux 209.009 allocated linux 20 # 68 copies: these values fit f64 products exactly, whose 256 multiply-adds per result item take 64 at a quarter copy each on rustymath's own kernels, with passes that find the operands' largest magnitudes, 1, convert both to f64, 2, and convert the result back, 1. On macOS, 13 copies, with Accelerate's product at its rate for a 1024×1024 product, 8.8, and the same passes. -linalg::PlusTimes at i64 26 95 +linalg::PlusTimes at i64 26 95 instructions linux 156.157 allocated linux 44.125 # 128 copies, for 256 steps of an integer comparison and an add per result item. -linalg::PlusEqual at i64 260 110 +linalg::PlusEqual at i64 260 110 instructions linux 211.944 allocated linux 20 # 16 copies, for a bitonic network on two-lane vectors (`search::networked`): 36 steps, each a minimum and a maximum for every two rows, about 8 copies, plus 15 passes that load and store every row, about 6, and about 2 for the shuffles of the steps across lanes, the write by columns and the NaN count. A block of 16 rows stays in registers for the steps within it. It costs about 17 copies on macOS. An unstable comparison sort of each row's keys cost about 41. -search::view_sort_along 32 21 +search::view_sort_along 32 21 instructions linux 35.767 allocated linux 8.156 # 60 copies, for an unstable comparison sort of each 32,768-item lane's keys: about 15 levels per item at about 4 copies a level. A radix sort of the keys costs about 71, at about 9 copies for each of its 8 passes. A sorting network would gain here only as the base case of a long sort, as "A vectorized sort kernel" in basedpl's `meta/rustymath.md` records. It costs about 60 copies on macOS, down from about 105 when each comparison computed two keys. -search::view_sort_along long lanes 120 92 +search::view_sort_along long lanes 120 92 instructions linux 141.632 allocated linux 16 # 15 copies, for a quickselect of each row's keys: about 3 comparisons per item at about 5 copies each, as the sorts measure comparisons, plus a pass that computes the keys and one that writes the floats back. A vectorized kernel that partitions would cost less, as "A vectorized sort kernel" in basedpl's `meta/rustymath.md` records. It costs about 15 copies on macOS, down from about 17 when each comparison computed two keys. CI measured 12.4 on 22 October 2026, so its Linux budget follows the same expected cost. -search::view_partition_along 30 17 +search::view_partition_along 30 17 instructions linux 28.2 allocated linux 8.156 # 15 copies, for selecting the two middle keys of each row, which costs about what one partition does. It costs about 15 copies on macOS, down from about 33 when each comparison computed two keys. CI measured 12.6 on 22 October 2026, so its Linux budget follows the same expected cost. -search::view_quantiles_along 30 18 +search::view_quantiles_along 30 18 instructions linux 30.231 allocated linux 0.188 # 1.5 copies, for two reads per item, for the mean and the squares, and two ops. -stats::variance 3 4.1 +stats::variance 3 4.1 instructions linux 3.101 allocated linux 0.564 # 1.5 copies, for reading the data and the weights once, with a multiply-add per item. It costs about 5.8 copies on macOS, because each chunk gathers the data and the weights into buffers, and a second pass gathers the weights again for their total. Two other methods measured slower on 21 October 2026: folding pairs of products and weights in one walk cost 6.9, because a buffer of pairs folds more slowly than the walk it saves, and reading each 64-item row in place cost 8.7, because each in-place read works out its row's offset with divisions and each row's fold pays its own setup. Reading the runs in place without copying them only ties with the gather at 64 items. A fold that sums each run straight into its output's accumulator, with no product buffer, would reach the expected cost, as step 4 in basedpl's `meta/next-phases.md` records. Both budgets stay held until then. -stats::average 74 9.6 +stats::average 74 9.6 instructions linux 9.106 allocated linux 9.049 # 22 copies, for 2n³/3 flops per n² items, as FMAs, plus the solve, at a quarter copy per FMA on rustymath's own kernels. It costs 44 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 10 copies: the copy into column order and the row swaps, 2.5, the products' 155 flops an item at Accelerate's rate for rank-32 updates, 5.2, and the 16-column leaves' 16 flops an item at the matrix-vector rate, 2.3. It costs 23.7 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::Lu::solve 53 60 +linalg::Lu::solve 53 60 instructions linux 106.343 allocated linux 28.843 # 64 copies, for 2n³ flops per n² items, at a quarter copy per FMA on rustymath's own kernels. CI measured 107 on 22 October 2026, so its Linux budget follows the expected cost. On macOS, 23 copies: the LU as `linalg::Lu::solve` gives it, 10, then the two triangular solves from the permuted identity, which skip no zeros, 512 flops an item at Accelerate's rate for 128×128 products, 12, and writing the inverse, 1. It costs 43 copies on macOS on 23 October 2026, about twice that, where a budget of twice it would fail by chance, so its macOS budget is unchanged until the cost is studied. -linalg::Lu::inverse 89 120 +linalg::Lu::inverse 89 120 instructions linux 275.197 allocated linux 67.023 # 11 copies, for n³/3 flops per n² items, at a quarter copy per FMA on rustymath's own kernels. It costs 23.2 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 4.8 copies: the copy in and the copy closed up, 2, and its n³/3 flops at Accelerate's rate for rank-32 updates, 2.8. It costs 10.8 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::cholesky 28 30 +linalg::cholesky 28 30 instructions linux 58.524 allocated linux 17.898 # 85 copies, for 8n³/3 flops per n² items, 4n³/3 to factor and 4n³/3 to form Q, at a quarter copy per FMA on rustymath's own kernels. It costs 163 copies on CI on 22 October 2026, about twice that, where a budget of twice it would fail by chance, so its Linux budget is unchanged until the cost is studied. On macOS, 25 copies: the working copy and the writes of Q and R, 2, and those flops at Accelerate's rate for rank-32 updates, 23. It costs 48.8 copies on macOS on 23 October 2026, about twice that, where a budget of twice it would fail by chance, so its macOS budget is unchanged until the cost is studied. -linalg::qr 130 170 +linalg::qr 130 170 instructions linux 411.716 allocated linux 129.611 # 384 copies, for about 12n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 868 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 143 copies, for Accelerate's GESDD: half the 8n³/3 flops of its bidiagonal reduction, which run as matrix-vector products, at the matrix-vector rate, 49, the rest of its 12n³ flops at the rate for rank-32 updates, 91, and three copies in and out. It costs 473 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::svd 970 790 +linalg::svd 970 790 instructions linux 1832.872 allocated linux 259.256 # 288 copies, for about 9n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 729 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 85 copies, for Accelerate's SYEVD: half the 4n³/3 flops of its tridiagonal reduction, which run as symmetric matrix-vector products that use each item they read twice, at twice the matrix-vector rate, 12, the rest of its 9n³ flops at the rate for rank-32 updates, 71, and two copies. It costs 318 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eigh 660 610 +linalg::eigh 660 610 instructions linux 1157.39 allocated linux 151.186 # 800 copies, for about 25n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 2840 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 234 copies, for Accelerate's GEEV: the 2n³/3 flops of its Hessenberg reduction that run as matrix-vector products, at the matrix-vector rate, 25, the rest of its 25n³ flops at the rate for rank-32 updates, 208, and two copies. It costs 1813 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eig 3600 3500 +linalg::eig 3600 3500 instructions linux 8255.355 allocated linux 145.8 # 320 copies, for about 10n³ flops per n² items, at a quarter copy per FMA on rustymath's own kernels. It costs 1486 copies on CI on 22 October 2026, more than twice that. On 23 and 24 October 2026 the same code cost 1053 to 1920 copies on CI's runner, and builds from before and after those dates cost the same on a Mac. Its Linux budget is 50% above the earlier 2700, to cover that variation in the runner, not a code change. On macOS, 105 copies, for GEEV without vectors: the same matrix-vector part, 25, the rest of its 10n³ flops at the rate for rank-32 updates, 80, and one copy. It costs 1176 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eigenvalues 2400 1400 +linalg::eigenvalues 2400 1400 instructions linux 3119.182 allocated linux 16.531 # 5 copies, for 2mn² flops on 4096×16 (32 per item) and reading the matrix, at a quarter copy per FMA on rustymath's own kernels. On CI's x86 runner it cost 35.4 copies on 22 October 2026, through CholeskyQR2, which then took this size on rustymath's own kernels. Earlier runs put Householder there about 1.5 times faster, and `tall_for_qr` now gives this size to Householder wherever Accelerate doesn't run. Householder's reflections are matrix-vector passes, far below the FMA rate this analysis assumes: on the Mac's own kernels, f64 costs about 32 copies. Through Householder, CI measured 29.5 later that day. Its Linux budget is unchanged until Householder's expected cost is worked out. On macOS, 12 copies, through CholeskyQR2: its two Gram matrices, 32 flops an item each, at Accelerate's rate for a 4096×16 matrix's transpose times the matrix, 4.8 each, the solve with R, 16 flops an item at about 16 flops in a copy's time, which Accelerate measured for a 4096×16 matrix times a 16×16 one, half its rank-k rate, 1, a copy of the matrix, 1, and the reduction of the right side, 0.5. -linalg::lstsq 24 47 +linalg::lstsq 24 47 instructions linux 28.516 allocated linux 17.409 # 21 copies, for 8n³/3 real flops per n² complex items, two floats per complex copy, at a quarter copy per FMA on rustymath's own kernels. It costs 61.9 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 9.3 copies: the copy into column order and the row swaps, 1.9, the products' 277 real flops an item at Accelerate's rate for complex rank-32 updates, 5.2, and the leaves' 64 at the complex matrix-vector rate, 2.2. It costs 29 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::Lu::solve 58 54 +complex linalg::Lu::solve 58 54 instructions linux 293.368 allocated linux 145.121 # 85 copies, for 32n³/3 real flops per n² complex items, half to factor and half to form Q, at a quarter copy per FMA on rustymath's own kernels. It costs 281 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 28 copies: the working copy and the writes, 2, and those flops at Accelerate's rate for complex rank-32 updates, 26. It costs 72.5 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::qr 180 180 +complex linalg::qr 180 180 instructions linux 1217.788 allocated linux 838.578 # 384 copies, for about 48n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. It costs 846 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 130 copies, for Accelerate's GESDD: the 16n³/3 real flops of its bidiagonal reduction that run as matrix-vector products, at the complex matrix-vector rate, 24, the rest of its 48n³ at the rate for complex rank-32 updates, 103, and three copies. It costs 465 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::svd 1100 640 +complex linalg::svd 1100 640 instructions linux 3723.965 allocated linux 1264.416 # 288 copies, for about 36n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. On macOS, 88 copies, for Accelerate's HEEVD: the 8n³/3 real flops of its tridiagonal reduction that run as Hermitian matrix-vector products, at twice the complex matrix-vector rate, 5.9, the rest of its 36n³ at the rate for complex rank-32 updates, 81, and two copies. It costs 502 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::eigh 1100 370 +complex linalg::eigh 1100 370 instructions linux 2081.207 allocated linux 692.151 # 800 copies, for about 100n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. It costs 2568 copies on CI on 22 October 2026, more than twice that. On 23 and 24 October 2026 the same code cost 2303 to 4673 copies on CI's runner, and builds from before and after those dates cost the same on a Mac. Its Linux budget is 50% above the earlier 4800, to cover that variation in the runner, not a code change. macOS runs this solver too, with only the vectors' last product on Accelerate, so the same expected cost applies there. It costs 2151 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::eig 4600 2000 +complex linalg::eig 4600 2000 instructions linux 13044.869 allocated linux 721.296 # 2 copies, for three reads and one write per item. -Clip 4 7.4 +Clip 4 7.4 instructions linux 1.514 allocated linux 0.007 # 2 copies, for three reads and one write per item. -Choose 4 8.4 +Choose 4 8.4 instructions linux 1.263 allocated linux 0.007 [f32] # 4.5 copies, for 18 vector ops: a test for each block of 64 items that every item lies within ±87, then reduction by ln 2, a degree-4 polynomial and one power of two. A block of items below -105 writes zeros. Any other block takes the clamp and two powers of two. -Exp 9 8.7 +Exp 9 8.7 instructions linux 4.038 allocated linux 0.002 # 4.5 copies, for 18 vector ops: exp's test and reduction, a degree-5 polynomial and one power of two, with the guards only in a block that holds an item beyond ±87. -Expm1 9 13 +Expm1 9 13 instructions linux 4.412 allocated linux 0.002 # 5.25 copies, for 21 vector ops: a test for each block of 64 items that every item is positive, normal and finite, then the split into mantissa and exponent, one division, a series in s² and the combine. A block of 64 items that holds a zero, a negative, a subnormal, an infinity or NaN takes the full form with its special values, which costs about 1.7 times as much on macOS. -Ln 11 17 +Ln 11 17 instructions linux 6.632 allocated linux 0.002 # 8.75 copies, for 35 vector ops: ln's steps plus the correction for 1 + x, with the block test on 1 + x and on whether x is 0. A block of 64 items that holds a zero, a negative, a subnormal, an infinity or NaN takes the full form with its special values, which costs about 1.5 times as much on macOS. -Log1p 18 17 +Log1p 18 17 instructions linux 7.881 allocated linux 0.002 # 5.25 copies, for 21 vector ops: ln's steps with a scaled combine. -Log2 11 12 +Log2 11 12 instructions linux 4.211 allocated linux 0.002 # 5.25 copies, for 21 vector ops: ln's steps with a scaled combine. -Log10 11 13 +Log10 11 13 instructions linux 4.398 allocated linux 0.002 # 5 copies, for 21 vector ops: reduction by π/2, a polynomial, the quadrant. -Sin 10 12 +Sin 10 12 instructions linux 5.6 allocated linux 0.002 # 5.5 copies, for 22 vector ops: sin's steps with a shifted quadrant. -Cos 11 15 +Cos 11 15 instructions linux 6.038 allocated linux 0.002 # 6.5 copies, for 26 vector ops: reduction, a polynomial, a division. -Tan 13 18 +Tan 13 18 instructions linux 5.663 allocated linux 0.002 # 6 copies, for 24 vector ops: a square root past 1/2, a polynomial, selects. -Asin 12 13 +Asin 12 13 instructions linux 4.757 allocated linux 0.002 # 6.5 copies, for 26 vector ops: asin's steps and a reflection. -Acos 13 14 +Acos 13 14 instructions linux 5.132 allocated linux 0.002 # 5.5 copies, for 22 vector ops: a reduction by division, a polynomial, selects. -Atan 11 15 +Atan 11 15 instructions linux 4.163 allocated linux 0.002 # 8 copies, for 32 vector ops: expm1, a division, selects. -Sinh 16 19 +Sinh 16 19 instructions linux 5.663 allocated linux 0.002 # 6.5 copies, for 26 vector ops: exp and its reciprocal. -Cosh 13 15 +Cosh 13 15 instructions linux 4.288 allocated linux 0.002 # 5.5 copies, for 22 vector ops: expm1 without its guards, which |x| up to 9 never needs, a division and the sign. -Tanh 11 15 +Tanh 11 15 instructions linux 5.538 allocated linux 0.002 # 9 copies, for 37 vector ops: ln, a square root, selects. -Asinh 18 29 +Asinh 18 29 instructions linux 8.663 allocated linux 0.002 # 9 copies, for 37 vector ops: ln and a square root. -Acosh 18 34 +Acosh 18 34 instructions linux 9.507 allocated linux 0.002 # 8.5 copies, for 34 vector ops: log1p of a quotient. -Atanh 17 24 +Atanh 17 24 instructions linux 8.132 allocated linux 0.002 # 7 copies, for 29 vector ops: split, a polynomial, two Newton steps with divisions. -Cbrt 14 30 +Cbrt 14 30 instructions linux 5.912 allocated linux 0.002 # 30 copies, for about 60 ops computed in f64 at half the lanes: ln and exp, special cases. -Pow 60 57 +Pow 60 57 instructions linux 22.558 allocated linux 0.005 # 60 copies, for Pow's 60 ops in scalar f64, pair by pair, at one f32 copy each. -Pow.one 120 120 +Pow.one 120 120 instructions linux 133.208 allocated linux 0 # 4 copies, for 16 vector ops: scaling, a square root, a division; two reads and a write take 1.5. -Hypot 8 12 +Hypot 8 12 instructions linux 2.806 allocated linux 0.005 # 30 copies, for Pow's steps in f64 with a reciprocal power. -Root 60 95 +Root 60 95 instructions linux 60.809 allocated linux 0.005 # 60 copies, for Root in scalar f64, pair by pair. -Root.one 120 110 +Root.one 120 110 instructions linux 122.002 allocated linux 0 # 1.5 copies, for two reads and one write per item. -Maximum 3 5.1 +Maximum 3 5.1 instructions linux 0.525 allocated linux 0.005 # 0.5 copies, for one read per item. -reduce::sum 1 1.8 +reduce::sum 1 1.8 instructions linux 0.175 allocated linux 0 # 0.5 copies, for one read per item. -reduce::max 1 1.9 +reduce::max 1 1.9 instructions linux 0.3 allocated linux 0 # 1 copies, for one read and one write per item; the scan runs in vector steps. It costs 2.81 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. -reduce::Sum::scan 2 7.1 +reduce::Sum::scan 2 7.1 instructions linux 3.853 allocated linux 0 # 1 copies, for one read and one write per item; the scan runs in vector steps. -reduce::Max::scan 2 3.8 +reduce::Max::scan 2 3.8 instructions linux 1.501 allocated linux 0 # 3.75 copies, for the negate (1), a read and write in place (1), 7 FMAs (1.75). -polynomial::evaluate 5.5 3.3 +polynomial::evaluate 5.5 3.3 instructions linux 2.88 allocated linux 0 # 75 copies, for 150 ops in f64 at half the lanes: a plain Lanczos sum, ln, exp, sinpi. -Gamma 150 150 +Gamma 150 150 instructions linux 48.994 allocated linux 0.01 # 50 copies, for 100 ops in f64 at half the lanes. -Lgamma 100 130 +Lgamma 100 130 instructions linux 46.494 allocated linux 0.01 # 12.5 copies, for 50 vector ops: both of erf's formulas for every item, x + x a(x²) with 6 terms and 1 - e^-(|x| + |x| b(|x|)) with 7 terms and f32's `exp`, then the selects. -Erf 25 39 +Erf 25 39 instructions linux 9.899 allocated linux 0.01 # 52 copies, for 209 vector instructions: the speed test's items, -5 to 10, make every block of 512 mixed, and f32's mixed blocks run both formulas. Their loop retires 209 instructions for each vector of four items (`bpl-tests disasm` of `erfc_blocks::`, 23 October 2026), about twice the 100 operations the formulas hold. Both formulas' coefficients don't fit in the 32 vector registers, so the loop reloads about 35 of them on each pass, and it adds selects and register moves. macOS measured 46.3 to 50.6 in five runs that day, with `erfc_blocks` on `slice::blocks` and on a loop of its own, and CI measured 50.3 on 22 October 2026. -Erfc 104 140 +Erfc 104 140 instructions linux 26.648 allocated linux 0.01 # 17.5 copies, for 70 vector ops. -Erfinv 35 63 +Erfinv 35 63 instructions linux 16.494 allocated linux 0.01 # 18 copies, for 72 vector ops: the same. -Erfcinv 36 63 +Erfcinv 36 63 instructions linux 15.777 allocated linux 0.01 # 420 copies, for 420 scalar f64 ops: incomplete_gamma's narrow route, a plain prefactor and about 25 steps, at one f32 copy each. -Gammainc 840 910 +Gammainc 840 910 instructions linux 310.609 allocated linux 0.312 # 420 copies, for Gammainc's narrow route. -Gammaincc 840 910 +Gammaincc 840 910 instructions linux 310.609 allocated linux 0.312 # 1700 copies, for about four evaluations of Gammainc's narrow route, 1,700 scalar f64 ops. It costs about 3531.6 copies on macOS and 4780 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -Gammaincinv 7400 8100 +Gammaincinv 7400 8100 instructions linux 2727.507 allocated linux 0.312 # 850 copies, for 850 scalar f64 ops for a narrow route like incomplete_gamma: a plain prefactor (400) and about 30 steps. It costs about 2898.6 copies on macOS and 3708 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -Betainc 6000 6100 +Betainc 6000 6100 instructions linux 2358.344 allocated linux 0.469 # 3400 copies, for about four evaluations of Betainc's narrow route. It costs about 8512 copies on macOS and 11278 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -Betaincinv 18000 19000 +Betaincinv 18000 19000 instructions linux 6793.408 allocated linux 0.469 # Times the per-item kernel through `time_map`: `complex::exp` on each item, exp of the real part and sin and cos of the imaginary part. The expected cost of 10 copies, for 78 f64 ops per complex item, was worked out for the split slice form that this row timed before that form was deleted. The per-item kernel's cost isn't studied yet, and both budgets stay held. -complex Exp 47 26 +complex Exp 47 26 instructions linux 38.486 allocated linux 28.01 # 12.5 copies, for 100 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: hypot, ln with care near |z| = 1, atan2. It costs about 30.5 copies on macOS and 27.1 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -complex Ln 59 36 +complex Ln 59 36 instructions linux 60.753 allocated linux 32.01 # 6.4 copies, for 51 instructions per vector of four complex items, at a quarter copy each and two floats per complex copy: about 17 for each block's test of 64 items, and 34 for the plain form, whose squares sum in f64 and whose roots and division need no scaling or special cases. A block holding a nonzero part below 2^-103, a NaN or ∞ part, or a 0 with both parts 0 takes the full form instead, with the scaling of tiny items, the special-value selects and the NaN check. It costs about 6.0 copies on macOS on 24 October 2026, and cost 9.91 on CI on 22 October 2026 through the full form alone. -complex Sqrt 13 13 +complex Sqrt 13 13 instructions linux 12.532 allocated linux 0.01 # 16 copies, for 126 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: sin, cos, sinh and cosh. -complex Sin 32 30 +complex Sin 32 30 instructions linux 28.689 allocated linux 44.01 # 17.5 copies, for 140 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: sin, cos, sinh, cosh and a division. -complex Tan 35 43 +complex Tan 35 43 instructions linux 40.961 allocated linux 52.01 # 22.5 copies, for 180 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex square root of 1 - z², a complex ln, products. It costs 38.6 copies on CI on 22 October 2026, about twice that, where a budget of twice it would fail by chance, so its Linux budget is unchanged until the cost is studied. -complex Asin 45 77 +complex Asin 45 77 instructions linux 80.629 allocated linux 60.01 # 16 copies, for 130 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex ln of a quotient. CI measured 26.2 on 22 October 2026, so its Linux budget follows the expected cost. -complex Atanh 32 46 +complex Atanh 32 46 instructions linux 51.872 allocated linux 48.01 # 104 copies, for f64's 420 ops computed in f64, which cost twice as many f32 copies. It costs 263 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. -complex Lgamma 210 340 +complex Lgamma 210 340 instructions linux 372.247 allocated linux 0.01 # 125 copies, for f64's 500 ops computed in f64, which cost twice as many f32 copies. It costs twice the f64 case's cost: about 254 copies on macOS, against f64's 132, and 353 on CI on 22 October 2026. Its macOS budget is twice the f64 case's held budget until that cost is studied, and its Linux budget is unchanged. -complex Gamma 560 400 +complex Gamma 560 400 instructions linux 524.247 allocated linux 0.01 # 2 copies, for 16 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: hypot; reading two floats and writing one take 0.75. -complex Abs 4 3.9 +complex Abs 4 3.9 instructions linux 3.897 allocated linux 0.01 # Times the per-item kernel through `time_map`: `complex::sqrt1mx2` on each item, two complex square roots, then a product and a rebalance. The expected cost of 11.5 copies, for 92 f64 ops per complex item, was worked out for the split slice form that this row timed before that form was deleted. Both budgets stay unchanged until the per-item kernel's cost is studied. -complex Sqrt1mx2 59 53 +complex Sqrt1mx2 59 53 instructions linux 65.1 allocated linux 52.01 # 23 copies, for 184 f64 ops per complex item, at a quarter copy per op and two floats per complex copy: a complex ln, a complex product, a complex exp. It costs about 52.3 copies on macOS and 51.5 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -complex Pow 110 81 +complex Pow 110 81 instructions linux 136.58 allocated linux 68.02 # 64 copies, for 256 FMAs per result item, a quarter copy each, on rustymath's own kernels. On macOS, 4.4 copies, for the same 512 flops at Accelerate's rate for a 1024×1024 product. -linalg::PlusTimes 8.8 83 +linalg::PlusTimes 8.8 83 instructions linux 74.956 allocated linux 10.062 # 0.5 copies: one read of the matrix, about half a copy, on rustymath's own kernels, where its one FMA per item, a quarter copy, runs alongside. On macOS, 0.4 copies, for 2 flops per item at Accelerate's matrix-vector rate. -linalg::PlusTimes by a vector 0.8 3.2 +linalg::PlusTimes by a vector 0.8 3.2 instructions linux 2.029 allocated linux 0.016 # 0.5 copies, as `linalg::PlusTimes by a vector`. -linalg::PlusTimes of a vector 0.8 1.1 +linalg::PlusTimes of a vector 0.8 1.1 instructions linux 0.681 allocated linux 0.078 # 128 copies, for 256 steps of a max and an add per result item, a quarter copy per op. -linalg::MaxPlus 260 150 +linalg::MaxPlus 260 150 instructions linux 108.905 allocated linux 10 # 128 copies, for 256 steps of a min and an add per result item, a quarter copy per op. -linalg::MinPlus 260 150 +linalg::MinPlus 260 150 instructions linux 108.905 allocated linux 10 # 128 copies, for 256 steps of a comparison and an add per result item, a quarter copy per op. -linalg::PlusEqual 260 140 +linalg::PlusEqual 260 140 instructions linux 107.228 allocated linux 10 # 35 copies of 64-bit items: these values fit f32 products exactly, whose 256 multiply-adds per result item take 64 f32 copies, 32 of 64-bit items, at a quarter copy each on rustymath's own kernels, with passes that find the operands' largest magnitudes, 1, convert both to f32, 1.5, and convert the result back, 0.75. On macOS, 5.6 copies, with Accelerate's f32 product at its rate for a 1024×1024 product, 2.4, and the same passes. -linalg::PlusTimes at i64 12 53 +linalg::PlusTimes at i64 12 53 instructions linux 85.867 allocated linux 26.062 # 128 copies, for 256 steps of an integer comparison and an add per result item. -linalg::PlusEqual at i64 260 45 +linalg::PlusEqual at i64 260 45 instructions linux 61.895 allocated linux 17 # 16 copies of 32-bit items, for the same network on four-lane vectors. Its minimums, maximums and passes over rows cost about as many copies as at f64, about 12, because a row holds twice the items and a copy takes half the time. Its passes over each item, the lane stages' shuffles, the write by columns and the NaN count, take the same time as at f64, about 4 copies of 32-bit items. It costs about 22 copies on macOS, more than this analysis gives, and the difference hasn't been studied. A radix sort of each row's keys cost about 77. -search::view_sort_along 32 44 +search::view_sort_along 32 44 instructions linux 25.097 allocated linux 4.078 # 52 copies, for a radix sort of each 32,768-item lane's keys alone: 4 byte passes, each scattering the keys to 256 places at about 13 copies, as measured on this hardware. A sorting network would gain here only as the base case of a long sort, as "A vectorized sort kernel" in basedpl's `meta/rustymath.md` records. It costs about 52 copies on macOS, down from about 72 when the sort carried each float beside its key. -search::view_sort_along long lanes 110 89 +search::view_sort_along long lanes 110 89 instructions linux 70.689 allocated linux 12 # 30 copies, for a quickselect of each row's keys: about 3 comparisons per item at about 10 copies each, as the sorts measure comparisons of half-width items, plus a pass that computes the keys and one that writes the floats back. A vectorized kernel that partitions would cost less, as "A vectorized sort kernel" in basedpl's `meta/rustymath.md` records. It costs about 25 copies on macOS, down from about 33 when each comparison computed two keys. CI measured 22 on 22 October 2026, so its Linux budget follows the same expected cost. -search::view_partition_along 60 49 +search::view_partition_along 60 49 instructions linux 26.391 allocated linux 4.078 # 30 copies, for selecting the two middle keys of each row, which costs about what one partition does. It costs about 27 copies on macOS, down from about 66 when each comparison computed two keys. CI measured 23.6 on 22 October 2026, so its Linux budget follows the same expected cost. -search::view_quantiles_along 60 56 +search::view_quantiles_along 60 56 instructions linux 29.158 allocated linux 0.094 # 1.5 copies, for two reads per item, for the mean and the squares, and two ops. -stats::variance 3 6.7 +stats::variance 3 6.7 instructions linux 2.216 allocated linux 0.283 # 1.5 copies, for reading the data and the weights once, with a multiply-add per item. It costs about 7.4 copies on macOS, because each chunk gathers the data and the weights into buffers, and a second pass gathers the weights again for their total. Two other methods measured slower on 21 October 2026: folding pairs of products and weights in one walk cost 10.6, because a buffer of pairs folds more slowly than the walk it saves, and reading each 64-item row in place cost 11.7, because each in-place read works out its row's offset with divisions and each row's fold pays its own setup. Reading the runs in place without copying them only ties with the gather at 64 items. A fold that sums each run straight into its output's accumulator, with no product buffer, would reach the expected cost, as step 4 in basedpl's `meta/next-phases.md` records. Both budgets stay held until then. -stats::average 150 16 +stats::average 150 16 instructions linux 6.632 allocated linux 4.541 # 22 copies, for 2n³/3 flops per n² items, as FMAs, plus the solve, at a quarter copy per FMA on rustymath's own kernels. It costs 67.6 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 11 copies: the copy into column order and the row swaps, 2.5, the products' 155 flops an item at Accelerate's rate for rank-32 updates, 5, and the 16-column leaves' 16 flops an item at the matrix-vector rate, 3.3. It costs 31.2 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::Lu::solve 73 92 +linalg::Lu::solve 73 92 instructions linux 72.652 allocated linux 14.453 # 64 copies, for 2n³ flops per n² items, at a quarter copy per FMA on rustymath's own kernels. It costs 149 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 22 copies: the LU as `linalg::Lu::solve` gives it, 11, then the two triangular solves from the permuted identity, which skip no zeros, 512 flops an item at Accelerate's rate for 128×128 products, 11, and writing the inverse, 1. -linalg::Lu::inverse 44 220 +linalg::Lu::inverse 44 220 instructions linux 169.598 allocated linux 33.543 # 11 copies, for n³/3 flops per n² items, at a quarter copy per FMA on rustymath's own kernels. It costs 37 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 4.8 copies: the copy in and the copy closed up, 2, and its n³/3 flops at Accelerate's rate for rank-32 updates, 2.8. It costs 11.6 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::cholesky 35 60 +linalg::cholesky 35 60 instructions linux 37.185 allocated linux 8.949 # 85 copies, for 8n³/3 flops per n² items, 4n³/3 to factor and 4n³/3 to form Q, at a quarter copy per FMA on rustymath's own kernels. It costs 234 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 24 copies: the working copy and the writes of Q and R, 2, and those flops at Accelerate's rate for rank-32 updates, 22. It costs 52.2 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::qr 160 300 +linalg::qr 160 300 instructions linux 252.129 allocated linux 64.812 # 384 copies, for about 12n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 1142 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 161 copies, for Accelerate's GESDD: half the 8n³/3 flops of its bidiagonal reduction, which run as matrix-vector products, at the matrix-vector rate, 70, the rest of its 12n³ flops at the rate for rank-32 updates, 88, and three copies in and out. It costs 633 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::svd 1500 1600 +linalg::svd 1500 1600 instructions linux 1224.097 allocated linux 130.179 # 288 copies, for about 9n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 799 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 88 copies, for Accelerate's SYEVD: half the 4n³/3 flops of its tridiagonal reduction, which run as symmetric matrix-vector products that use each item they read twice, at twice the matrix-vector rate, 17, the rest of its 9n³ flops at the rate for rank-32 updates, 69, and two copies. It costs 435 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eigh 940 1200 +linalg::eigh 940 1200 instructions linux 841.712 allocated linux 75.997 # 800 copies, for about 25n³ flops per n² items, with the vectors, at a quarter copy per FMA on rustymath's own kernels. It costs 3067 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 238 copies, for Accelerate's GEEV: the 2n³/3 flops of its Hessenberg reduction that run as matrix-vector products, at the matrix-vector rate, 35, the rest of its 25n³ flops at the rate for rank-32 updates, 201, and two copies. It costs 2316 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eig 5100 7600 +linalg::eig 5100 7600 instructions linux 6142.335 allocated linux 72.904 # 320 copies, for about 10n³ flops per n² items, at a quarter copy per FMA on rustymath's own kernels. It costs 1673 copies on CI on 22 October 2026, more than twice that. On 23 and 24 October 2026 the same code cost 1401 to 2732 copies on CI's runner, and builds from before and after those dates cost the same on a Mac. Its Linux budget is 50% above the earlier 3400, to cover that variation in the runner, not a code change. On macOS, 113 copies, for GEEV without vectors: the same matrix-vector part, 35, the rest of its 10n³ flops at the rate for rank-32 updates, 77, and one copy. It costs 1356 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -linalg::eigenvalues 3000 2900 +linalg::eigenvalues 3000 2900 instructions linux 2290.579 allocated linux 8.266 # 5 copies, for 2mn² flops on 4096×16 (32 per item) and reading the matrix, at a quarter copy per FMA on rustymath's own kernels. On CI's x86 runner it cost 52.8 copies on 22 October 2026, through CholeskyQR2, which then took this size on rustymath's own kernels. Earlier runs put Householder there about 1.35 times faster, and `tall_for_qr` now gives this size to Householder wherever Accelerate doesn't run. Householder's reflections are matrix-vector passes, far below the FMA rate this analysis assumes: on the Mac's own kernels, f64 costs about 32 copies. Through Householder, CI measured 40.8 later that day. Its Linux budget is unchanged until Householder's expected cost is worked out. On macOS, 17 copies, through CholeskyQR2: its two Gram matrices, 32 flops an item each, at Accelerate's rate for a 4096×16 matrix's transpose times the matrix, 7.4 each, the solve with R, 16 flops an item at about 15 flops in a copy's time, which Accelerate measured for a 4096×16 matrix times a 16×16 one, half its rank-k rate, 1.1, a copy of the matrix, 1, and the reduction of the right side, 0.5. -linalg::lstsq 34 86 +linalg::lstsq 34 86 instructions linux 19.345 allocated linux 8.71 # 21 copies, for 8n³/3 real flops per n² complex items, two floats per complex copy, at a quarter copy per FMA on rustymath's own kernels. It costs 75.3 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 10 copies: the copy into column order and the row swaps, 1.9, the products' 277 real flops an item at Accelerate's rate for complex rank-32 updates, 4.6, and the leaves' 64 at the complex matrix-vector rate, 3.6. It costs 29.2 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::Lu::solve 81 83 +complex linalg::Lu::solve 81 83 instructions linux 204.852 allocated linux 72.623 # 85 copies, for 32n³/3 real flops per n² complex items, half to factor and half to form Q, at a quarter copy per FMA on rustymath's own kernels. It costs 399 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 25 copies: the working copy and the writes, 2, and those flops at Accelerate's rate for complex rank-32 updates, 23. It costs 98.6 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::qr 240 330 +complex linalg::qr 240 330 instructions linux 785.064 allocated linux 419.297 # 384 copies, for about 48n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. It costs 1209 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 132 copies, for Accelerate's GESDD: the 16n³/3 real flops of its bidiagonal reduction that run as matrix-vector products, at the complex matrix-vector rate, 38, the rest of its 48n³ at the rate for complex rank-32 updates, 91, and three copies. It costs 768 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::svd 1600 1100 +complex linalg::svd 1600 1100 instructions linux 2394.024 allocated linux 632.815 # 288 copies, for about 36n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. It costs 707 copies on CI on 22 October 2026, more than twice that, so its Linux budget is unchanged until the cost is studied. On macOS, 83 copies, for Accelerate's HEEVD: the 8n³/3 real flops of its tridiagonal reduction that run as Hermitian matrix-vector products, at twice the complex matrix-vector rate, 9.5, the rest of its 36n³ at the rate for complex rank-32 updates, 71, and two copies. It costs 622 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::eigh 1400 670 +complex linalg::eigh 1400 670 instructions linux 1395.476 allocated linux 346.659 # 800 copies, for about 100n³ real flops per n² complex items, at a quarter copy per FMA on rustymath's own kernels. It costs 2558 copies on CI on 22 October 2026, more than twice that. On 23 and 24 October 2026 the same code cost 2368 to 5532 copies on CI's runner, and builds from before and after those dates cost the same on a Mac. Its Linux budget is 50% above the earlier 5300, to cover that variation in the runner, not a code change. macOS runs this solver too, with only the vectors' last product on Accelerate, so the same expected cost applies there. It costs 2445 copies on macOS on 23 October 2026, more than twice that, so its macOS budget is unchanged until the cost is studied. -complex linalg::eig 5000 3700 +complex linalg::eig 5000 3700 instructions linux 8410.665 allocated linux 360.656 # 1 copies, for reading two items and writing one per lane of 2. It costs about 2.4 copies on macOS and 3.54 on CI on 22 October 2026, more than twice that, so both budgets are unchanged until the cost is studied. -Array::differences short lanes 5.2 8.4 +Array::differences short lanes 5.2 8.4 instructions linux 4.418 allocated linux 2.574 # 1 copies, for one read and one write per item. -Array::differences 2 3.2 +Array::differences 2 3.2 instructions linux 0.683 allocated linux 4.023 # 0.5 copies, for one write per output item; each input is read once for 64 outputs. It costs about 1.1 copies on macOS, more than twice that, so its macOS budget is unchanged until the cost is studied. CI measured 0.8 on 22 October 2026, so its Linux budget follows the expected cost. -Array::broadcasted 2.2 2.1 +Array::broadcasted 2.2 2.1 instructions linux 0.803 allocated linux 4.012 # 2 copies, for three reads and one write per item. -Clip 4 13 +Clip 4 13 instructions linux 0.794 allocated linux 0.007 # 2 copies, for three reads and one write per item. -Choose 4 12 +Choose 4 12 instructions linux 0.669 allocated linux 0.007 [i64] # 0.75 copies, for reading 4 bytes and writing 8 per item, against a copy that moves 16. -Buffer::cast i32 1.5 1.5 +Buffer::cast i32 1.5 1.5 instructions linux 0.721 allocated linux 8.003 diff --git a/tests/speed_python.txt b/tests/speed_python.txt index c7ad2f7..6d702c4 100644 --- a/tests/speed_python.txt +++ b/tests/speed_python.txt @@ -1,79 +1,80 @@ # The budgets that `test_speed.py` checks, for rustymath's Python API. Each section starts with an item type in brackets. Each line under it holds a Python expression and its two native budgets, separated by tabs: first for macOS on Apple silicon, where matrix products and decompositions run on Accelerate, then for Linux on CI's x86 runner, where they run on rustymath's own kernels. `rm.timing().column` picks the budget a machine checks. The expression names the arrays that `test_speed.py` makes. A budget counts copies of 65,536 items of the section's type, the number of items in each case's main input. -# A budget is the case's expected cost times the multiplier that `rustymath::timing::budget` holds, as basedpl's DEV.md describes, and at least 0.5. A line such as `# held linux 14.2: REASON` above a case's line holds that column at the budget for the cost it gives, until the gap it names is fixed, and `# busy` does the same for CI's busy hosts, as `rustymath::timing::hold` reads them. The expected cost starts from a first-principles analysis of what the case should cost. A later expected cost comes from a fuller understanding of the case than that analysis gives, such as a platform difference or a cost the analysis missed. A comment above each case, or above its group, gives the expected cost and its reason, as in "1 copy: one read and one write per item", and records each later change with its reason. +# A budget is the case's expected cost times the multiplier that `rustymath_timing::budget` holds, as basedpl's DEV.md describes, and at least 0.5. A line such as `# held linux 14.2: REASON` above a case's line holds that column at the budget for the cost it gives, until the gap it names is fixed, and `# busy` does the same for CI's busy hosts, as `rustymath_timing::hold` reads them. The expected cost starts from a first-principles analysis of what the case should cost. A later expected cost comes from a fuller understanding of the case than that analysis gives, such as a platform difference or a cost the analysis missed. A comment above each case, or above its group, gives the expected cost and its reason, as in "1 copy: one read and one write per item", and records each later change with its reason. # One expected cost serves both platforms at first. A platform's budget differs only where a specific difference on that platform is found, such as Accelerate's matrix products on macOS. # A case that costs more than its budget needs study, which finds either a speed bug to fix or something the analysis missed. Avoid raising a budget to match a run without understanding why the case costs what it does. +# After its budgets, a case's line can record the instructions that it retires for each item in one more tab-separated field, as `rustymath_timing::count` reads it, such as `instructions linux 41.5`. `test_speed.py` fails a count more than `rustymath_timing::COUNT_MARGIN` times its record. `basedpl.bpltests.set_counts` writes the counts from a calibration run's log. [float64] # 1.6 copies, for two reads and one write per item, plus a tenth of a copy for the Python call. -x+y 3.2 4.7 +x+y 3.2 4.7 instructions linux 1.226 # 1.1 copies, for one read and one write, plus a tenth of a copy for the Python call. -x*2 2.2 2.5 +x*2 2.2 2.5 instructions linux 1.019 # 1.1 copies, for one read and one write, plus a tenth of a copy for the Python call. --x 2.2 2 +-x 2.2 2 instructions linux 0.941 # 7.1 copies, for Exp's 7 copies, plus a tenth of a copy for the Python call. -rm.exp(x) 15 9.7 +rm.exp(x) 15 9.7 instructions linux 10.44 # 1.9 copies, for reading the Boolean mask, x and y, a select, one write, plus a tenth of a copy for the Python call. -rm.where(mask, x, y) 3.8 5.6 +rm.where(mask, x, y) 3.8 5.6 instructions linux 2.173 # 1.6 copies, for one read, two comparisons, one write, plus a tenth of a copy for the Python call. -rm.clip(x, -0.5, 0.5) 3.2 2 +rm.clip(x, -0.5, 0.5) 3.2 2 instructions linux 1.304 # 1.9 copies, for reading m and a, a select and one write, with a 256-item mask that broadcasts over m's rows and stays in cache, plus a tenth of a copy for the Python call. The broadcast mask takes the general ternary route through `read_runs`, which reads each 256-item repeat of the mask as one item. -rm.where(row > 0, m, a) 3.8 4.3 +rm.where(row > 0, m, a) 3.8 4.3 instructions linux 2.728 # 1.6 copies, for one read of m, two comparisons and one write, with bounds from a 256-item row that broadcasts over m's rows and stays in cache, plus a tenth of a copy for the Python call. The broadcast bounds take the general ternary route through `read_runs`, which reads each 256-item repeat of a bound as one item. -rm.clip(m, -row, row) 3.2 3.8 +rm.clip(m, -row, row) 3.2 3.8 instructions linux 2.147 # 0.6 copies, for one read per item, plus a tenth of a copy for the Python call. -rm.sum(x) 1.2 0.67 +rm.sum(x) 1.2 0.67 instructions linux 0.542 # 1.1 copies, for one read and one write, plus a tenth of a copy for the Python call. -rm.cumsum(x) 2.2 3 +rm.cumsum(x) 2.2 3 instructions linux 4.136 # 1.1 copies, for reading m and writing the result; the row stays in cache, plus a tenth of a copy for the Python call. -m+row 2.2 2.2 +m+row 2.2 2.2 instructions linux 1.373 # 2.6 copies, for a blocked transposed read (about 1), a read and a write, plus a tenth of a copy for the Python call. -m.T+m 5.2 5.8 +m.T+m 5.2 5.8 instructions linux 4.539 # 0.6 copies, for one read per item, plus a tenth of a copy for the Python call. -rm.sum(m.T, axis=0) 1.2 1.3 +rm.sum(m.T, axis=0) 1.2 1.3 instructions linux 1.256 # 0.6 copies, for one read per item, plus a tenth of a copy for the Python call. -rm.max(m.T, axis=1) 1.2 1.1 +rm.max(m.T, axis=1) 1.2 1.1 instructions linux 1.399 # 0.6 copies, for one reversed read per item, plus a tenth of a copy for the Python call. -rm.sum(x[::-1]) 1.2 0.93 +rm.sum(x[::-1]) 1.2 0.93 instructions linux 0.817 # 0.8 copies, for one read and two ops per item, plus a tenth of a copy for the Python call. -rm.minmax(m.T) 1.6 1.4 +rm.minmax(m.T) 1.6 1.4 instructions linux 1.754 # 1.6 copies, for the kernel's read and write (1), a gather of each 4,096-item chunk of the reversed view into a cached buffer (0.4), and a tenth of a copy for each of two Python calls, one of which makes the view. Raised on 21 October 2026 from 1.1 copies, which counted the kernel reading through the view in one pass. The route gathers first because kernels read only forward contiguous slices. At this size, a pass over a cached buffer costs about as much as any other pass. A kernel run over the block in storage order, with the result taking the view's mapping, would skip the gather at about 1.15 copies. Nobody has built it. -x[::-1]+1 3.2 3.9 +x[::-1]+1 3.2 3.9 instructions linux 2.729 # 2.4 copies, for two strided gathers into cached buffers, each reading every cache line of its argument and writing half its items (0.75 each), the kernel's two reads and one write on half the items (0.7), and a tenth of a copy for each of three Python calls. Raised on 21 October 2026 from 1.4 copies, which counted the kernel reading through the views in one pass. The route gathers first because kernels read only forward contiguous slices. At this size, a pass over a cached buffer costs about as much as any other pass. A storage-order kernel would still need these gathers, because half of each view's storage holds items outside it. -x[::2]+y[::2] 4.8 6.5 +x[::2]+y[::2] 4.8 6.5 instructions linux 5.242 # 64 copies on Linux, for 256 FMAs per result item at about 4 FMAs in a copy's time on rustymath's own kernels. On macOS, 8.8 copies, at Accelerate's rate for large square products in the header of `tests/speed.txt`, as `linalg::PlusTimes` expects there, and 9.8 for `a[::-1]@b`, which copies its reversed rows first. On macOS they measured 10 to 13 copies on 23 October 2026. -a@b 18 80 -a.T@b 18 83 -a[::-1]@b 20 82 +a@b 18 80 instructions linux 147.313 +a.T@b 18 83 instructions linux 149.282 +a[::-1]@b 20 82 instructions linux 148.853 # 3 copies, for a random gather from L2, with about 16 loads in flight (2), and reading the 8-byte positions (0.5), plus a tenth of a copy for the Python call. The analysis misses the pass that checks the positions before the gather, about 0.5 copies. It measured 5.3 to 5.9 copies on macOS on 23 October 2026, since Python reads index arrays in place as basedpl's `⌷` does, down from 6.1 to 6.3 when it copied them into a list of positions first. On the same 65,536 positions, basedpl's `[p]⌷v` measured 4.7 copies under dynamic kinds and 6.3 under static kinds. Nobody has found the cause of that difference. -x[idx] 6 8.9 +x[idx] 6 8.9 instructions linux 1.668 # 4 copies, for reading both position arrays (1), forming each offset, a random gather from L2 (2), a write, plus a tenth of a copy for the Python call. It measured about 16 copies on macOS and 14.2 on CI on 23 October 2026. # held macos 16: the cause isn't known yet -m[r, c] 8 29 +m[r, c] 8 29 instructions linux 37.683 # 4 copies, for reading idx and y (1), a random read and write of x in L2 (3), plus a tenth of a copy for the Python call. It measured 8.5 to 9.7 copies on macOS on 23 October 2026, down from about 10.6 before writes stopped checking each position that selection had already checked, and 7.86 on 30 October 2026. CI measured 7.55 on 23 October 2026. # held macos 9.7: the cause isn't known yet -rm.at(rm.add, x, idx, y) 8 18 +rm.at(rm.add, x, idx, y) 8 18 instructions linux 17.284 # 4 copies, as `rm.at(rm.add, x, idx, y)`, with a maximum's step in place of a sum's: a place written more than once folds its values through the maximum's reduction (`Combine`). It measured 7.89 copies on macOS on 30 October 2026, and CI measured 7.47 on 23 October 2026. # held macos 7.89: as for `rm.at(rm.add, x, idx, y)`, the cause isn't known yet -rm.at(rm.maximum, x, idx, y) 8 18 +rm.at(rm.maximum, x, idx, y) 8 18 instructions linux 17.265 # 56 copies, for an unstable comparison sort of 65,536 float keys, computed once: about 16 levels per item at about 3.5 copies a level, the standard library sort's cost on random keys, which falls with lane length from about 5 at 256 items, plus a pass that computes the keys, one that writes the floats back, and a tenth of a copy for the Python call. A radix sort of the keys costs more below 2^17 items. A sorting network would gain here only as the base case of a long sort, as "A vectorized sort kernel" in basedpl's `meta/rustymath.md` records. It costs about 54 copies on macOS, down from about 87 when each comparison computed two keys. -rm.sort(x) 120 91 +rm.sort(x) 120 91 instructions linux 149.869 # 3.3 copies, for 9 FMAs per output item (2.25), one read and one write, plus a tenth of a copy for the Python call. On macOS, 5.6 copies, because NEON's two lanes of f64 take twice the FMAs' time (4.5). It measured 6.0 to 6.2 copies there on 23 October 2026, and 6.6 to 7.6 after rustymath's `COLUMN_STRIP` widened `by_columns`' strips to 32 results on 27 October 2026, which left 30 results in each 254-wide output row to one-at-a-time code. Strips of 8 for those results, with each result written rather than added to the zeroed output, brought it to 5.1 to 5.8 on 28 October 2026. CI measured 5.31 on 23 October 2026, and its Linux budget follows its expected cost. -rm.correlate(img, k) 12 7.8 +rm.correlate(img, k) 12 7.8 instructions linux 11.475 # 5 copies, for 16 FMAs per output item (4), one read and one write, plus a tenth of a copy for the Python call. It measured 9.2 to 9.3 copies on macOS on 23 October 2026, and 7.02 on 30 October 2026. CI measured 8.24 on 23 October 2026. -rm.convolve(x, kern) 10 9 +rm.convolve(x, kern) 10 9 instructions linux 11.874 # 1.1 copies, for reading each frame item once and adding it into the output, plus a tenth of a copy for the Python call. -rm.overlap_add(frames, 32) 2.2 1.7 +rm.overlap_add(frames, 32) 2.2 1.7 instructions linux 2.354 # 1.1 copies, for one copy of the NumPy array's items, plus a tenth of a copy for the Python call. -rm.array(xn) 2.2 1.8 +rm.array(xn) 2.2 1.8 instructions linux 0.143 # 0.1 copies, for the call alone, since the buffer protocol lends the items without copying. -np.asarray(x) 0.5 0.5 +np.asarray(x) 0.5 0.5 instructions linux 0.29 [float32] # 1.7 copies, for two reads and one write per item, plus the call, which costs twice as much against a 32-bit copy. -xf+yf 3.4 3.2 +xf+yf 3.4 3.2 instructions linux 0.735 # 5.7 copies, for Exp's 5.5 copies at f32, plus the call. -rm.exp(xf) 12 6.5 +rm.exp(xf) 12 6.5 instructions linux 4.193 # 2.1 copies, for reading the mask, xf and yf, a select, one write,, plus the call. -rm.where(mask, xf, yf) 4.2 5.5 +rm.where(mask, xf, yf) 4.2 5.5 instructions linux 1.116 # 0.7 copies, for one read per item, plus the call. -rm.sum(xf) 1.4 1.2 +rm.sum(xf) 1.4 1.2 instructions linux 0.389 # 1 copy, for one read per item, its conversion to f64 and a fused multiply-add into an f64 running sum, in lanes that hold half as many items as f32 lanes, plus the call. The squares are f64 so that the square of `1e20` doesn't overflow. Before 22 October 2026, the squares were f32 and went through a buffer, at about 1.2 copies. -rm.norm(xf) 2 1.6 +rm.norm(xf) 2 1.6 instructions linux 0.736 diff --git a/tests/test_speed.py b/tests/test_speed.py index e9c1b65..37725c5 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -1,9 +1,11 @@ -"How fast rustymath's Python API runs. Each case in `speed_python.txt` is timed as `speed.rs` times the Rust slice forms, and checked against its budget." +"How fast rustymath's Python API runs. Each case in `speed_python.txt` is timed as `speed.rs` times the Rust slice forms, and checked against its budget and against any instruction count that the file records for this machine's column. `RUSTYMATH_COSTS` asks for a calibration run, as `speed.rs` describes." import gc, os, sys, time +from contextlib import contextmanager from pathlib import Path -import numpy as np, pytest, rustymath as rm +import numpy as np, rustymath as rm N, RUNS = 65_536, 5 +CALIBRATING = 'RUSTYMATH_COSTS' in os.environ def arrays(): "The arrays that the cases name. Each case's main input holds `N` items." @@ -25,53 +27,67 @@ def fastest(run): del result return best -def cost(code, namespace, copy, itemsize, timing): - "The time that `code` takes for each of `N` items, in copies that take `copy` seconds an item and in seconds, and its footprint, as `timing.footprint` gives it for `N` items of `itemsize` bytes and the most bytes `code` held at once." - case = compile(code, code, 'eval') - best, held = timing.peak_held(lambda: fastest(lambda: eval(case, namespace))) - return best / N / copy, best / N, timing.footprint(itemsize, N, held) - -@pytest.fixture -def timing(): - "Hold the timing lock exclusively through `rm.timing`, as `speed.rs` does, with the garbage collector off. Gives the hold, whose `pauses`, `warm`, `copy`, `peak_held`, `footprint` and `line` the test uses. As a fixture, its wait for the lock falls outside the test's timeout." - with rm.timing() as hold: +@contextmanager +def held(hold): + "Keep `hold`, a hold on the timing lock from `rm.timing` or `rm.try_timing`, with the garbage collector off." + with hold: gc.disable() try: yield hold finally: gc.enable() -def over_budget(cases, namespace, timing, show): - "Time `cases`, as `timing.budgets` gives them, in copies of the time that `timing.copy` gives for their section's item type, then time each one over its budget again after each of `timing.pauses`, in seconds, and `timing.warm`, printing the earlier and new costs. Prints each case whose gap's hold has outlived the gap, as `Budget.stale` decides from its first cost, marked `stale`. Gives the lines of the cases still over budget. With `show`, prints each item type's copy line and every case's line, in the formats `basedpl.bpltests.logged_costs` reads." - units = {s: timing.copy(np.dtype(s).itemsize, N) for s in dict.fromkeys(s for s, _, _ in cases)} - if show: - for _, line in units.values(): print(line, file=sys.stderr) - def attempt(case): - section, code, budget = case - copies, seconds, footprint = cost(code, namespace, units[section][0], np.dtype(section).itemsize, timing) - return timing.line(section, code, copies, '-' if budget is None else str(budget), seconds, footprint), budget is not None and budget.over(copies), budget is not None and budget.stale(copies) - over, stale = [], [] - for case in cases: - line, failed, outlived = attempt(case) - if show: print(line, file=sys.stderr) - if failed: over.append((case, line)) - if outlived: stale.append(line) - for pause in timing.pauses: - if not over: break - for _, line in over: print(f'{line}: over, timed again after {pause:g} s', file=sys.stderr) - time.sleep(pause) - timing.warm() - again = [] - for case, _ in over: - line, failed, _ = attempt(case) - print(line, file=sys.stderr) - if failed: again.append((case, line)) - over = again - for line in stale: print(f'{line}: stale', file=sys.stderr) - return [line for _, line in over] +def attempt(case, namespace, units, timing): + "The line of `case`, a `(section, code, budget, counts)` from `timing.budgets`, its cost in copies of the copy time that `units` gives for its section's item type, and the instructions that one run retires for each of `N` items, or `None` where they can't be counted. The counted run comes before the timed runs." + section, code, budget, _ = case + expr = compile(code, code, 'eval') + _, instructions = timing.instructions(lambda: eval(expr, namespace)) + best, peak = timing.peak_held(lambda: fastest(lambda: eval(expr, namespace))) + copies, per_item = best / N / units[section][0], None if instructions is None else instructions / N + footprint = timing.footprint(np.dtype(section).itemsize, N, peak) + return timing.line(section, code, copies, '-' if budget is None else str(budget), best / N, footprint, (per_item, None)), copies, per_item + +def retimed(over, namespace, units, timing): + "Time each of `over`'s `(case, line)` pairs again, printing its new line, and give those still over budget, with their new lines." + again = [] + for case, _ in over: + line, copies, _ = attempt(case, namespace, units, timing) + print(line, file=sys.stderr) + if case[2].over(copies): again.append((case, line)) + return again -def test_speed(capsys, timing): - "Every case costs no more than its budget, on the CPU that the budgets were set for (`timing.checks_budgets`). On any other CPU, it prints every cost, then each case still over its budget, marked `over`, and fails on none. Each case whose gap's hold has outlived the gap is printed, marked `stale`. `RUSTYMATH_COSTS` prints every cost." +def test_speed(capsys): + "Every case costs no more than its budget, and retires no more instructions than its recorded count allows, on the CPU that the budgets were set for (`timing.checks_budgets`). On any other CPU, it prints every cost, then each case still over, marked `over`, and fails on none. An ordinary run holds the timing lock shared, on one thread, beside builds and other tests. It then times each case over its budget again if it can hold the lock exclusively at once, and otherwise marks the case `unconfirmed` and doesn't fail it. A calibration run, which `RUSTYMATH_COSTS` asks for, holds the lock exclusively, prints every cost and times each case over its budget again after each of `timing.pauses`. Each case whose gap's hold has outlived the gap is printed, marked `stale`, and each whose instructions fall below its record divided by the margin is marked `fewer instructions`." + timing = rm.timing(shared=not CALIBRATING) namespace, cases = arrays(), timing.budgets(Path(__file__).with_name('speed_python.txt').read_text()) + show, failures = not timing.checks_budgets or CALIBRATING, [] with capsys.disabled(): - over = over_budget(cases, namespace, timing, not timing.checks_budgets or 'RUSTYMATH_COSTS' in os.environ) - if timing.checks_budgets: assert not over, '\n'.join(over) - for line in over: print(f'{line}: over', file=sys.stderr) + with held(timing): + units = {s: timing.copy(np.dtype(s).itemsize, N) for s in dict.fromkeys(case[0] for case in cases)} + if show: + for _, line in units.values(): print(line, file=sys.stderr) + measured = [(case, *attempt(case, namespace, units, timing)) for case in cases] + for (_, _, budget, counts), line, copies, instructions in measured: + if show: print(line, file=sys.stderr) + if budget is not None and budget.stale(copies): print(f'{line}: stale', file=sys.stderr) + record = counts[timing.measures.index('instructions')] + if record is None or instructions is None: continue + if record.over(instructions, timing.count_margin): failures.append(f'{line}: instructions over the {record.value} recorded') + elif instructions * timing.count_margin < record.value: print(f'{line}: fewer instructions than the {record.value} recorded', file=sys.stderr) + over = [(case, line) for case, line, copies, _ in measured if case[2] is not None and case[2].over(copies)] + for pause in timing.pauses if CALIBRATING else (): + if not over: break + for _, line in over: print(f'{line}: over, timed again after {pause:g} s', file=sys.stderr) + time.sleep(pause) + timing.warm() + over = retimed(over, namespace, units, timing) + if over and not CALIBRATING: + if (quiet := rm.try_timing()) is None: + for _, line in over: print(f'{line}: unconfirmed, because builds or tests hold the timing lock', file=sys.stderr) + over = [] + else: + with held(quiet): + timing.warm() + for _, line in over: print(f'{line}: over, timed again with the timing lock held exclusively', file=sys.stderr) + over = retimed(over, namespace, units, timing) + failures += [line for _, line in over] + if timing.checks_budgets: assert not failures, '\n'.join(failures) + for line in failures: print(f'{line}: over', file=sys.stderr) diff --git a/timing/Cargo.toml b/timing/Cargo.toml new file mode 100644 index 0000000..3d914b9 --- /dev/null +++ b/timing/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rustymath-timing" +version.workspace = true +edition.workspace = true +rust-version = "1.98" +license = "Apache-2.0" +description = "What rustymath's and basedpl's speed tests share: the machine's timing lock, the timing loop, instruction and byte counts, copy times and budget files." +repository = "https://github.com/AnswerDotAI/rustymath" + +[lib] +test = false +doctest = false + +[dependencies] +rustymath-core.workspace = true + +[target.'cfg(unix)'.dependencies] +libc.workspace = true diff --git a/src/timing.rs b/timing/src/lib.rs similarity index 63% rename from src/timing.rs rename to timing/src/lib.rs index 0911ffa..097f732 100644 --- a/src/timing.rs +++ b/timing/src/lib.rs @@ -1,25 +1,159 @@ -//! What speed tests share: the machine's timing lock, the timing loop, the copy that costs count in with the copy times stored for each machine that runs them, the copy of a case's footprint that its line shows, the format of cost lines, the format of budget files, the CPU that budgets were set for, and the retries. rustymath's and basedpl's speed tests use all of it, and basedpl's browser build uses all but the lock. +//! What speed tests share: the machine's timing lock, the timing loop, the instructions that a call retires, the copy that costs count in with the copy times stored for each machine that runs them, the copy of a case's footprint that its line shows, the format of cost lines, the format of budget files with their budgets and counts, the CPU that budgets were set for, the margin of ordinary runs, and the retries. rustymath's and basedpl's speed tests use all of it, and basedpl's browser build uses all but the lock and the counter. //! -//! The lock keeps timings apart from builds and test runs on one machine. A timing takes it exclusively. A build or a test run takes it shared. It is an `flock` on the file `LOCK`. A timing holds a second file, `GATE`, exclusively from before it takes `LOCK` until it releases it, and a shared hold passes `GATE` before it takes `LOCK`. A waiting timing then stops new builds and test runs from starting, and it runs once the holds already taken end. `fastws.timelock` takes the same locks from Python. The browser build has no timing lock. +//! The lock keeps timings apart from builds and test runs on one machine. A timing takes it exclusively. A build or a test run takes it shared. It is an `flock` on the file `LOCK`. A timing holds a second file, `GATE`, exclusively from before it takes `LOCK` until it releases it, and a shared hold passes `GATE` before it takes `LOCK`. A waiting timing then stops new builds and test runs from starting, and it runs once the holds already taken end. A speed test's ordinary run times beside builds and test runs under a shared hold on one thread. It takes the lock exclusively only to confirm a case over its budget, and only when no one else holds it. `fastws.timelock` takes the same locks from Python. The browser build has no timing lock. #[cfg(not(target_arch = "wasm32"))] -pub use lock::{GATE, Hold, LOCK, exclusive, hold_at, shared}; -/// The clock that timings and `parallel`'s rounds read: the browser's in a browser build, where `std::time::Instant` panics. -#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] -pub(crate) use std::time::Instant; -use std::{ops::Neg, time::Duration}; -#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] -pub(crate) use web_time::Instant; - -/// How many calls `fastest` times. +pub use lock::{GATE, Hold, LOCK, exclusive, hold_at, shared, try_exclusive}; +use rustymath_core::parallel::Instant; +use std::{alloc::{GlobalAlloc, Layout}, cell::Cell, ops::Neg, time::Duration}; + +/// How many calls `fastest` makes, unless they reach `ENOUGH` sooner or its caller asks them to take longer. pub const RUNS: usize = 5; -/// The time of the fastest of `RUNS` calls of `run`, in seconds. Call `k` is passed `k`, so it can read its own inputs. -pub fn fastest(mut run: impl FnMut(usize)) -> f64 { - (0..RUNS).map(|k| { +/// How long `fastest`'s calls take together before it stops after its second call: 20 ms. A case that runs that long varies little from one call to the next, and each further call would add its time to every test run. +const ENOUGH: Duration = Duration::from_millis(20); + +/// The time of the fastest of `run`'s calls, in seconds. It stops after the second call, or any later one, once the calls have taken `ENOUGH` since the first began. Otherwise it makes `RUNS` calls, and goes on calling until the calls have taken `least`: a short case's fastest call then comes from many calls, and varies less between test runs. Call `k` is passed `k`, so it can read its own inputs. A caller whose inputs hold one set for each of `RUNS` calls passes `Duration::ZERO`. +pub fn fastest(least: Duration, mut run: impl FnMut(usize)) -> f64 { + let (begun, mut best) = (Instant::now(), Duration::MAX); + for k in 0.. { let start = Instant::now(); run(k); - start.elapsed() - }).min().expect("RUNS is positive").as_secs_f64() + best = best.min(start.elapsed()); + let spent = begun.elapsed(); + if k > 0 && spent >= ENOUGH || k + 1 >= RUNS && spent >= least { break; } + } + best.as_secs_f64() +} + +/// What `f` gives, and the instructions retired in user space while it ran. On Linux a `perf_event_open` counter counts the calling thread, without any threads that `f` starts. On macOS `proc_pid_rusage` counts the whole process. The speed tests run each case on one thread, where the two agree. `None` where the platform has no counter or refuses one, as Linux does where `/proc/sys/kernel/perf_event_paranoid` is above 2, and in the browser build. +pub fn instructions(f: impl FnOnce() -> R) -> (R, Option) { + let counter = counter::Counter::start(); + let result = f(); + (result, counter.and_then(counter::Counter::read)) +} + +#[cfg(target_os = "linux")] +mod counter { + /// The first 64 bytes of Linux's `perf_event_attr`, its original size, which every kernel with `perf_event_open` accepts. + #[repr(C)] + #[allow(dead_code, reason = "the kernel reads the fields")] + struct Attr { kind: u32, size: u32, config: u64, period: u64, sample_type: u64, read_format: u64, flags: u64, wakeup: u32, bp_type: u32, config1: u64 } + + /// An open counter of the instructions that the calling thread retires in user space, counting from when it opened. Dropping it closes it. + pub struct Counter(libc::c_int); + + impl Counter { + pub fn start() -> Option { + // Type 0 and config 1 are PERF_TYPE_HARDWARE and PERF_COUNT_HW_INSTRUCTIONS. Flag bits 5 and 6 leave out the kernel and the hypervisor, and a clear bit 0 starts the count at once. + let attr = Attr { kind: 0, size: size_of::() as u32, config: 1, period: 0, sample_type: 0, read_format: 0, flags: 1 << 5 | 1 << 6, wakeup: 0, + bp_type: 0, config1: 0 }; + // SAFETY: `attr` is a `perf_event_attr` of the size it gives. Pid 0 and cpu -1 count the calling thread on any CPU, group -1 starts no group, and 8 is PERF_FLAG_FD_CLOEXEC. + let fd = unsafe { + libc::syscall(libc::SYS_perf_event_open, &raw const attr, 0 as libc::c_long, -1 as libc::c_long, -1 as libc::c_long, 8 as libc::c_ulong) + }; + (fd >= 0).then(|| Self(fd as libc::c_int)) + } + + pub fn read(self) -> Option { + let mut count = 0u64; + // SAFETY: reads at most 8 bytes, into `count`, from the counter's open descriptor. + let read = unsafe { libc::read(self.0, (&raw mut count).cast(), size_of::()) }; + (read == size_of::() as isize).then_some(count) + } + } + + impl Drop for Counter { + // SAFETY: closes the descriptor that `start` opened, which nothing else holds. + fn drop(&mut self) { + unsafe { libc::close(self.0) }; + } + } +} + +#[cfg(target_os = "macos")] +mod counter { + /// The instructions that the process had retired when the counter started. + pub struct Counter(u64); + + /// The instructions that the process has retired, as `proc_pid_rusage` gives them. + fn retired() -> Option { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: for `RUSAGE_INFO_V4`, `proc_pid_rusage` fills a `rusage_info_v4` for the given process, here this one. + let filled = unsafe { libc::proc_pid_rusage(libc::getpid(), libc::RUSAGE_INFO_V4, info.as_mut_ptr().cast()) } == 0; + // SAFETY: `proc_pid_rusage` filled `info`, which started zeroed. + filled.then(|| unsafe { info.assume_init() }.ri_instructions) + } + + impl Counter { + pub fn start() -> Option { retired().map(Self) } + pub fn read(self) -> Option { Some(retired()? - self.0) } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +mod counter { + /// No counter: this platform has none that `instructions` reads. + pub enum Counter {} + + impl Counter { + pub fn start() -> Option { None } + pub fn read(self) -> Option { match self {} } + } +} + +thread_local! { + // The bytes that [`Counting`] has seen the calling thread allocate in total, hold now, and hold at most since [`bytes`] last started. A thread can free memory that another thread allocated, so the bytes it holds can fall below zero. + static ALLOCATED: Cell = const { Cell::new(0) }; + static LIVE: Cell = const { Cell::new(0) }; + static PEAK: Cell = const { Cell::new(0) }; +} + +/// Counts an allocation of `size` bytes on the calling thread that replaces `freed` bytes. +fn allocate(size: usize, freed: usize) { + ALLOCATED.set(ALLOCATED.get() + size); + LIVE.set(LIVE.get() + size as isize - freed as isize); + PEAK.set(PEAK.get().max(LIVE.get())); +} + +/// A global allocator that allocates through `A` and counts, for each thread, the bytes that the thread allocates, holds at most and keeps, which [`bytes`] reads. A speed test registers it over the memory guard: `#[global_allocator] static ALLOCATOR: Counting = Counting(memory::Guard::new());`. A reallocation counts as allocating its new size. +pub struct Counting(pub A); + +// SAFETY: each method passes its caller's contract through to `A` unchanged, and only adds to the calling thread's counts. +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { self.0.alloc(layout) }; + if !p.is_null() { allocate(layout.size(), 0); } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { self.0.alloc_zeroed(layout) }; + if !p.is_null() { allocate(layout.size(), 0); } + p + } + + unsafe fn dealloc(&self, p: *mut u8, layout: Layout) { + unsafe { self.0.dealloc(p, layout) }; + LIVE.set(LIVE.get() - layout.size() as isize); + } + + unsafe fn realloc(&self, p: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let moved = unsafe { self.0.realloc(p, layout, new_size) }; + if !moved.is_null() { allocate(new_size, layout.size()); } + moved + } +} + +/// The bytes that a call allocated on the calling thread, as [`Counting`] counts them: in total, the most that it held at once beyond what the thread held when it started, and what it still held when it ended. All are 0 in a binary that doesn't register `Counting`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Bytes { pub allocated: usize, pub peak: usize, pub kept: usize } + +/// What `f` gives, and the [`Bytes`] that it allocated on the calling thread. A call of `bytes` inside `f` restarts the outer call's peak. +pub fn bytes(f: impl FnOnce() -> R) -> (R, Bytes) { + let (allocated, live) = (ALLOCATED.get(), LIVE.get()); + PEAK.set(live); + let result = f(); + (result, Bytes { allocated: ALLOCATED.get() - allocated, peak: (PEAK.get() - live) as usize, kept: (LIVE.get() - live).max(0) as usize }) } /// One copy, the unit that speed budgets count in: each item of `data`, negated, written into `out`. The loop reads and writes each item once. It calls no `memcpy` and allocates nothing, because their speed differs between platforms far more than a loop's does. @@ -29,7 +163,7 @@ pub fn copy>(data: &[T], out: &mut [T]) { for (o, &v) #[derive(Clone, Copy, Debug)] pub struct CopyTime { pub seconds: f64, pub items: usize, pub bytes: usize, pub stored: bool } -/// The most bytes that a copy's source and destination take together for [`copy_time`] to use a stored time. Below it, the copy's data stays in cache, and its time is a property of the CPU. Above it, the copy runs at the speed of memory, which a shared runner varies between runs, and cases of the same size vary with it. On CI's runner under V8, the 36-million-item copy and the cases at that size both varied up to 2.1 times between runs, and dividing by the copy measured in the same run cut the cases' spread from 2.24 times to 1.32. The speed tests' copies take at most 16 MB below it, or 576 MB above it. +/// The most bytes that a copy's source and destination take together for [`copy_time`] to use a stored time. Below it, the copy's data stays in cache, and its time is a property of the CPU. Above it, the copy runs at the speed of memory, which a shared runner varies between runs, and cases of the same size vary with it. On CI's runner under V8, the 36-million-item copy and the cases at that size both varied up to 2.1 times between runs, and dividing by the copy measured in the same run cut the cases' spread from 2.24 times to 1.32. The speed tests' copies take at most 16 MB below it, or 144 MB above it. const CACHED_BYTES: usize = 64 << 20; /// Whether a copy of `items` items of `bytes` bytes each, with its destination, fits in `CACHED_BYTES`. @@ -141,7 +275,7 @@ fn measured_copy>(items: usize) -> f64 { let (data, out) = (&mut data[..items], &mut out[..items]); copy(data, out); copy(out, data); - fastest(|_| { + fastest(Duration::ZERO, |_| { copy(std::hint::black_box(&*data), out); std::hint::black_box(&mut *out); }) / items.max(1) as f64 @@ -164,22 +298,63 @@ pub fn footprint>(items: usize, held: usize) pub fn copy_line(t: &CopyTime) -> String { let source = match (t.stored, cpu()) { (true, Some(c)) => format!("stored for {c}"), - (false, Some(c)) if cached(t.items, t.bytes) => format!("measured on {c}: add it to rustymath::timing::COPIES"), + (false, Some(c)) if cached(t.items, t.bytes) => format!("measured on {c}: add it to rustymath_timing::COPIES"), (false, Some(c)) => format!("measured on {c}"), (_, None) => "measured on a CPU with no name".into() }; format!("copy of {} {}-byte items: {:.3} ns an item, {source}", t.items, t.bytes, t.seconds * 1e9) } -/// A case's line in a speed test's log: its section, code, cost, budget and time for each item, then, where it has one, its [`Footprint`]. `basedpl.bpltests.logged_costs` reads these lines. -pub fn line(section: &str, code: &str, cost: f64, budget: &str, seconds: f64, footprint: Option) -> String { +/// The counts for each item that a case has, given in the order of [`MEASURES`], as a speed test's log gives them: `instructions 33.125, allocated 8.000`, or empty for none. [`line`] ends with them, and basedpl's count-only lines give them after the case's label. +pub fn counted(counts: [Option; 2]) -> String { + MEASURES.iter().zip(counts).filter_map(|(measure, c)| c.map(|c| format!("{measure} {c:.3}"))).collect::>().join(", ") +} + +/// A case's line in a speed test's log: its section, code, cost, budget and time for each item, then, where it has one, its [`Footprint`], then its [`counted`] counts. `basedpl.bpltests.logged_costs` reads these lines. +pub fn line(section: &str, code: &str, cost: f64, budget: &str, seconds: f64, footprint: Option, counts: [Option; 2]) -> String { let mut line = format!("{section} {code}: {cost:.2}, budget {budget}, {:.3} ns", seconds * 1e9); if let Some(f) = footprint { line += &format!(", footprint {:.1} MiB copied at {:.3} ns", f.bytes as f64 / (1 << 20) as f64, f.seconds * 1e9); } + let counts = counted(counts); + if !counts.is_empty() { + line += ", "; + line += &counts; + } line } /// The budget columns, in the order that basedpl's speed-file footers and both projects' budget files give them: native macOS on Apple silicon, native Linux on CI's x86 runner, and basedpl's browser build on CI's runner, which only basedpl's files have. pub const COLUMNS: [&str; 3] = ["macos", "linux", "wasm"]; +/// What a budget file can count for each item of a case, beside its time: the instructions that the case retires, as [`instructions`] counts them, and the bytes that it allocates, as [`bytes`] counts them. +pub const MEASURES: [&str; 2] = ["instructions", "allocated"]; + +/// How far a speed test usually lets a count rise above the count that its budget file records: 1.25 times. A case that counts the cost of each call of an operator, rather than of each item, can use a smaller margin. +pub const COUNT_MARGIN: f64 = 1.25; + +/// A count that a budget file records for each item of a case: its measure, as an index into [`MEASURES`], its column, as an index into [`COLUMNS`], and the count. +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct Count { pub measure: usize, pub column: usize, pub value: f64 } + +impl Count { + /// Whether `count`, for each item, is more than `margin` times this count, where `margin` is usually [`COUNT_MARGIN`]. Bytes allocated may also rise by one byte for each item, so a case recorded as allocating nothing can still allocate a little. + pub fn over(&self, count: f64, margin: f64) -> bool { count > margin * self.value + if MEASURES[self.measure] == "allocated" { 1.0 } else { 0.0 } } +} + +/// Adds to `counts` the counts that `field` records, and says whether it was such a field. The field gives a name from [`MEASURES`], then each column's name from [`COLUMNS`] with its count for each item, as in `instructions linux 33.1 macos 20.2`. A field that starts with a measure's name in any other form is an error, as is a column that `counts` already holds for that measure. +pub fn count(field: &str, counts: &mut Vec) -> Result { + let mut words = field.split_whitespace(); + let Some(measure) = words.next().and_then(|w| MEASURES.iter().position(|&m| m == w)) else { return Ok(false) }; + let fail = || format!("{field:?} should be a measure, then each column with its count for each item"); + let words: Vec<&str> = words.collect(); + if words.is_empty() || words.len() % 2 == 1 { return Err(fail()); } + for pair in words.chunks(2) { + let column = COLUMNS.iter().position(|&c| c == pair[0]).ok_or_else(fail)?; + let value = pair[1].parse::().ok().filter(|v| v.is_finite() && *v >= 0.0).ok_or_else(fail)?; + if counts.iter().any(|c| (c.measure, c.column) == (measure, column)) { return Err(format!("{field:?}: the {} column is counted twice", pair[0])); } + counts.push(Count { measure, column, value }); + } + Ok(true) +} + /// A case's budget for a cost, by the rule in basedpl's DEV.md: in copies, the cost times 2 under 50 copies, 1.5 from 50 to 500 and 1.3 over 500, or 2 for an `estimate`, rounded up to two significant figures and at least 0.5; in the section `widths`, a 32/64-bit ratio plus 0.25, rounded up to two decimal places. pub fn budget(section: &str, cost: f64, estimate: bool) -> f64 { // Rounds `x` up to `places` decimal places, or to tens, hundreds and so on for negative `places`. The tolerance keeps a product such as 2 × 0.27, which f64 holds a little above 0.54, from rounding up a step. @@ -231,7 +406,7 @@ pub fn hold(text: &str, holds: &mut Vec) -> Result { pub struct Budget { pub value: f64, pub rule: f64, pub cause: Option } impl Budget { - /// The budget in `column` of a case in `section` whose footer or budget line gives `budgets`, held where `holds` holds that column. `None` where `column` is `None` or the case has no budget there. + /// The budget in `column` of a case in `section` whose footer or budget line gives `budgets`, held where `holds` holds that column. `None` where `column` is `None` or the case has no budget there, which its file writes as `-`. pub fn of(section: &str, budgets: &[impl AsRef], holds: &[Held], column: Option) -> Option { let column = column?; let rule: f64 = budgets.get(column)?.as_ref().parse().ok()?; @@ -255,10 +430,19 @@ impl std::fmt::Display for Budget { } } -/// One case of a budget file: its section, its code, its budgets as written, the columns that its `held` and `busy` lines hold, and the byte span of its line. -pub struct Listed<'a> { pub section: &'a str, pub code: &'a str, pub budgets: Vec<&'a str>, pub holds: Vec, pub span: (usize, usize) } +/// One case of a budget file: its section, its code, its budgets as written, the counts that it records, the columns that its `held` and `busy` lines hold, and the byte span of its line. +pub struct Listed<'a> { + pub section: &'a str, pub code: &'a str, pub budgets: Vec<&'a str>, pub counts: Vec, pub holds: Vec, pub span: (usize, usize) +} + +impl Listed<'_> { + /// The count that the case records in `column` for `measure`, one of [`MEASURES`]. + pub fn count(&self, measure: &str, column: Option) -> Option { + self.counts.iter().copied().find(|c| MEASURES[c.measure] == measure && Some(c.column) == column) + } +} -/// The cases of `text`, a budget file such as rustymath's `tests/speed.txt`. A line `[name]`, or `[name] ` and more text, starts each section. Each line under it holds a case's code, then its budgets, separated by tabs. A line `# held ...` or `# busy ...` above a case's line holds columns of that case, as [`hold`] reads it. Blank lines and other lines that start with `#` are skipped. +/// The cases of `text`, a budget file such as rustymath's `tests/speed.txt`. A line `[name]`, or `[name] ` and more text, starts each section. Each line under it holds a case's code, then its budgets, each a number or `-` for no budget in that column, then any counts, as [`count`] reads them, separated by tabs. A line `# held ...` or `# busy ...` above a case's line holds columns of that case, as [`hold`] reads it. Blank lines and other lines that start with `#` are skipped. pub fn cases(text: &str) -> Result>, String> { let (mut cases, mut section, mut holds, mut start) = (Vec::new(), None, Vec::new(), 0); for whole in text.split_inclusive('\n') { @@ -272,23 +456,28 @@ pub fn cases(text: &str) -> Result>, String> { if let Some((name, _)) = named { section = Some(name); continue; } let section = section.ok_or_else(|| format!("{line:?} comes before the first section"))?; let mut fields = line.split('\t'); - let (code, budgets): (&str, Vec<&str>) = (fields.next().unwrap_or_default(), fields.collect()); - if budgets.iter().any(|b| b.parse::().map_or(true, |b| !b.is_finite() || b < 0.0)) { - return Err(format!("{line:?} has a budget that isn't a number")); + let (code, mut budgets, mut counts) = (fields.next().unwrap_or_default(), Vec::new(), Vec::new()); + for field in fields { + if count(field, &mut counts)? { continue; } + if !counts.is_empty() || field != "-" && field.parse::().map_or(true, |b| !b.is_finite() || b < 0.0) { + return Err(format!("{line:?} should give its budgets, each a number or `-`, then its counts")); + } + budgets.push(field); } - cases.push(Listed { section, code, budgets, holds: std::mem::take(&mut holds), span }); + cases.push(Listed { section, code, budgets, counts, holds: std::mem::take(&mut holds), span }); } if holds.is_empty() { Ok(cases) } else { Err("a held or busy line ends the file, with no case after it".into()) } } -/// Each case of `text`, a budget file, as its section, its code and its budget in `column`, as [`Budget::of`] gives it, or with `None` for every case where `column` is `None`. A case with no budget in `column` is an error. -pub fn budgeted(text: &str, column: Option) -> Result)>, String> { - cases(text)?.into_iter().map(|c| match (column, Budget::of(c.section, &c.budgets, &c.holds, column)) { - (Some(i), None) => Err(format!("{} {} has no {} budget", c.section, c.code, COLUMNS[i])), (_, budget) => Ok((c.section, c.code, budget)) +/// Each case of `text`, a budget file, with its budget in `column`, as [`Budget::of`] gives it, or with `None` for every case where `column` is `None`. A case whose line has no field for `column` is an error. One whose field is `-` has no budget there. +pub fn budgeted(text: &str, column: Option) -> Result, Option)>, String> { + cases(text)?.into_iter().map(|c| match column { + Some(i) if c.budgets.get(i).is_none() => Err(format!("{} {} has no {} budget", c.section, c.code, COLUMNS[i])), + _ => { let budget = Budget::of(c.section, &c.budgets, &c.holds, column); Ok((c, budget)) } }).collect() } -/// The pauses, in seconds, before each retry in `retried`. All of a case's runs together take a fraction of a millisecond, and one disturbance on a shared machine can last that long. The last pause waits out a neighbour's longer burst: in basedpl's `Test` run 62 on CI's runner, cases over their budgets ran slower at each of the first two retries. +/// The pauses, in seconds, before each retry in `retried`, which speed tests' calibration runs make. All of a case's runs together take a fraction of a millisecond, and one disturbance on a shared machine can last that long. The last pause waits out a neighbour's longer burst: in basedpl's `Test` run 62 on CI's runner, cases over their budgets ran slower at each of the first two retries. pub const RETRY_PAUSES: [f64; 3] = [0.1, 1.0, 10.0]; /// Keeps the calling thread busy for 0.1 s by reading the clock, so that a timing after a pause runs at the core's full speed. An idle core runs slower until it has been busy for a while. A timing's `RUNS` calls of a small case finish within microseconds, before the core speeds up: on the development Mac, cases of 65,536 items that cost 0.74-0.84 copies cost 2.28-3.06 straight after a 10 s pause (30 October 2026). [`retried`] calls it after each pause, and the Python and browser speed tests call it after theirs. @@ -308,7 +497,7 @@ pub const BUDGET_CPU: Option<&str> = match NATIVE_COLUMN { /// Whether this machine's speed tests fail on a case still over its budget after the retries: only where [`cpu`] is [`BUDGET_CPU`]. On any other CPU they print every cost, then each case still over its budget, marked `over`, and fail on none. pub fn checks_budgets() -> bool { BUDGET_CPU.is_some_and(|c| cpu() == Some(c)) } -/// The failures of a timing test that still fail after every retry. After each pause in `RETRY_PAUSES`, and [`warm`] after it, `retime` gets the failures and the pause, times them again, prints the retry and gives those that still fail. A speed test calls it after timing every case. +/// The failures of a timing test that still fail after every retry. After each pause in `RETRY_PAUSES`, and [`warm`] after it, `retime` gets the failures and the pause, times them again, prints the retry and gives those that still fail. A speed test's calibration run calls it after timing every case. An ordinary run times its failures once more under [`try_exclusive`] instead. pub fn retried(mut failures: Vec, mut retime: impl FnMut(Vec, f64) -> Vec) -> Vec { for pause in RETRY_PAUSES { if failures.is_empty() { break; } @@ -321,7 +510,7 @@ pub fn retried(mut failures: Vec, mut retime: impl FnMut(Vec, f64) -> V #[cfg(not(target_arch = "wasm32"))] mod lock { - use crate::parallel::set_threads; + use rustymath_core::parallel::set_threads; use std::{cell::RefCell, fs::{File, TryLockError}, io, sync::{Arc, Weak}, time::{Duration, Instant}}; thread_local! { static HELD: RefCell<(bool, Weak)> = const { RefCell::new((true, Weak::new())) }; } @@ -337,20 +526,33 @@ mod lock { _file: Arc, /// `GATE`, which an exclusive hold keeps until it's dropped. _gate: Option, - /// The thread setting that an exclusive hold replaced, to restore when dropped. + /// The thread setting that an exclusive hold, or `on_one_thread`, replaced, to restore when dropped. threads: Option, } impl Drop for Hold { fn drop(&mut self) { if let Some(n) = self.threads { set_threads(n); } } } + impl Hold { + /// The hold, now running every call on its calling thread alone until it's dropped, as an exclusive hold always does. A speed test's ordinary run takes its shared hold this way, because speed budgets are for one thread. + pub fn on_one_thread(mut self) -> Self { self.threads = self.threads.or_else(|| Some(set_threads(1))); self } + } + /// Takes the timing lock exclusively, for a timing. It waits up to 5 minutes for builds, test runs and other timings, then fails. From the start of its wait it holds `GATE`, so no build or test run starts meanwhile. After waiting 30 seconds it prints, once, the other processes that have the lock's file open. On macOS it also sets the calling thread's QoS class to user-interactive. macOS then favours the performance cores for that thread. QoS isn't CPU affinity. pub fn exclusive() -> io::Result { hold_at(LOCK, false) } /// Takes the timing lock shared, for a build or a test run. It waits behind any timing that holds `GATE` or the lock, for up to 5 minutes, then fails. A nested shared hold acquired by the same thread reuses its file. pub fn shared() -> io::Result { hold_at(LOCK, true) } + /// Takes the timing lock exclusively if no build, test run or other timing holds it or waits for it, or gives `None` at once, without waiting. Otherwise it is as [`exclusive`]. A speed test's ordinary run uses it to confirm a case over its budget on a quiet machine. It must drop its shared hold first, since a hold can't change mode. + pub fn try_exclusive() -> io::Result> { + match held_at(LOCK, false, false) { Err(e) if e.kind() == io::ErrorKind::WouldBlock => Ok(None), held => held.map(Some) } + } + /// Takes the timing lock whose file is `lock`, exclusively or `shared`, as [`exclusive`] and [`shared`] take `LOCK`. Its gate is `GATE` for `LOCK`, and `.gate` for any other file. A test of the protocol holds a file of its own, which no build or timing holds. - pub fn hold_at(lock: &str, shared: bool) -> io::Result { + pub fn hold_at(lock: &str, shared: bool) -> io::Result { held_at(lock, shared, true) } + + /// Takes the timing lock whose file is `lock`, as [`hold_at`] does where it may `wait`, or else with one try at its gate and its lock. + fn held_at(lock: &str, shared: bool, wait: bool) -> io::Result { if let Some((was_shared, file)) = HELD.with(|h| { let h = h.borrow(); h.1.upgrade().map(|f| (h.0, f)) }) { if shared && was_shared { return Ok(Hold { _file: file, _gate: None, threads: None }); } return Err(io::Error::new(io::ErrorKind::InvalidInput, "timing locks can only nest shared inside shared")); @@ -362,7 +564,7 @@ mod lock { let open = |path: &str| File::options().create(true).append(true).open(path).map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}"))); let gate = if lock == LOCK { GATE.into() } else { format!("{lock}.gate") }; let (gate_file, file) = (open(&gate)?, open(lock)?); - let mut wait = Wait { start: Instant::now(), shared, waiting: false, reported: false }; + let mut wait = Wait { start: Instant::now(), shared, wait, waiting: false, reported: false }; wait.take(&gate_file, &gate)?; wait.take(&file, lock)?; let file = Arc::new(file); @@ -370,11 +572,11 @@ mod lock { Ok(Hold { _file: file, _gate: (!shared).then_some(gate_file), threads: (!shared).then(|| set_threads(1)) }) } - /// One hold's wait for its gate and then its lock, against one deadline of 5 minutes. - struct Wait { start: Instant, shared: bool, waiting: bool, reported: bool } + /// One hold's attempt on its gate and then its lock: one try each, or, where it may `wait`, tries against one deadline of 5 minutes. + struct Wait { start: Instant, shared: bool, wait: bool, waiting: bool, reported: bool } impl Wait { - /// Locks `file`, which is the file at `path`, as the hold needs it, waiting up to 5 minutes before failing. + /// Locks `file`, which is the file at `path`, as the hold needs it. Where it may wait, it waits up to 5 minutes before failing. Otherwise a lock that's taken gives a `WouldBlock` error. fn take(&mut self, file: &File, path: &str) -> io::Result<()> { let others = if self.shared { "a timing" } else { "a build or another timing" }; loop { @@ -382,6 +584,7 @@ mod lock { Ok(()) => return Ok(()), Err(TryLockError::WouldBlock) => {} Err(TryLockError::Error(e)) => return Err(io::Error::new(e.kind(), format!("{path}: {e}"))) } + if !self.wait { return Err(io::Error::new(io::ErrorKind::WouldBlock, format!("{others} holds {path}"))); } if self.start.elapsed() > Duration::from_secs(300) { return Err(io::Error::new( io::ErrorKind::TimedOut, format!("{others} has held {path} for 5 minutes. Processes with it open: {}", users(path))));