diff --git a/.Rbuildignore b/.Rbuildignore index 4288dd6b..829595c8 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,6 +6,10 @@ ^\.Rproj\.user$ ^ChangeLog +^dev-notes$ +^\.vscode +^CLAUDE\.md$ + ^inst/benchmarks/ ^inst/profiling/ ^inst/vignettes/ @@ -13,58 +17,9 @@ ^\.git ^src/Makevars(?!.in$|.win$) -^src/Cogaps.o -^src/GapsParameters.o -^src/GapsResult.o -^src/GapsRunner.o -^src/GapsStatistics.o -^src/RcppExports.o -^src/test-runner.o -^src/atomic/AtomicDomain.o -^src/atomic/ProposalQueue.o -^src/cpp_tests/testAtomicDomain.o -^src/cpp_tests/testDenseGibbsSampler.o -^src/cpp_tests/testFileParsers.o -^src/cpp_tests/testHashSets.o -^src/cpp_tests/testHybridMatrix.o -^src/cpp_tests/testHybridVector.o -^src/cpp_tests/testMatrix.o -^src/cpp_tests/testRandom.o -^src/cpp_tests/testSerialization.o -^src/cpp_tests/testSparseGibbsSampler.o -^src/cpp_tests/testSparseIterator.o -^src/cpp_tests/testSparseMatrix.o -^src/cpp_tests/testSparseVector.o -^src/cpp_tests/testVector.o -^src/data_structures/HashSets.o -^src/data_structures/HybridMatrix.o -^src/data_structures/HybridVector.o -^src/data_structures/Matrix.o -^src/data_structures/SparseIterator.o -^src/data_structures/SparseMatrix.o -^src/data_structures/SparseVector.o -^src/data_structures/Vector.o -^src/file_parser/CsvParser.o -^src/file_parser/GctParser.o -^src/file_parser/FileParser.o -^src/file_parser/TsvParser.o -^src/file_parser/MtxParser.o -^src/gibbs_sampler/AlphaParameters.o -^src/gibbs_sampler/DenseStoragePolicy.o -^src/gibbs_sampler/SparseStoragePolicy.o -^src/math/Math.o -^src/math/MatrixMath.o -^src/math/Random.o -^src/math/VectorMath.o -^src/atomic/Atom.o -^src/atomic/ConcurrentAtom.o -^src/atomic/ConcurrentAtomicDomain.o -^src/file_parser/CharacterDelimitedParser.o -^src/file_parser/MatrixElement.o -^src/gibbs_sampler/DenseNormalModel.o -^src/gibbs_sampler/SparseNormalModel.o - -^src/math/VectorMath\.o$ +^src/.*\.o$ +^src/.*\.so$ +^src/.*\.dll$ ^nextflow* ^main\.nf @@ -72,4 +27,3 @@ \.nextflow \.cirro ^tests/nextflow - diff --git a/.gitignore b/.gitignore index 4b6c4366..1d237ebc 100755 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,9 @@ src/Makevars # ignore checkpoint files *.out + +# editor / IDE local configs +.vscode/ + +# built package tarballs +CoGAPS_*.tar.gz diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..f317080c --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,18 @@ +{ + "configurations": [ + { + "name": "Mac", + "includePath": [ + "${workspaceFolder}/**", + "/opt/homebrew/lib/R/4.6/site-library/**", + "/opt/homebrew/Cellar/r/4.6.1/lib/R/include" + ], + "defines": [], + "compilerPath": "/usr/bin/clang", + "cStandard": "c17", + "cppStandard": "c++17", + "intelliSenseMode": "macos-clang-x64" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..7c93d2f8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,261 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +CoGAPS (Coordinated Gene Activity in Pattern Sets) is a Bioconductor R package wrapping a C++ +Bayesian MCMC matrix factorization algorithm (GAPS). It factors a data matrix `D ≈ A × P`, where +`A` (featureLoadings) is genes × patterns and `P` (sampleFactors) is patterns × samples, and links +the result to gene set statistics. + +Developer notes live in `dev-notes/`, indexed by `dev-notes/README.md` — including the log of fixed +defects (`dev-notes/132-LLM-assisted-solved-issues.md`), which is the best entry point for why the +C++ looks the way it does. Working agreements with the maintainer are in +`dev-notes/rus/agent-rules-rus.md`, the one Russian-language file here. The whole directory is +excluded from the package build via `.Rbuildignore`. + +## Setting up a fresh machine + +Verified against R 4.6.1 / Bioconductor 3.23. + +```r +install.packages("BiocManager") +BiocManager::install(c("devtools", "testthat", "roxygen2", "BiocStyle", + "SingleCellExperiment", "fgsea", "gplots", "SeuratObject")) +devtools::install_deps(".", dependencies = TRUE) # the rest, 21 in DESCRIPTION +``` + +Three of these are easy to overlook, because nothing fails until it does: + +- **`testthat`** is in `LinkingTo`, not just `Suggests` — it ships the Catch2 header + the C++ test suite compiles against. Without it `src/` does not build at all. +- **`xml2`** is what `tests/testthat/test_cpp.R` uses to turn the Catch report into + per-case expectations. It is only suggested, so the test silently falls back to a + single pass/fail check when it is absent. +- **`SeuratObject`** is only needed for `R CMD check`, which refuses to run a + complete check while a suggested package is missing. + +Outside R (Homebrew names; use the distro equivalent elsewhere): + +```bash +brew install autoconf autoconf-archive pandoc gh +xcode-select --install # macOS only, if the compiler is missing +``` + +- **`autoconf` + `autoconf-archive`** regenerate `configure`, and only matter when + `configure.ac` changes. `autoconf-archive` supplies the `AX_COMPILER_*` macros — + see the note under Build & test for why `aclocal` has to run first. +- **`pandoc`** builds the vignette: `vignettes/CoGAPS.Rmd` goes through + `VignetteBuilder: knitr` → rmarkdown → pandoc. `R CMD check` builds vignettes, so + without it the check that CI runs fails here even though the package is fine. + (RStudio ships its own copy; a plain shell does not.) +- **`gh`** is not needed to build or test anything — it is how issues and PRs are + read from the terminal. It needs `gh auth login` once per machine. +- The compiler itself comes from the **Xcode Command Line Tools**, not from + Homebrew. Verified against Apple clang 17. + +Versions on the current machine: autoconf 2.73, autoconf-archive 2024.10.16, +pandoc 3.10.1, gh 2.97.0. + +Do **not** upgrade the generated documentation casually: `DESCRIPTION` pins +`RoxygenNote: 7.3.3`, and running `devtools::document()` under a newer roxygen2 +rewrites all 77 `.Rd` files. + +## Build & test + +All commands are run from the package root. + +```r +devtools::load_all() # load package +devtools::load_all(recompile = TRUE) # force C++ recompile +devtools::test() # all R tests +testthat::test_file("tests/testthat/test_top_level.R") # one R test file +devtools::document() # regenerate roxygen2 docs + NAMESPACE +Rcpp::compileAttributes() # after changing // [[Rcpp::export]] signatures +``` + +```bash +R CMD check --no-manual . # what CI runs (FertigLab/actions r-build-check) + +# after editing configure.ac -- BOTH commands, in this order +aclocal -I /opt/homebrew/share/aclocal # autoconf-archive macros +autoconf +``` + +`autoconf` alone is not enough: `configure.ac` uses `AX_COMPILER_VENDOR` and +`AX_COMPILER_VERSION` from autoconf-archive, and it is `aclocal` that makes them +available. Skipping it leaves both macros unexpanded in `configure`, where they +become literal shell commands — `configure` still completes, but prints +`AX_COMPILER_VENDOR: command not found`, leaves `$ax_cv_cxx_compiler_vendor` +empty, and so silently turns `--enable-warnings` into a no-op. That is the state +`configure` is in on `master`; on this branch it is generated correctly, so keep +it that way. `aclocal.m4` is an artefact of this and is not committed. + +Note that `--enable-warnings` currently fails the build with `-Werror`, on +`-Wcast-function-type-mismatch` raised inside Rcpp's own `routines.h` under +newer clang. No CoGAPS source file produces a warning. + +`DESCRIPTION` pins `RoxygenNote: 7.3.3`. Running `devtools::document()` under a newer roxygen2 +rewrites all 77 `.Rd` files and bumps that field — check the resulting diff before committing it. + +### C++ unit tests (Catch2, shipped with the `testthat` R package) + +These are **not** exported — reach them with `:::`. The file-parser tests read their data paths +from the global environment, so set `gistCsvPath`/`gistTsvPath`/`gistMtxPath`/`gistGctPath` (with +`<<-`) before calling the runner directly. + +```r +CoGAPS:::run_catch_unit_tests() # all, console output +CoGAPS:::run_catch_unit_tests_by_tag("Test Vector.h") # by name +CoGAPS:::run_catch_unit_tests_by_tag("[vector]") # by tag +CoGAPS:::run_catch_unit_tests_by_tag("[vector][green]") # AND +CoGAPS:::run_catch_unit_tests_by_tag("[vector],[green]") # OR +CoGAPS:::catch_test_case_names() # what is compiled in +``` + +The suite also runs inside `devtools::test()` via `tests/testthat/test_cpp.R`, so a broken C++ +test breaks `R CMD check`. That file does **not** use the console form: Catch writes its report +from C++ directly to stdout, where testthat cannot capture it, and one `expect_equal(…, 0L)` would +collapse the whole suite into a single pass/fail. It passes `reporter = "xml"` plus an `output` +file instead, then parses it with `xml2` so each `` becomes its own expectation, named +and located on failure. `output = ""` (the default) still writes to stdout, which is what keeps +the interactive call unchanged. + +`catch_test_case_names()` guards against a vacuous pass: with `--disable-cpp-tests`, or on Windows +where `Makevars.win` lists no `cpp_tests` objects, no test case is registered and the runner +returns 0 regardless. The test skips in that case rather than reporting success. See +`src/cpp_tests/README.md` for the longer write-up. + +### Build options + +Options are declared in `configure.ac`; pass them by setting the matching env var before loading: + +```r +Sys.setenv(enable_debug = "yes") # -g -O0 +devtools::load_all(recompile = TRUE) +``` + +Available toggles: `--enable-debug` (`-g -O0`), `--enable-gaps-debug` (`-DGAPS_DEBUG`), +`--enable-cpp-tests` (on by default; disable to cut compile time), `--enable-checkpoints` +(**off** by default — `-DGAPS_DISABLE_CHECKPOINTS` is set unless enabled), `--enable-warnings` +(`-Wall -Wextra -Werror`), `--enable-simd` (on by default; `sse` disables AVX). + +`src/Makevars.win` is maintained by hand (Windows has no `configure`): it hard-codes +`-DGAPS_DISABLE_CHECKPOINTS` and lists no `cpp_tests/*.o`, so the Catch suite is empty on Windows +and `test_cpp.R` passes trivially there. Adding a source file means editing **both** +`configure.ac` and `Makevars.win`. + +## Architecture + +### Layers + +``` +R/ Public API and S4 classes + CoGAPS.R CoGAPS(), scCoGAPS(), GWCoGAPS() entry points + DistributedCogaps.R subset orchestration, pattern matching, stitching + SubsetData.R how data is split into sets (explicit / weighted / uniform) + class-*.R S4 class definitions + generics + methods-*.R S4 method implementations + HelperFunctions.R input validation, dim names, file/RDS handling + RcppExports.R auto-generated — never edit by hand + +src/ C++ core + GapsRunner.cpp/.h C++ entry: gaps::run() (in-memory or from file) + Cogaps.cpp Rcpp glue (cogaps_cpp, cogaps_from_file_cpp) + GapsParameters/Result/Statistics + gibbs_sampler/ DenseNormalModel, SparseNormalModel, SingleThreadedGibbsSampler + atomic/ atomic domain backing the Gibbs sampler + data_structures/ Matrix, SparseMatrix, HybridMatrix, Vector, SparseIterator, ... + file_parser/ CSV / TSV / MTX / GCT readers + math/ Random, VectorMath, MatrixMath, SIMD + utils/ header-only: Archive, GapsAssert, GapsPrint, GlobalConfig + cpp_tests/ Catch2 tests + test-runner.cpp exposes the Catch suite to R (sits in src/, not in cpp_tests/) +``` + +### Dispatch + +`CoGAPS(data, params, nPatterns, ...)` validates inputs, then picks one of three paths: + +- `cogaps_cpp` — in-memory matrix (default) +- `cogaps_from_file_cpp` — `data` is a file path +- `distributedCogaps` — `params@distributed` is `"genome-wide"` or `"single-cell"`; splits the data + into subsets, runs them in parallel via BiocParallel, matches patterns across sets + (`findConsensusMatrix`/`patternMatch`), then `stitchTogether`s the result + +`nPatterns` is required — the `CogapsParams` initializer errors without it, and the `params` +default is `new("CogapsParams", nPatterns = nPatterns)`. Distributed runs want on-disk data +(mtx/tsv/csv/gct); an in-memory matrix warns. + +In `stitchTogether`, the **fixed** matrix must be read from +`result[[1]]@metadata$params@fixedPatterns`, not from the worker's `@featureLoadings` / +`@sampleFactors`: when a matrix is fixed its statistics are never accumulated, so those slots are +all zeros. + +### Sampler geometry (see `src/README.md` for the full write-up) + +Every run drives two samplers over `D ≈ A × P`. Each sampler owns its own copy of `AP` (synced by +`sync()`, built by `extraInitialization()`), its purpose matrix, an uncertainty matrix the size of +`AP`, and a `const` reference to its counterpart. One sampler of the pair is transposed: + +- **ASampler** (transposed when `D` is not transposed): `AP` is l×m, `A` is m×k +- **PSampler** (non-transposed): `AP` is m×l, `P` is l×k +- Invariants: `nrows(MyMatrix) == ncols(APMatrix)`, `ncols(MyMatrix) == k`, and after `sync()`, + `APMatrix == t(other_sampler.APMatrix)` + +### Uncertainty model + +`S = max(0.1·D, 0.1)` by default — a relative-error model floored so that zeros and `D < 1` get +`S = 0.1`. `DenseNormalModel` materialises it as `mSMatrix`; `SparseNormalModel` never materialises +it (that would defeat sparse storage) and instead recomputes `1/S²` on the fly via the file-local +`invSSq()` helper, with the constant zero-entry uncertainty folded into `mBeta`. The two must agree +— that is what makes `sparseOptimization=TRUE` match a dense run. A user-supplied `uncertainty=` +matrix is used verbatim, and `sparseOptimization` rejects one outright. + +**`Vector::pad(val)` fills every allocated element; `padSIMD(val)` fills only the SIMD tail.** Never +use `pad()` on a matrix whose contents matter — that bug silently discarded user uncertainty for +years. SIMD loops read up to `SIMD_INC - 1` elements past the end, so the tail must be non-zero to +avoid `0/0 = NaN`. + +### Threading + +The asynchronous/OpenMP multi-threaded sampler was **removed** — it broke MCMC detailed balance. +CoGAPS always runs single-threaded. `nThreads` and `asynchronousUpdates` remain in the `CoGAPS()` +signature for backward compatibility, warn when set to a non-default value, and are otherwise +ignored. `compiledWithOpenMPSupport()` is kept, also for compatibility, and always returns `FALSE`. +Do not reintroduce any of these into `allParams` or the C++ parameter struct. + +## Conventions + +### R + +- S4 throughout: new generics go in `class-*.R`, implementations in `methods-*.R`. +- `CogapsResult` extends `LinearEmbeddingMatrix`: `featureLoadings` = Amean, `sampleFactors` = Pmean, + `loadingStdDev` = Asd, `factorStdDev` = Psd. +- Distributed params (`nSets`, `cut`, `minNS`, `maxNS`) cannot be passed to the `CogapsParams` + constructor — the initializer errors out. Use `setDistributedParams()` afterwards. +- `scCoGAPS()` / `GWCoGAPS()` are deprecated wrappers for `CoGAPS(..., distributed = ...)`. +- New files must be added to the `Collate:` field in `DESCRIPTION`. + +### C++ + +- Each Catch2 test file needs both `#include ` and `#include "testthat-tweak.h"` + (the tweak enables tag-based selection). **A test case with no `SECTION` block is never run.** +- Adding any new `.cpp` (source or test) requires adding its `.o` to `GAPS_SOURCE_FILES` in + `configure.ac`, then running `autoconf`. There is no wildcard build. + +### Working with the maintainer + +- Conversation is in Russian; everything an outside reader sees — comments, docs, README, commit + messages — is in English. +- **Commit only when explicitly asked.** Never commit on your own initiative. +- Do not rewrite history (`rebase -i`, force push) once Bioconductor review has started. +- Don't do things "just in case" without asking; ask rather than guess when something is unclear. +- Comments only where a clarification is genuinely needed. + +### Merging master into a long-lived branch + +`git checkout --ours ` takes the branch's version of the **whole file**, silently discarding +master changes that had merged cleanly elsewhere in it. Use `git checkout -m -- ` to get the +real three-way merge back and resolve only the conflicting hunks. diff --git a/DESCRIPTION b/DESCRIPTION index 76d8d7b7..a40f1ac4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: CoGAPS -Version: 3.33.1 -Date: 2025-03-11 +Version: 3.33.2 +Date: 2026-08-03 Title: Coordinated Gene Activity in Pattern Sets Author: Jeanette Johnson, Ashley Tsang, Jacob Mitchell, Thomas Sherman, Wai-shing Lee, Conor Kelton, Ondrej Maxian, Jacob Carey, Genevieve Stein-O'Brien, Michael Considine, Maggie Wodicka, John Stansfield, diff --git a/LICENSE b/LICENSE index 3420b69a..441bd2c3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,29 +1,3 @@ -BSD 3-Clause License - -Copyright (c) 2020, FertigLab -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +YEAR: 2020 +COPYRIGHT HOLDER: FertigLab +ORGANIZATION: FertigLab diff --git a/NAMESPACE b/NAMESPACE index 013c3434..b1e8db9b 100755 --- a/NAMESPACE +++ b/NAMESPACE @@ -41,7 +41,7 @@ export(toCSV) exportClasses(CogapsParams) exportClasses(CogapsResult) import(dplyr) -import(fgsea) +#import(fgsea) importClassesFrom(S4Vectors,Annotated) importClassesFrom(S4Vectors,character_OR_NULL) importClassesFrom(SingleCellExperiment,LinearEmbeddingMatrix) @@ -93,6 +93,7 @@ importFrom(stats,cor) importFrom(stats,cutree) importFrom(stats,hclust) importFrom(stats,manova) +importFrom(stats,setNames) importFrom(stats,weighted.mean) importFrom(tools,file_ext) importFrom(utils,packageVersion) diff --git a/NEWS b/NEWS index 73c1431f..3cf07727 100755 --- a/NEWS +++ b/NEWS @@ -8,3 +8,4 @@ CoGAPS implements a MCMC non-negative matrix factorization algorithm and corresp 15Jul2013 - Added processed data from Colantuoni et al. (2011) Temporal dynamics and genetic control of transcription in the human prefrontal cortex. Nature, 478:519-523 used for Fertig et al. (2013) Pattern identification in time course gene expression data with the CoGAPS matrix factorization. Chapter 6 in MF Ochs (ed) Methods in Molecular Biology: Gene Function Analysis, 2nd Edition, Springer, New York. 14Sep2014 - Removed dependency on GAPS-JAGS core to enable complete installation from Bioconductor. 17Aug2015 - Datasets limited to those needed for simple package simulations. Added ability to have pre-determined patterns in the factorization (maps) +31Jul2026 - Removed the asynchronous (OpenMP) Gibbs sampler: it broke MCMC detailed balance, so CoGAPS now always runs single-threaded. The nThreads and asynchronousUpdates arguments are accepted for backward compatibility but ignored, and compiledWithOpenMPSupport() always returns FALSE. Fixed the uncertainty model: a user-supplied uncertainty matrix was previously overwritten with 1.0 and thus ignored, and the default uncertainty is now the data-driven S = max(0.1*D, 0.1), applied consistently by the dense and sparse samplers. Added an --enable-checkpoints configure option (checkpoints stay off by default) and a number of correctness fixes in the atomic domain, serialization, sparse containers and the SIMD paths, with the C++ unit test suite revived and extended. diff --git a/R/CoGAPS.R b/R/CoGAPS.R index 03de2c51..c8ce0151 100755 --- a/R/CoGAPS.R +++ b/R/CoGAPS.R @@ -28,12 +28,13 @@ checkpointsEnabled <- function() #' Check if compiler supported OpenMP #' @export #' -#' @return true/false if OpenMP was supported +#' @return FALSE (OpenMP support was removed together with the asynchronous +#' sampler; CoGAPS now always runs single-threaded) #' @examples #' CoGAPS::compiledWithOpenMPSupport() compiledWithOpenMPSupport <- function() { - compiledWithOpenMPSupport_cpp() + FALSE } #' CoGAPS Matrix Factorization Algorithm @@ -47,7 +48,7 @@ compiledWithOpenMPSupport <- function() #' @param data File name or R object (see details for supported types) #' @param params CogapsParams object #' @param nPatterns rank of the nmf decomposition -#' @param nThreads maximum number of threads to run on +#' @param nThreads deprecated and ignored; CoGAPS now always runs single-threaded #' @param messages T/F for displaying output #' @param outputFrequency number of iterations between each output (set to 0 to #' disable status updates, other output is controlled by @code messages) @@ -66,7 +67,8 @@ compiledWithOpenMPSupport <- function() #' only worker 1 prints output and each worker outputs when it finishes, this #' is not neccesary when using the default parallel methods (i.e. distributed #' CoGAPS) but only when the user is manually calling CoGAPS in parallel -#' @param asynchronousUpdates enable asynchronous updating which allows for multi-threaded runs +#' @param asynchronousUpdates deprecated and ignored; the asynchronous sampler was +#' removed because it broke MCMC detailed balance #' @param nSnapshots how many snapshots to take in each phase, setting this to 0 disables #' snapshots #' @param snapshotPhase which phase to take snapsjots in e.g. "equilibration", "sampling", @@ -104,23 +106,18 @@ CoGAPS <- function(data, params=new("CogapsParams", nPatterns=nPatterns), params <- getValueOrRds(params) validObject(params) - # OpenMP availability determines whether the asynchronous sampler can run. - # Without OpenMP, CoGAPS falls back to the sequential sampler path. - if (!compiledWithOpenMPSupport()) - { - if (asynchronousUpdates | nThreads > 1) - warning(paste( - "OpenMP is not available in this CoGAPS build;", - "running with asynchronousUpdates=FALSE and nThreads=1;", - "this may change results in a platform-dependent manner." - )) - asynchronousUpdates = FALSE - nThreads = 1 - } + # The asynchronous multi-threaded sampler was removed (it broke MCMC detailed + # balance); CoGAPS now always runs single-threaded. The 'nThreads' and + # 'asynchronousUpdates' arguments are kept for backward compatibility but are + # ignored. Warn only when a non-default value is requested. + if (!identical(as.numeric(nThreads), 1) || isTRUE(asynchronousUpdates)) + warning("'nThreads' and 'asynchronousUpdates' are deprecated and ignored; ", + "CoGAPS now always runs single-threaded (async broke MCMC balance)") # store all parameters in a list and parse parameters from ... + # (nThreads/asynchronousUpdates are deprecated no-ops -- they are accepted as + # arguments for backward compatibility but not threaded through to the sampler) allParams <- list("gaps"=params, - "nThreads"=nThreads, "messages"=messages, "outputFrequency"=outputFrequency, "nSnapshots"=nSnapshots, @@ -134,7 +131,6 @@ CoGAPS <- function(data, params=new("CogapsParams", nPatterns=nPatterns), "BPPARAM"=BPPARAM, "outputToFile"=NULL, "workerID"=workerID, - "asynchronousUpdates"=asynchronousUpdates, "dataName"=dataName ) allParams <- parseExtraParams(allParams, list(...)) @@ -175,7 +171,8 @@ CoGAPS <- function(data, params=new("CogapsParams", nPatterns=nPatterns), #' params <- setParam(params, "nIterations", 100) #' result <- scCoGAPS(t(GIST.matrix), params, BPPARAM=BiocParallel::SerialParam()) #' } -scCoGAPS <- function(data, params=new("CogapsParams"), nThreads=1, messages=TRUE, +scCoGAPS <- function(data, params=new("CogapsParams", nPatterns=nPatterns), +nPatterns, nThreads=1, messages=TRUE, outputFrequency=500, uncertainty=NULL, checkpointOutFile="gaps_checkpoint.out", checkpointInterval=1000, checkpointInFile=NULL, transposeData=FALSE, BPPARAM=NULL, workerID=1, asynchronousUpdates=FALSE, ...) @@ -215,7 +212,8 @@ BPPARAM=NULL, workerID=1, asynchronousUpdates=FALSE, ...) #' params <- setParam(params, "nIterations", 100) #' result <- GWCoGAPS(GIST.matrix, params, BPPARAM=BiocParallel::SerialParam()) #' } -GWCoGAPS <- function(data, params=new("CogapsParams"), nThreads=1, messages=TRUE, +GWCoGAPS <- function(data, params=new("CogapsParams", nPatterns=nPatterns), +nPatterns, nThreads=1, messages=TRUE, outputFrequency=500, uncertainty=NULL, checkpointOutFile="gaps_checkpoint.out", checkpointInterval=1000, checkpointInFile=NULL, transposeData=FALSE, BPPARAM=NULL, workerID=1, asynchronousUpdates=FALSE, ...) diff --git a/R/DistributedCogaps.R b/R/DistributedCogaps.R index bd6cc17c..619f2dfb 100755 --- a/R/DistributedCogaps.R +++ b/R/DistributedCogaps.R @@ -1,282 +1,276 @@ -#' make correct call to internal CoGAPS dispatch function, CoGAPS could be -#' called directly, but to avoid any re-entrant behavior this function is called -#' instead. It is a light wrapper around cogaps_cpp that handles setting -#' the distributed parameters -#' @keywords internal -#' @param data data in a supported format -#' @param allParams list of all parameters -#' @param uncertainty uncertainty of data in the same format as data -#' @param subsetIndices indices of the subset of data to run on -#' @param workerID worker ID for parallelization -#' @return CogapsResult object -callInternalCoGAPS <- function(data, allParams, uncertainty, subsetIndices, -workerID) -{ - # identify which mode of parallelization - genomeWide <- allParams$gaps@distributed == "genome-wide" - allParams$gaps@distributed <- NULL - - # subset gene/sample names - if (genomeWide) - allParams$geneNames <- allParams$geneNames[subsetIndices] - else - allParams$sampleNames <- allParams$sampleNames[subsetIndices] - - allParams$gaps@subsetIndices <- subsetIndices - allParams$gaps@subsetDim <- ifelse(genomeWide, 1, 2) - allParams$workerID <- workerID - - # Distributed CoGAPS parallelizes across data subsets instead of using the - # OpenMP asynchronous sampler within each worker. Each worker is therefore - # run with asynchronousUpdates=FALSE and nThreads=1. - allParams$asynchronousUpdates <- FALSE - allParams$nThreads <- 1 - - # call CoGAPS - internal <- ifelse(is(data, "character"), cogaps_from_file_cpp, cogaps_cpp) - raw <- internal(data, allParams, uncertainty) - return(createCogapsResult(raw, allParams)) -} - -#' CoGAPS Distributed Matrix Factorization Algorithm -#' @keywords internal -#' -#' @description runs CoGAPS over subsets of the data and stitches the results -#' back together -#' @details For file types CoGAPS supports csv, tsv, and mtx -#' @param data File name or R object (see details for supported types) -#' @param allParams list of all parameters used in computation -#' @param uncertainty uncertainty matrix (same supported types as data) -#' @return list -#' @importFrom BiocParallel bplapply MulticoreParam -distributedCogaps <- function(data, allParams, uncertainty) -{ - # randomly sample either rows or columns into subsets to break the data up - set.seed(allParams$gaps@seed) - sets <- createSets(data, allParams) - if (min(sapply(sets, length)) < allParams$gaps@nPatterns) - stop("data subset dimension less than nPatterns") - - if (is.null(allParams$BPPARAM)) - allParams$BPPARAM <- BiocParallel::MulticoreParam(workers=length(sets)) - - initialResult <- NULL - if (is.null(allParams$gaps@fixedPatterns)) - { - # run Cogaps normally on each subset of the data - gapsCat(allParams, "Running Across Subsets...\n\n") - initialResult <- bplapply(1:length(sets), BPPARAM=allParams$BPPARAM, - FUN=function(i) - { - callInternalCoGAPS(data, allParams, uncertainty, sets[[i]], i) - }) - - # get all unmatched patterns - if (allParams$gaps@distributed == "genome-wide") - unmatchedPatterns <- lapply(initialResult, function(x) x@sampleFactors) - else - unmatchedPatterns <- lapply(initialResult, function(x) x@featureLoadings) - - # match patterns in either A or P matrix - gapsCat(allParams, "\nMatching Patterns Across Subsets...\n") - matchedPatterns <- findConsensusMatrix(unmatchedPatterns, allParams$gaps) - } - else - { - matchedPatterns <- list(consensus=allParams$gaps@fixedPatterns) - } - - # set fixed matrix - allParams$gaps@nPatterns <- ncol(matchedPatterns$consensus) - allParams$gaps@fixedPatterns <- matchedPatterns$consensus - allParams$gaps@whichMatrixFixed <- ifelse(allParams$gaps@distributed - == "genome-wide", "P", "A") - - # run final phase with fixed matrix - gapsCat(allParams, "Running Final Stage...\n\n") - finalResult <- bplapply(1:length(sets), BPPARAM=allParams$BPPARAM, - FUN=function(i) - { - callInternalCoGAPS(data, allParams, uncertainty, sets[[i]], i) - }) - - # concatenate final result - fullResult <- stitchTogether(finalResult, allParams, sets) - - # add diagnostic information about initial run before returning - if (!is.null(initialResult)) # check that initial phase was run - { - fullResult$diagnostics$firstPass <- initialResult - fullResult$diagnostics$unmatchedPatterns <- unmatchedPatterns - fullResult$diagnostics$clusteredPatterns <- matchedPatterns$clusteredPatterns - fullResult$diagnostics$CorrToMeanPattern <- lapply(matchedPatterns$clusteredPatterns, corrToMeanPattern) - } - - # include the subsets used - if (allParams$gaps@distributed == "genome-wide") - fullResult$diagnostics$subsets <- lapply(sets, function(s) fullResult$geneNames[s]) - else - fullResult$diagnostics$subsets <- lapply(sets, function(s) fullResult$sampleNames[s]) - - # return list, calling function will process this into a CogapsResult object - return(fullResult) -} - - -#' find the consensus pattern matrix across all subsets -#' @export -#' -#' @param unmatchedPatterns list of all unmatched pattern matrices from initial -#' run of CoGAPS -#' @param gapsParams list of all CoGAPS parameters -#' @return matrix of consensus patterns -findConsensusMatrix <- function(unmatchedPatterns, gapsParams) -{ - allPatterns <- do.call(cbind, unmatchedPatterns) - comb <- expand.grid(1:gapsParams@nSets, 1:gapsParams@nPatterns) - colnames(allPatterns) <- paste(comb[,1], comb[,2], sep=".") - return(patternMatch(allPatterns, gapsParams)) -} - -#' Match Patterns Across Multiple Runs -#' @keywords internal -#' -#' @param allPatterns matrix of patterns stored in the columns -#' @param gapsParams CoGAPS parameters object -#' @return a matrix of consensus patterns -#' @importFrom stats weighted.mean -patternMatch <- function(allPatterns, gapsParams) -{ - # cluster patterns - clusters <- corcut(allPatterns, gapsParams@cut, gapsParams@minNS) - - # function to split a cluster in two (might fail to do so) - splitCluster <- function(list, index, minNS) - { - split <- corcut(list[[index]], 2, minNS) - list[[index]] <- split[[1]] - if (length(split) > 1) - list <- append(list, split[2]) - return(list) - } - - # split large clusters into two - tooLarge <- function(x) ncol(x) > gapsParams@maxNS - indx <- which(sapply(clusters, tooLarge)) - while (length(indx) > 0) - { - clusters <- splitCluster(clusters, indx[1], gapsParams@minNS) - indx <- which(sapply(clusters, tooLarge)) - } - names(clusters) <- as.character(1:length(clusters)) - - # create matrix of mean patterns - weighted by correlation to mean pattern - meanPatterns <- sapply(clusters, function(clust) apply(clust, 1, - function(row) weighted.mean(row, corrToMeanPattern(clust)^3))) - colnames(meanPatterns) <- paste("Pattern", 1:length(clusters)) - - # returned patterns after scaling max to 1 - return(list("clusteredPatterns"=clusters, - "consensus"=apply(meanPatterns, 2, function(col) col / max(col)))) -} - -#' calculate correlation of each pattern in a cluster to the cluster mean -#' @keywords internal -#' @return correlation of each pattern -corrToMeanPattern <- function(cluster) -{ - meanPat <- rowMeans(cluster) - sapply(1:ncol(cluster), function(j) round(cor(x=cluster[,j], y=meanPat), 3)) -} - -#' cluster patterns together -#' @keywords internal -#' -#' @param allPatterns matrix of all patterns across subsets -#' @param cut number of branches at which to cut dendrogram -#' @param minNS minimum of individual set contributions a cluster must contain -#' @return patterns listed by which cluster they belong to -#' @importFrom cluster agnes -#' @importFrom stats cutree as.hclust cor -corcut <- function(allPatterns, cut, minNS) -{ - corr.dist <- cor(allPatterns) - corr.dist <- 1 - corr.dist - - if (any(is.na(corr.dist))) - { - stop("NA values in correlation of patterns") - } - - clusterSummary <- cluster::agnes(x=corr.dist, diss=TRUE, "complete") - clusterIds <- stats::cutree(stats::as.hclust(clusterSummary), k=cut) - - clusters <- list() - for (id in unique(clusterIds)) - { - if (sum(clusterIds==id) >= minNS) - clusters[[as.character(id)]] <- allPatterns[,clusterIds==id,drop=FALSE] - } - return(clusters) -} - -#' concatenate final results across subsets -#' @keywords internal -#' -#' @param result list of CogapsResult object from all runs across subsets -#' @param allParams list of all CoGAPS parameters -#' @param sets indices of sets used to break apart data -#' @return list with all CoGAPS output -stitchTogether <- function(result, allParams, sets) -{ - setIndices <- unlist(sets) - if (allParams$gaps@distributed == "genome-wide") - { - # combine A matrices, re-order so it matches original data - Amean <- do.call(rbind, lapply(result, function(x) x@featureLoadings)) - Asd <- do.call(rbind, lapply(result, function(x) x@loadingStdDev)) - - # copy P matrix - same for all sets - Pmean <- result[[1]]@metadata$params@fixedPatterns - Psd <- matrix(0, nrow=nrow(Pmean), ncol=ncol(Pmean)) - - # if each feature was used once, re-order to match data - if (nrow(Amean) == length(setIndices)) - { - indices <- 1:nrow(Amean) - if (identical(sort(indices), sort(setIndices))) - { - reorder <- match(indices, setIndices) - Amean <- Amean[reorder,] - Asd <- Asd[reorder,] - } - } - } - else - { - # combine P matrices, re-order so it matches original data - Pmean <- do.call(rbind, lapply(result, function(x) x@sampleFactors)) - Psd <- do.call(rbind, lapply(result, function(x) x@factorStdDev)) - - # copy A matrix - same for all sets - Amean <- result[[1]]@metadata$params@fixedPatterns - Asd <- matrix(0, nrow=nrow(Amean), ncol=ncol(Amean)) - - # if each sample was used once, re-order to match data - if (nrow(Pmean) == length(setIndices)) - { - indices <- 1:nrow(Pmean) - if (identical(sort(indices), sort(setIndices))) - { - reorder <- match(indices, setIndices) - Pmean <- Pmean[reorder,] - Psd <- Psd[reorder,] - } - } - } - - return(list("Amean"=Amean, "Asd"=Asd, "Pmean"=Pmean, "Psd"=Psd, - "seed"=allParams$gaps@seed, "geneNames"=rownames(Amean), - "sampleNames"=rownames(Pmean), - "meanChiSq"=sum(sapply(result, function(r) r@metadata$meanChiSq)))) -} +#' make correct call to internal CoGAPS dispatch function, CoGAPS could be +#' called directly, but to avoid any re-entrant behavior this function is called +#' instead. It is a light wrapper around cogaps_cpp that handles setting +#' the distributed parameters +#' @keywords internal +#' @param data data in a supported format +#' @param allParams list of all parameters +#' @param uncertainty uncertainty of data in the same format as data +#' @param subsetIndices indices of the subset of data to run on +#' @param workerID worker ID for parallelization +#' @return CogapsResult object +callInternalCoGAPS <- function(data, allParams, uncertainty, subsetIndices, +workerID) +{ + # identify which mode of parallelization + genomeWide <- allParams$gaps@distributed == "genome-wide" + allParams$gaps@distributed <- NULL + + # subset gene/sample names + if (genomeWide) + allParams$geneNames <- allParams$geneNames[subsetIndices] + else + allParams$sampleNames <- allParams$sampleNames[subsetIndices] + + allParams$gaps@subsetIndices <- subsetIndices + allParams$gaps@subsetDim <- ifelse(genomeWide, 1, 2) + allParams$workerID <- workerID + + # call CoGAPS + internal <- ifelse(is(data, "character"), cogaps_from_file_cpp, cogaps_cpp) + raw <- internal(data, allParams, uncertainty) + return(createCogapsResult(raw, allParams)) +} + +#' CoGAPS Distributed Matrix Factorization Algorithm +#' @keywords internal +#' +#' @description runs CoGAPS over subsets of the data and stitches the results +#' back together +#' @details For file types CoGAPS supports csv, tsv, and mtx +#' @param data File name or R object (see details for supported types) +#' @param allParams list of all parameters used in computation +#' @param uncertainty uncertainty matrix (same supported types as data) +#' @return list +#' @importFrom BiocParallel bplapply MulticoreParam +distributedCogaps <- function(data, allParams, uncertainty) +{ + # randomly sample either rows or columns into subsets to break the data up + set.seed(allParams$gaps@seed) + sets <- createSets(data, allParams) + if (min(sapply(sets, length)) < allParams$gaps@nPatterns) + stop("data subset dimension less than nPatterns") + + if (is.null(allParams$BPPARAM)) + allParams$BPPARAM <- BiocParallel::MulticoreParam(workers=length(sets)) + + initialResult <- NULL + if (is.null(allParams$gaps@fixedPatterns)) + { + # run Cogaps normally on each subset of the data + gapsCat(allParams, "Running Across Subsets...\n\n") + initialResult <- bplapply(1:length(sets), BPPARAM=allParams$BPPARAM, + FUN=function(i) + { + callInternalCoGAPS(data, allParams, uncertainty, sets[[i]], i) + }) + + # get all unmatched patterns + if (allParams$gaps@distributed == "genome-wide") + unmatchedPatterns <- lapply(initialResult, function(x) x@sampleFactors) + else + unmatchedPatterns <- lapply(initialResult, function(x) x@featureLoadings) + + # match patterns in either A or P matrix + gapsCat(allParams, "\nMatching Patterns Across Subsets...\n") + matchedPatterns <- findConsensusMatrix(unmatchedPatterns, allParams$gaps) + } + else + { + matchedPatterns <- list(consensus=allParams$gaps@fixedPatterns) + } + + # set fixed matrix + allParams$gaps@nPatterns <- ncol(matchedPatterns$consensus) + allParams$gaps@fixedPatterns <- matchedPatterns$consensus + allParams$gaps@whichMatrixFixed <- ifelse(allParams$gaps@distributed + == "genome-wide", "P", "A") + + # run final phase with fixed matrix + gapsCat(allParams, "Running Final Stage...\n\n") + finalResult <- bplapply(1:length(sets), BPPARAM=allParams$BPPARAM, + FUN=function(i) + { + callInternalCoGAPS(data, allParams, uncertainty, sets[[i]], i) + }) + + # concatenate final result + fullResult <- stitchTogether(finalResult, allParams, sets) + + # add diagnostic information about initial run before returning + if (!is.null(initialResult)) # check that initial phase was run + { + fullResult$diagnostics$firstPass <- initialResult + fullResult$diagnostics$unmatchedPatterns <- unmatchedPatterns + fullResult$diagnostics$clusteredPatterns <- matchedPatterns$clusteredPatterns + fullResult$diagnostics$CorrToMeanPattern <- lapply(matchedPatterns$clusteredPatterns, corrToMeanPattern) + } + + # include the subsets used + if (allParams$gaps@distributed == "genome-wide") + fullResult$diagnostics$subsets <- lapply(sets, function(s) fullResult$geneNames[s]) + else + fullResult$diagnostics$subsets <- lapply(sets, function(s) fullResult$sampleNames[s]) + + # return list, calling function will process this into a CogapsResult object + return(fullResult) +} + + +#' find the consensus pattern matrix across all subsets +#' @export +#' +#' @param unmatchedPatterns list of all unmatched pattern matrices from initial +#' run of CoGAPS +#' @param gapsParams list of all CoGAPS parameters +#' @return matrix of consensus patterns +findConsensusMatrix <- function(unmatchedPatterns, gapsParams) +{ + allPatterns <- do.call(cbind, unmatchedPatterns) + comb <- expand.grid(1:gapsParams@nSets, 1:gapsParams@nPatterns) + colnames(allPatterns) <- paste(comb[,1], comb[,2], sep=".") + return(patternMatch(allPatterns, gapsParams)) +} + +#' Match Patterns Across Multiple Runs +#' @keywords internal +#' +#' @param allPatterns matrix of patterns stored in the columns +#' @param gapsParams CoGAPS parameters object +#' @return a matrix of consensus patterns +#' @importFrom stats weighted.mean +patternMatch <- function(allPatterns, gapsParams) +{ + # cluster patterns + clusters <- corcut(allPatterns, gapsParams@cut, gapsParams@minNS) + + # function to split a cluster in two (might fail to do so) + splitCluster <- function(list, index, minNS) + { + split <- corcut(list[[index]], 2, minNS) + list[[index]] <- split[[1]] + if (length(split) > 1) + list <- append(list, split[2]) + return(list) + } + + # split large clusters into two + tooLarge <- function(x) ncol(x) > gapsParams@maxNS + indx <- which(sapply(clusters, tooLarge)) + while (length(indx) > 0) + { + clusters <- splitCluster(clusters, indx[1], gapsParams@minNS) + indx <- which(sapply(clusters, tooLarge)) + } + names(clusters) <- as.character(1:length(clusters)) + + # create matrix of mean patterns - weighted by correlation to mean pattern + meanPatterns <- sapply(clusters, function(clust) apply(clust, 1, + function(row) weighted.mean(row, corrToMeanPattern(clust)^3))) + colnames(meanPatterns) <- paste("Pattern", 1:length(clusters)) + + # returned patterns after scaling max to 1 + return(list("clusteredPatterns"=clusters, + "consensus"=apply(meanPatterns, 2, function(col) col / max(col)))) +} + +#' calculate correlation of each pattern in a cluster to the cluster mean +#' @keywords internal +#' @return correlation of each pattern +corrToMeanPattern <- function(cluster) +{ + meanPat <- rowMeans(cluster) + sapply(1:ncol(cluster), function(j) round(cor(x=cluster[,j], y=meanPat), 3)) +} + +#' cluster patterns together +#' @keywords internal +#' +#' @param allPatterns matrix of all patterns across subsets +#' @param cut number of branches at which to cut dendrogram +#' @param minNS minimum of individual set contributions a cluster must contain +#' @return patterns listed by which cluster they belong to +#' @importFrom cluster agnes +#' @importFrom stats cutree as.hclust cor +corcut <- function(allPatterns, cut, minNS) +{ + corr.dist <- cor(allPatterns) + corr.dist <- 1 - corr.dist + + if (any(is.na(corr.dist))) + { + stop("NA values in correlation of patterns") + } + + clusterSummary <- cluster::agnes(x=corr.dist, diss=TRUE, "complete") + clusterIds <- stats::cutree(stats::as.hclust(clusterSummary), k=cut) + + clusters <- list() + for (id in unique(clusterIds)) + { + if (sum(clusterIds==id) >= minNS) + clusters[[as.character(id)]] <- allPatterns[,clusterIds==id,drop=FALSE] + } + return(clusters) +} + +#' concatenate final results across subsets +#' @keywords internal +#' +#' @param result list of CogapsResult object from all runs across subsets +#' @param allParams list of all CoGAPS parameters +#' @param sets indices of sets used to break apart data +#' @return list with all CoGAPS output +stitchTogether <- function(result, allParams, sets) +{ + setIndices <- unlist(sets) + if (allParams$gaps@distributed == "genome-wide") + { + # combine A matrices, re-order so it matches original data + Amean <- do.call(rbind, lapply(result, function(x) x@featureLoadings)) + Asd <- do.call(rbind, lapply(result, function(x) x@loadingStdDev)) + + # copy P matrix - same for all sets + Pmean <- result[[1]]@metadata$params@fixedPatterns + Psd <- matrix(0, nrow=nrow(Pmean), ncol=ncol(Pmean)) + + # if each feature was used once, re-order to match data + if (nrow(Amean) == length(setIndices)) + { + indices <- 1:nrow(Amean) + if (identical(sort(indices), sort(setIndices))) + { + reorder <- match(indices, setIndices) + Amean <- Amean[reorder,] + Asd <- Asd[reorder,] + } + } + } + else + { + # combine P matrices, re-order so it matches original data + Pmean <- do.call(rbind, lapply(result, function(x) x@sampleFactors)) + Psd <- do.call(rbind, lapply(result, function(x) x@factorStdDev)) + + # copy A matrix - same for all sets + Amean <- result[[1]]@metadata$params@fixedPatterns + Asd <- matrix(0, nrow=nrow(Amean), ncol=ncol(Amean)) + + # if each sample was used once, re-order to match data + if (nrow(Pmean) == length(setIndices)) + { + indices <- 1:nrow(Pmean) + if (identical(sort(indices), sort(setIndices))) + { + reorder <- match(indices, setIndices) + Pmean <- Pmean[reorder,] + Psd <- Psd[reorder,] + } + } + } + + return(list("Amean"=Amean, "Asd"=Asd, "Pmean"=Pmean, "Psd"=Psd, + "seed"=allParams$gaps@seed, "geneNames"=rownames(Amean), + "sampleNames"=rownames(Pmean), + "meanChiSq"=sum(sapply(result, function(r) r@metadata$meanChiSq)))) +} diff --git a/R/HelperFunctions.R b/R/HelperFunctions.R index d433c5c5..adfa3f01 100755 --- a/R/HelperFunctions.R +++ b/R/HelperFunctions.R @@ -1,360 +1,354 @@ -#' get specified number of retina subsets -#' @export -#' -#' @description combines retina subsets from extdata directory -#' @param n number of subsets to use -#' @return matrix of RNA counts -#' @examples -#' retSubset <- getRetinaSubset() -#' dim(retSubset) -#' @importFrom rhdf5 h5read -getRetinaSubset <- function(n=1) -{ - if (!(n %in% 1:4)) - stop("invalid number of subsets requested") - - subset_1_path <- system.file("extdata/retina_subset_1.h5", package="CoGAPS") - subset_2_path <- system.file("extdata/retina_subset_2.h5", package="CoGAPS") - subset_3_path <- system.file("extdata/retina_subset_3.h5", package="CoGAPS") - subset_4_path <- system.file("extdata/retina_subset_4.h5", package="CoGAPS") - - data <- rhdf5::h5read(subset_1_path, "counts") - cNames <- rhdf5::h5read(subset_1_path, "cellNames") - if (n > 1) - { - data <- cbind(data, rhdf5::h5read(subset_2_path, "counts")) - cNames <- c(cNames, rhdf5::h5read(subset_2_path, "cellNames")) - } - if (n > 2) - { - data <- cbind(data, rhdf5::h5read(subset_3_path, "counts")) - cNames <- c(cNames, rhdf5::h5read(subset_3_path, "cellNames")) - } - if (n > 3) - { - data <- cbind(data, rhdf5::h5read(subset_4_path, "counts")) - cNames <- c(cNames, rhdf5::h5read(subset_4_path, "cellNames")) - } - - colnames(data) <- cNames - rownames(data) <- rhdf5::h5read(subset_1_path, "geneNames") - return(data) -} - -#' wrapper around cat -#' @keywords internal -#' -#' @description cleans up message printing -#' @param allParams all cogaps parameters -#' @param ... arguments forwarded to cat -#' @return conditionally print message -gapsCat <- function(allParams, ...) -{ - if (allParams$messages) - cat(...) -} - -#' checks if file is supported -#' @keywords internal -#' -#' @param file path to file -#' @return TRUE if file is supported, FALSE if not -#' @importFrom tools file_ext -supported <- function(file) -{ - if (!is(file, "character")) - return(FALSE) - return(tools::file_ext(file) %in% c("tsv", "csv", "mtx", "gct")) -} - -#' checks if file is rds format -#' @keywords internal -#' -#' @param file path to file -#' @return TRUE if file is .rds, FALSE if not -#' @importFrom tools file_ext -isRdsFile <- function(file) -{ - if (is.null(file)) - return(FALSE) - if (length(file) == 0) - return(FALSE) - if (!is(file, "character")) - return(FALSE) - return(tools::file_ext(file) == "rds") -} - -#' get input that might be an RDS file -#' @keywords internal -#' -#' @param input some user input -#' @return if input is an RDS file, read it - otherwise return input -getValueOrRds <- function(input) -{ - if (isRdsFile(input)) - return(readRDS(input)) - return(input) -} - -#' get number of rows from supported file name or matrix -#' @keywords internal -#' -#' @param data either a file name or a matrix -#' @return number of rows -#' @importFrom tools file_ext -nrowHelper <- function(data) -{ - if (is(data, "character")) - { - return(getFileInfo_cpp(data)[["dimensions"]][1]) - } - return(nrow(data)) -} - -#' get number of columns from supported file name or matrix -#' @keywords internal -#' -#' @param data either a file name or a matrix -#' @return number of columns -#' @importFrom tools file_ext -ncolHelper <- function(data) -{ - if (is(data, "character")) - { - return(getFileInfo_cpp(data)[["dimensions"]][2]) - } - return(ncol(data)) -} - -#' write start up message -#' @keywords internal -#' -#' @param data data set -#' @param allParams list of all parameters -#' @return message displayed to screen -#' @importFrom methods show -startupMessage <- function(data, allParams) -{ - nGenes <- ifelse(allParams$transposeData, ncolHelper(data), nrowHelper(data)) - nSamples <- ifelse(allParams$transposeData, nrowHelper(data), ncolHelper(data)) - - dist_message <- "Standard" - if (!is.null(allParams$gaps@distributed)) - dist_message <- allParams$gaps@distributed - - cat("\nThis is CoGAPS version", as.character(packageVersion("CoGAPS")), "\n") - cat("Running", dist_message, "CoGAPS on", allParams$dataName, - paste("(", nGenes, " genes and ", nSamples, " samples)", sep="")) - - if (allParams$messages) - { - cat(" with parameters:\n\n") - methods::show(allParams$gaps) - } - cat("\n") -} - -#' parse parameters passed through the ... variable -#' @keywords internal -#' -#' @param allParams list of all parameters -#' @param extraParams list of parameters in ... -#' @return allParams with any valid parameters in extraParams added -#' @note will halt with an error if any parameters in extraParams are invalid -#' @importFrom methods slotNames -parseExtraParams <- function(allParams, extraParams) -{ - # parse direct params - deprecatedSlots <- c("singleCell") - for (s in c(slotNames(allParams$gaps), deprecatedSlots)) - { - if (!is.null(extraParams[[s]])) - { - allParams$gaps <- setParam(allParams$gaps, s, extraParams[[s]]) - extraParams[[s]] <- NULL - } - } - - # check for unrecognized options - if (length(extraParams) > 0) - stop(paste("unrecognized argument:", names(extraParams)[1])) - - return(allParams) -} - -## TODO these checks should be in the C++ code so that file names are checked -## just as much as R variables -#' check that provided data is valid -#' @keywords internal -#' -#' @param data data matrix -#' @param uncertainty uncertainty matrix, can be null -#' @param params CogapsParams object -#' @return throws an error if data has problems -checkDataMatrix <- function(data, uncertainty, params) -{ - if (any(is.na(data))) - stop("NA values in data") - if (!all(apply(data, 2, is.numeric))) - stop("data is not numeric") - if (sum(data < 0) > 0 | sum(uncertainty < 0) > 0) - stop("negative values in data and/or uncertainty matrix") - if (nrow(data) <= params@nPatterns | ncol(data) <= params@nPatterns) - stop("nPatterns must be less than dimensions of data") - if (sum(uncertainty < 1e-5) > 0) - warning("small values in uncertainty matrix detected") -} - -#' check that all inputs are valid -#' @keywords internal -#' -#' @param data data matrix -#' @param uncertainty uncertainty matrix, can be null -#' @param allParams list of all parameters -#' @return throws an error if inputs are invalid -checkInputs <- function(data, uncertainty, allParams) -{ - if (is(data, "character") & !is.null(uncertainty) & !is(uncertainty, "character")) - stop("uncertainty must be same data type as data (file name)") - if (is(uncertainty, "character") & !supported(uncertainty)) - stop("unsupported file extension for uncertainty") - if (!is(data, "character") & !is.null(uncertainty) & !is(uncertainty, "matrix")) - stop("uncertainty must be a matrix unless data is a file path") - if (!is.null(uncertainty) & allParams$gaps@sparseOptimization) - stop("must use default uncertainty when enabling sparseOptimization") - if (!is.null(allParams$checkpointInFile) & !CoGAPS::checkpointsEnabled()) - stop("CoGAPS was built with checkpoints disabled") - if (!(allParams$snapshotPhase %in% c('equilibration', 'sampling', 'all'))) - stop("snapshotPhase must be either equilibration, sampling, or all") - if (allParams$nSnapshots > 0) - warning("Snapshots slow down computation and should only be used for testing") - - - if (!is.null(allParams$gaps@distributed)) - { - if (allParams$asynchronousUpdates | allParams$nThreads > 1) - warning(paste( - "Distributed CoGAPS parallelizes across data subsets and does", - "not use OpenMP asynchronous updates within each worker;", - "running workers with asynchronousUpdates=FALSE and nThreads=1." - )) - if (!is.null(allParams$checkpointInFile)) - stop("checkpoints not supported for distributed cogaps") - if (!is(data, "character")) - warning("running distributed cogaps without mtx/tsv/csv/gct data") - } - - if (!is(data, "character")) - checkDataMatrix(data, uncertainty, allParams$gaps) - if (is.null(allParams$geneNames)) - stop("no gene names in parameters") - if (is.null(allParams$sampleNames)) - stop("no sample names in parameters") -} - -#' extract gene names from data -#' @keywords internal -#' @return vector of gene names -getGeneNames <- function(data, transpose) -{ - if (transpose) - return(getSampleNames(data, FALSE)) - if (is(data, "character")) - names <- getFileInfo_cpp(data)[["rowNames"]] - else - names <- rownames(data) - if (is.null(names) | length(names) == 0) - return(paste("Gene", 1:nrowHelper(data), sep="_")) - return(names) -} - -#' extract sample names from data -#' @keywords internal -#' @return vector of sample names -getSampleNames <- function(data, transpose) -{ - if (transpose) - return(getGeneNames(data, FALSE)) - if (is(data, "character")) - names <- getFileInfo_cpp(data)[["colNames"]] - else - names <- colnames(data) - if (is.null(names) | length(names) == 0) - return(paste("Sample", 1:ncolHelper(data), sep="_")) - return(names) -} - -#' extracts gene/sample names from the data -#' @keywords internal -#' -#' @param data data matrix -#' @param allParams list of all parameters -#' @return list of all parameters with added gene names -getDimNames <- function(data, allParams) -{ - # get user supplied names - geneNames <- allParams$gaps@geneNames - sampleNames <- allParams$gaps@sampleNames - - # if user didn't supply any names, pull from data set or use default labels - if (is.null(allParams$gaps@geneNames)) - geneNames <- getGeneNames(data, allParams$transposeData) - if (is.null(allParams$gaps@sampleNames)) - sampleNames <- getSampleNames(data, allParams$transposeData) - - # get the number of genes/samples - nGenes <- ifelse(allParams$transposeData, ncolHelper(data), nrowHelper(data)) - nSamples <- ifelse(allParams$transposeData, nrowHelper(data), ncolHelper(data)) - - # handle any subsetting - if (allParams$gaps@subsetDim == 1) - { - nGenes <- length(allParams$gaps@subsetIndices) - geneNames <- geneNames[allParams$gaps@subsetIndices] - } - else if (allParams$gaps@subsetDim == 2) - { - nSamples <- length(allParams$gaps@subsetIndices) - sampleNames <- sampleNames[allParams$gaps@subsetIndices] - } - - # check that names align with expected number of genes/samples - if (length(geneNames) != nGenes) - stop(length(geneNames), " != ", nGenes, " incorrect number of gene names given") - if (length(sampleNames) != nSamples) - stop(length(sampleNames), " != ", nSamples, " incorrect number of sample names given") - - # store processed gene/sample names directly in allParams list - # this is an important distinction - allParams@gaps contains the - # gene/sample names originally passed by the user, allParams contains - # the procseed gene/sample names to be used when labeling the result - allParams$geneNames <- geneNames - allParams$sampleNames <- sampleNames - return(allParams) -} - -#' convert any acceptable data input to a numeric matrix -#' @keywords internal -#' -#' @description convert supported R objects containing the data to a -#' numeric matrix, if data is a file name do nothing. Exits with an error -#' if data is not a supported type. -#' @param data data input -#' @return data matrix -#' @importFrom methods is -#' @importFrom SummarizedExperiment assay -convertDataToMatrix <- function(data) -{ - if (is(data, "character") & !supported(data)) - stop("unsupported file extension for data") - else if (is(data, "matrix") | is(data, "character")) - return(data) - else if (is(data, "data.frame")) - return(data.matrix(data)) - else if (is(data, "SummarizedExperiment")) - return(SummarizedExperiment::assay(data, "counts")) - else if (is(data, "SingleCellExperiment")) - return(SummarizedExperiment::assay(data, "counts")) - else - stop("unsupported data type") +#' get specified number of retina subsets +#' @export +#' +#' @description combines retina subsets from extdata directory +#' @param n number of subsets to use +#' @return matrix of RNA counts +#' @examples +#' retSubset <- getRetinaSubset() +#' dim(retSubset) +#' @importFrom rhdf5 h5read +getRetinaSubset <- function(n=1) +{ + if (!(n %in% 1:4)) + stop("invalid number of subsets requested") + + subset_1_path <- system.file("extdata/retina_subset_1.h5", package="CoGAPS") + subset_2_path <- system.file("extdata/retina_subset_2.h5", package="CoGAPS") + subset_3_path <- system.file("extdata/retina_subset_3.h5", package="CoGAPS") + subset_4_path <- system.file("extdata/retina_subset_4.h5", package="CoGAPS") + + data <- rhdf5::h5read(subset_1_path, "counts") + cNames <- rhdf5::h5read(subset_1_path, "cellNames") + if (n > 1) + { + data <- cbind(data, rhdf5::h5read(subset_2_path, "counts")) + cNames <- c(cNames, rhdf5::h5read(subset_2_path, "cellNames")) + } + if (n > 2) + { + data <- cbind(data, rhdf5::h5read(subset_3_path, "counts")) + cNames <- c(cNames, rhdf5::h5read(subset_3_path, "cellNames")) + } + if (n > 3) + { + data <- cbind(data, rhdf5::h5read(subset_4_path, "counts")) + cNames <- c(cNames, rhdf5::h5read(subset_4_path, "cellNames")) + } + + colnames(data) <- cNames + rownames(data) <- rhdf5::h5read(subset_1_path, "geneNames") + return(data) +} + +#' wrapper around cat +#' @keywords internal +#' +#' @description cleans up message printing +#' @param allParams all cogaps parameters +#' @param ... arguments forwarded to cat +#' @return conditionally print message +gapsCat <- function(allParams, ...) +{ + if (allParams$messages) + cat(...) +} + +#' checks if file is supported +#' @keywords internal +#' +#' @param file path to file +#' @return TRUE if file is supported, FALSE if not +#' @importFrom tools file_ext +supported <- function(file) +{ + if (!is(file, "character")) + return(FALSE) + return(tools::file_ext(file) %in% c("tsv", "csv", "mtx", "gct")) +} + +#' checks if file is rds format +#' @keywords internal +#' +#' @param file path to file +#' @return TRUE if file is .rds, FALSE if not +#' @importFrom tools file_ext +isRdsFile <- function(file) +{ + if (is.null(file)) + return(FALSE) + if (length(file) == 0) + return(FALSE) + if (!is(file, "character")) + return(FALSE) + return(tools::file_ext(file) == "rds") +} + +#' get input that might be an RDS file +#' @keywords internal +#' +#' @param input some user input +#' @return if input is an RDS file, read it - otherwise return input +getValueOrRds <- function(input) +{ + if (isRdsFile(input)) + return(readRDS(input)) + return(input) +} + +#' get number of rows from supported file name or matrix +#' @keywords internal +#' +#' @param data either a file name or a matrix +#' @return number of rows +#' @importFrom tools file_ext +nrowHelper <- function(data) +{ + if (is(data, "character")) + { + return(getFileInfo_cpp(data)[["dimensions"]][1]) + } + return(nrow(data)) +} + +#' get number of columns from supported file name or matrix +#' @keywords internal +#' +#' @param data either a file name or a matrix +#' @return number of columns +#' @importFrom tools file_ext +ncolHelper <- function(data) +{ + if (is(data, "character")) + { + return(getFileInfo_cpp(data)[["dimensions"]][2]) + } + return(ncol(data)) +} + +#' write start up message +#' @keywords internal +#' +#' @param data data set +#' @param allParams list of all parameters +#' @return message displayed to screen +#' @importFrom methods show +startupMessage <- function(data, allParams) +{ + nGenes <- ifelse(allParams$transposeData, ncolHelper(data), nrowHelper(data)) + nSamples <- ifelse(allParams$transposeData, nrowHelper(data), ncolHelper(data)) + + dist_message <- "Standard" + if (!is.null(allParams$gaps@distributed)) + dist_message <- allParams$gaps@distributed + + cat("\nThis is CoGAPS version", as.character(packageVersion("CoGAPS")), "\n") + cat("Running", dist_message, "CoGAPS on", allParams$dataName, + paste("(", nGenes, " genes and ", nSamples, " samples)", sep="")) + + if (allParams$messages) + { + cat(" with parameters:\n\n") + methods::show(allParams$gaps) + } + cat("\n") +} + +#' parse parameters passed through the ... variable +#' @keywords internal +#' +#' @param allParams list of all parameters +#' @param extraParams list of parameters in ... +#' @return allParams with any valid parameters in extraParams added +#' @note will halt with an error if any parameters in extraParams are invalid +#' @importFrom methods slotNames +parseExtraParams <- function(allParams, extraParams) +{ + # parse direct params + deprecatedSlots <- c("singleCell") + for (s in c(slotNames(allParams$gaps), deprecatedSlots)) + { + if (!is.null(extraParams[[s]])) + { + allParams$gaps <- setParam(allParams$gaps, s, extraParams[[s]]) + extraParams[[s]] <- NULL + } + } + + # check for unrecognized options + if (length(extraParams) > 0) + stop(paste("unrecognized argument:", names(extraParams)[1])) + + return(allParams) +} + +## TODO these checks should be in the C++ code so that file names are checked +## just as much as R variables +#' check that provided data is valid +#' @keywords internal +#' +#' @param data data matrix +#' @param uncertainty uncertainty matrix, can be null +#' @param params CogapsParams object +#' @return throws an error if data has problems +checkDataMatrix <- function(data, uncertainty, params) +{ + if (any(is.na(data))) + stop("NA values in data") + if (!all(apply(data, 2, is.numeric))) + stop("data is not numeric") + if (sum(data < 0) > 0 | sum(uncertainty < 0) > 0) + stop("negative values in data and/or uncertainty matrix") + if (nrow(data) <= params@nPatterns | ncol(data) <= params@nPatterns) + stop("nPatterns must be less than dimensions of data") + if (sum(uncertainty < 1e-5) > 0) + warning("small values in uncertainty matrix detected") +} + +#' check that all inputs are valid +#' @keywords internal +#' +#' @param data data matrix +#' @param uncertainty uncertainty matrix, can be null +#' @param allParams list of all parameters +#' @return throws an error if inputs are invalid +checkInputs <- function(data, uncertainty, allParams) +{ + if (is(data, "character") & !is.null(uncertainty) & !is(uncertainty, "character")) + stop("uncertainty must be same data type as data (file name)") + if (is(uncertainty, "character") & !supported(uncertainty)) + stop("unsupported file extension for uncertainty") + if (!is(data, "character") & !is.null(uncertainty) & !is(uncertainty, "matrix")) + stop("uncertainty must be a matrix unless data is a file path") + if (!is.null(uncertainty) & allParams$gaps@sparseOptimization) + stop("must use default uncertainty when enabling sparseOptimization") + if (!is.null(allParams$checkpointInFile) & !CoGAPS::checkpointsEnabled()) + stop("CoGAPS was built with checkpoints disabled") + if (!(allParams$snapshotPhase %in% c('equilibration', 'sampling', 'all'))) + stop("snapshotPhase must be either equilibration, sampling, or all") + if (allParams$nSnapshots > 0) + warning("Snapshots slow down computation and should only be used for testing") + + + if (!is.null(allParams$gaps@distributed)) + { + if (!is.null(allParams$checkpointInFile)) + stop("checkpoints not supported for distributed cogaps") + if (!is(data, "character")) + warning("running distributed cogaps without mtx/tsv/csv/gct data") + } + + if (!is(data, "character")) + checkDataMatrix(data, uncertainty, allParams$gaps) + if (is.null(allParams$geneNames)) + stop("no gene names in parameters") + if (is.null(allParams$sampleNames)) + stop("no sample names in parameters") +} + +#' extract gene names from data +#' @keywords internal +#' @return vector of gene names +getGeneNames <- function(data, transpose) +{ + if (transpose) + return(getSampleNames(data, FALSE)) + if (is(data, "character")) + names <- getFileInfo_cpp(data)[["rowNames"]] + else + names <- rownames(data) + if (is.null(names) | length(names) == 0) + return(paste("Gene", 1:nrowHelper(data), sep="_")) + return(names) +} + +#' extract sample names from data +#' @keywords internal +#' @return vector of sample names +getSampleNames <- function(data, transpose) +{ + if (transpose) + return(getGeneNames(data, FALSE)) + if (is(data, "character")) + names <- getFileInfo_cpp(data)[["colNames"]] + else + names <- colnames(data) + if (is.null(names) | length(names) == 0) + return(paste("Sample", 1:ncolHelper(data), sep="_")) + return(names) +} + +#' extracts gene/sample names from the data +#' @keywords internal +#' +#' @param data data matrix +#' @param allParams list of all parameters +#' @return list of all parameters with added gene names +getDimNames <- function(data, allParams) +{ + # get user supplied names + geneNames <- allParams$gaps@geneNames + sampleNames <- allParams$gaps@sampleNames + + # if user didn't supply any names, pull from data set or use default labels + if (is.null(allParams$gaps@geneNames)) + geneNames <- getGeneNames(data, allParams$transposeData) + if (is.null(allParams$gaps@sampleNames)) + sampleNames <- getSampleNames(data, allParams$transposeData) + + # get the number of genes/samples + nGenes <- ifelse(allParams$transposeData, ncolHelper(data), nrowHelper(data)) + nSamples <- ifelse(allParams$transposeData, nrowHelper(data), ncolHelper(data)) + + # handle any subsetting + if (allParams$gaps@subsetDim == 1) + { + nGenes <- length(allParams$gaps@subsetIndices) + geneNames <- geneNames[allParams$gaps@subsetIndices] + } + else if (allParams$gaps@subsetDim == 2) + { + nSamples <- length(allParams$gaps@subsetIndices) + sampleNames <- sampleNames[allParams$gaps@subsetIndices] + } + + # check that names align with expected number of genes/samples + if (length(geneNames) != nGenes) + stop(length(geneNames), " != ", nGenes, " incorrect number of gene names given") + if (length(sampleNames) != nSamples) + stop(length(sampleNames), " != ", nSamples, " incorrect number of sample names given") + + # store processed gene/sample names directly in allParams list + # this is an important distinction - allParams@gaps contains the + # gene/sample names originally passed by the user, allParams contains + # the procseed gene/sample names to be used when labeling the result + allParams$geneNames <- geneNames + allParams$sampleNames <- sampleNames + return(allParams) +} + +#' convert any acceptable data input to a numeric matrix +#' @keywords internal +#' +#' @description convert supported R objects containing the data to a +#' numeric matrix, if data is a file name do nothing. Exits with an error +#' if data is not a supported type. +#' @param data data input +#' @return data matrix +#' @importFrom methods is +#' @importFrom SummarizedExperiment assay +convertDataToMatrix <- function(data) +{ + if (is(data, "character") & !supported(data)) + stop("unsupported file extension for data") + else if (is(data, "matrix") | is(data, "character")) + return(data) + else if (is(data, "data.frame")) + return(data.matrix(data)) + else if (is(data, "SummarizedExperiment")) + return(SummarizedExperiment::assay(data, "counts")) + else if (is(data, "SingleCellExperiment")) + return(SummarizedExperiment::assay(data, "counts")) + else + stop("unsupported data type") } \ No newline at end of file diff --git a/R/Package.R b/R/Package.R index 7584ec08..cf6f5f58 100755 --- a/R/Package.R +++ b/R/Package.R @@ -43,4 +43,9 @@ NULL #' CoGAPS result from running on GIST dataset #' @docType data #' @name GIST.result -NULL \ No newline at end of file +NULL + +# Column names used non-standardly inside dplyr/ggplot2 pipelines in +# getPatternGeneSet() and plotPatternGeneSet(); declaring them keeps +# "no visible binding for global variable" out of R CMD check. +utils::globalVariables(c("gene.set", "padj", "neg.log.padj")) diff --git a/R/RcppExports.R b/R/RcppExports.R index 4b4624d7..780c64f0 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -17,19 +17,19 @@ checkpointsEnabled_cpp <- function() { .Call('_CoGAPS_checkpointsEnabled_cpp', PACKAGE = 'CoGAPS') } -compiledWithOpenMPSupport_cpp <- function() { - .Call('_CoGAPS_compiledWithOpenMPSupport_cpp', PACKAGE = 'CoGAPS') -} - getFileInfo_cpp <- function(path) { .Call('_CoGAPS_getFileInfo_cpp', PACKAGE = 'CoGAPS', path) } -run_catch_unit_tests <- function(reporter = "console") { - .Call('_CoGAPS_run_catch_unit_tests', PACKAGE = 'CoGAPS', reporter) +run_catch_unit_tests <- function(reporter = "console", output = "") { + .Call('_CoGAPS_run_catch_unit_tests', PACKAGE = 'CoGAPS', reporter, output) +} + +run_catch_unit_tests_by_tag <- function(tag = "", reporter = "console", output = "") { + .Call('_CoGAPS_run_catch_unit_tests_by_tag', PACKAGE = 'CoGAPS', tag, reporter, output) } -run_catch_unit_tests_by_tag <- function(tag = "", reporter = "console") { - .Call('_CoGAPS_run_catch_unit_tests_by_tag', PACKAGE = 'CoGAPS', tag, reporter) +catch_test_case_names <- function() { + .Call('_CoGAPS_catch_test_case_names', PACKAGE = 'CoGAPS') } diff --git a/R/SubsetData.R b/R/SubsetData.R index bc216db8..650b888e 100755 --- a/R/SubsetData.R +++ b/R/SubsetData.R @@ -1,116 +1,115 @@ -#' use user provided subsets -#' @keywords internal -#' -#' @param allParams list of all CoGAPS parameters -#' @param total total number of rows (cols) that are being paritioned -#' @return list of subsets -sampleWithExplictSets <- function(allParams) -{ - if (all(sapply(allParams$gaps@explicitSets, function(s) is(s, "numeric")))) - { - gapsCat(allParams, "using provided indexed subsets\n") - return(allParams$gaps@explicitSets) - } - else if (all(sapply(allParams$gaps@explicitSets, function(s) is(s, "character")))) - { - gapsCat(allParams, "using provided named subsets\n") - if (allParams$gaps@distributed == "genome-wide") - allNames <- allParams$geneNames - else - allNames <- allParams$sampleNames - return(lapply(allParams$gaps@explicitSets, function(set) - { - if (any(!(set %in% allNames))) - stop("some named genes in explicitSets not found") - return(which(allNames %in% set)) - })) - } -} - -#' subset rows (cols) proportional to the user provided weights -#' @keywords internal -#' -#' @param allParams list of all CoGAPS parameters -#' @param setSize the size of each subset of the total -#' @return list of subsets -sampleWithAnnotationWeights <- function(allParams, setSize) -{ - # sort annotation group and weights so they match up - weight <- allParams$gaps@samplingWeight - weight <- weight[order(names(weight))] - groups <- unique(allParams$gaps@samplingAnnotation) - groups <- sort(groups) - - # sample accordingly - return(lapply(1:allParams$gaps@nSets, function(i) - { - groupCount <- sample(groups, size=setSize, replace=TRUE, prob=weight) - sort(unlist(sapply(groups, function(g) - { - groupNdx <- which(allParams$gaps@samplingAnnotation == g) - sample(groupNdx, size=sum(groupCount == g), replace=TRUE) - }))) - })) -} - -#' subset data by uniformly partioning rows (cols) -#' @keywords internal -#' -#' @param allParams list of all CoGAPS parameters -#' @param total total number of rows (cols) that are being paritioned -#' @param setSize the size of each subset of the total -#' @return list of subsets -sampleUniformly <- function(allParams, total, setSize) -{ - sets <- list() - remaining <- 1:total - for (n in 1:(allParams$gaps@nSets - 1)) - { - selected <- sample(remaining, setSize, replace=FALSE) - sets[[n]] <- sort(selected) - remaining <- setdiff(remaining, selected) - } - sets[[allParams$gaps@nSets]] <- sort(remaining) - return(sets) -} - -#' partition genes/samples into subsets -#' @keywords internal -#' -#' @description either genes or samples or partitioned depending on the type -#' of distributed CoGAPS (i.e. genome-wide or single-cell) -#' @param data either file name or matrix -#' @param allParams list of all CoGAPS parameters -#' @return list of sorted subsets of either genes or samples -createSets <- function(data, allParams) -{ - subsetRows <- xor(allParams$transposeData, - allParams$gaps@distributed == "genome-wide") - total <- ifelse(subsetRows, nrowHelper(data), ncolHelper(data)) - setSize <- floor(total / allParams$gaps@nSets) - - gapsCat(allParams, "Creating subsets...") - - if (!is.null(allParams$gaps@explicitSets)) - { - if (length(allParams$gaps@explicitSets) != allParams$gaps@nSets) - stop("nSets does not match number of explicit sets given") - sets <- sampleWithExplictSets(allParams) - } - else if (!is.null(allParams$gaps@samplingAnnotation)) - { - gapsCat(allParams, "sampling with annotation weights\n") - sets <- sampleWithAnnotationWeights(allParams, setSize) - } - else - { - gapsCat(allParams, "\n") - sets <- sampleUniformly(allParams, total, setSize) - } - - gapsCat(allParams, "set sizes (min, mean, max): (", - min(sapply(sets, length)), ", ", - mean(sapply(sets, length)), ", ", - max(sapply(sets, length)), ")\n", sep="") - return(sets) -} +#' use user provided subsets +#' @keywords internal +#' +#' @param allParams list of all CoGAPS parameters +#' @return list of subsets +sampleWithExplictSets <- function(allParams) +{ + if (all(sapply(allParams$gaps@explicitSets, function(s) is(s, "numeric")))) + { + gapsCat(allParams, "using provided indexed subsets\n") + return(allParams$gaps@explicitSets) + } + else if (all(sapply(allParams$gaps@explicitSets, function(s) is(s, "character")))) + { + gapsCat(allParams, "using provided named subsets\n") + if (allParams$gaps@distributed == "genome-wide") + allNames <- allParams$geneNames + else + allNames <- allParams$sampleNames + return(lapply(allParams$gaps@explicitSets, function(set) + { + if (any(!(set %in% allNames))) + stop("some named genes in explicitSets not found") + return(which(allNames %in% set)) + })) + } +} + +#' subset rows (cols) proportional to the user provided weights +#' @keywords internal +#' +#' @param allParams list of all CoGAPS parameters +#' @param setSize the size of each subset of the total +#' @return list of subsets +sampleWithAnnotationWeights <- function(allParams, setSize) +{ + # sort annotation group and weights so they match up + weight <- allParams$gaps@samplingWeight + weight <- weight[order(names(weight))] + groups <- unique(allParams$gaps@samplingAnnotation) + groups <- sort(groups) + + # sample accordingly + return(lapply(1:allParams$gaps@nSets, function(i) + { + groupCount <- sample(groups, size=setSize, replace=TRUE, prob=weight) + sort(unlist(sapply(groups, function(g) + { + groupNdx <- which(allParams$gaps@samplingAnnotation == g) + sample(groupNdx, size=sum(groupCount == g), replace=TRUE) + }))) + })) +} + +#' subset data by uniformly partioning rows (cols) +#' @keywords internal +#' +#' @param allParams list of all CoGAPS parameters +#' @param total total number of rows (cols) that are being paritioned +#' @param setSize the size of each subset of the total +#' @return list of subsets +sampleUniformly <- function(allParams, total, setSize) +{ + sets <- list() + remaining <- 1:total + for (n in 1:(allParams$gaps@nSets - 1)) + { + selected <- sample(remaining, setSize, replace=FALSE) + sets[[n]] <- sort(selected) + remaining <- setdiff(remaining, selected) + } + sets[[allParams$gaps@nSets]] <- sort(remaining) + return(sets) +} + +#' partition genes/samples into subsets +#' @keywords internal +#' +#' @description either genes or samples or partitioned depending on the type +#' of distributed CoGAPS (i.e. genome-wide or single-cell) +#' @param data either file name or matrix +#' @param allParams list of all CoGAPS parameters +#' @return list of sorted subsets of either genes or samples +createSets <- function(data, allParams) +{ + subsetRows <- xor(allParams$transposeData, + allParams$gaps@distributed == "genome-wide") + total <- ifelse(subsetRows, nrowHelper(data), ncolHelper(data)) + setSize <- floor(total / allParams$gaps@nSets) + + gapsCat(allParams, "Creating subsets...") + + if (!is.null(allParams$gaps@explicitSets)) + { + if (length(allParams$gaps@explicitSets) != allParams$gaps@nSets) + stop("nSets does not match number of explicit sets given") + sets <- sampleWithExplictSets(allParams) + } + else if (!is.null(allParams$gaps@samplingAnnotation)) + { + gapsCat(allParams, "sampling with annotation weights\n") + sets <- sampleWithAnnotationWeights(allParams, setSize) + } + else + { + gapsCat(allParams, "\n") + sets <- sampleUniformly(allParams, total, setSize) + } + + gapsCat(allParams, "set sizes (min, mean, max): (", + min(sapply(sets, length)), ", ", + mean(sapply(sets, length)), ", ", + max(sapply(sets, length)), ")\n", sep="") + return(sets) +} diff --git a/R/class-CogapsResult.R b/R/class-CogapsResult.R index e9299a75..2ce6ca89 100755 --- a/R/class-CogapsResult.R +++ b/R/class-CogapsResult.R @@ -161,8 +161,8 @@ setGeneric("getMeanChiSq", function(object) #' @aliases getPatternGeneSet #' @param object an object of type CogapsResult #' @param gene.sets a list of gene sets to test. List names should be the names of the gene sets -#' @param method enrichment or overrepresentation. Conducts a test for gene set enrichment using {fgsea::gsea} ranking features by pattern amplitude or a test for gene set overrepresentation in pattern markers using {fgsea::fora}, respectively. -#' @param ... additional parameters passed to {patternMarkers} if using overrepresentation method +#' @param method enrichment or overrepresentation. Conducts a test for gene set enrichment using \code{fgsea::gsea} ranking features by pattern amplitude or a test for gene set overrepresentation in pattern markers using \code{fgsea::fora}, respectively. +#' @param ... additional parameters passed to \code{patternMarkers} if using overrepresentation method #' @return list of dataframes containing gene set enrichment or gene set overrepresentation statistics #' @examples #' data(GIST) diff --git a/R/methods-CogapsParams.R b/R/methods-CogapsParams.R index a3478956..c543b2cd 100755 --- a/R/methods-CogapsParams.R +++ b/R/methods-CogapsParams.R @@ -80,7 +80,7 @@ function(object) cat("Warning!! Setting checkpointInterval=0 disables checkpoint logging.", "\n") } if(!is.null(object@checkpointInFile)){ - cat("checkpointInFile ", checkpointInFile, "\n") + cat("checkpointInFile ", object@checkpointInFile, "\n") } if (!is.null(object@checkpointOutFile)){ cat("checkpointOutFile ", object@checkpointOutFile, "\n") diff --git a/R/methods-CogapsResult.R b/R/methods-CogapsResult.R index f666cff1..f2ae037b 100755 --- a/R/methods-CogapsResult.R +++ b/R/methods-CogapsResult.R @@ -251,7 +251,9 @@ function(object, genes) setMethod("binaryA", signature(object="CogapsResult"), function(object, threshold) { - binA <- ifelse(calcZ(object) > threshold, 1, 0) + # binaryA thresholds the z-scores of the A (amplitude) matrix; calcZ has no + # default for whichMatrix, so omitting it made every call fail + binA <- ifelse(calcZ(object, "featureLoadings") > threshold, 1, 0) gplots::heatmap.2(binA, Rowv = FALSE, Colv = FALSE, dendrogram="none", scale="none", col = brewer.pal(3,"Blues"), trace="none", @@ -465,6 +467,7 @@ function(object, threshold, lp, axis){ }) #' @noRd +#' @importFrom stats setNames .patternMarkers_all <- function(ssranks) { pIndx<-apply(ssranks,1,which.min) pNames<-setNames(seq_along(colnames(ssranks)), colnames(ssranks)) @@ -647,11 +650,14 @@ function(object, save_location) setMethod("fromCSV", signature(save_location="character"), function(save_location) { - featureLoadings <- read.csv(file = paste0(save_location, "/featureLoadings.csv")) - sampleFactors <- read.csv(file = paste0(save_location, "/sampleFactors.csv")) - - loadingStdDev <- read.csv(file = paste0(save_location, "/loadingStdDev.csv")) - factorStdDev <- read.csv(file = paste0(save_location, "/factorStdDev.csv")) + # as.matrix: read.csv returns a data.frame, but these become the matrix slots + # of a LinearEmbeddingMatrix, so a round-trip through CSV has to give back + # matrices rather than data.frames + featureLoadings <- as.matrix(read.csv(file = paste0(save_location, "/featureLoadings.csv"))) + sampleFactors <- as.matrix(read.csv(file = paste0(save_location, "/sampleFactors.csv"))) + + loadingStdDev <- as.matrix(read.csv(file = paste0(save_location, "/loadingStdDev.csv"))) + factorStdDev <- as.matrix(read.csv(file = paste0(save_location, "/factorStdDev.csv"))) geneNames <- read.csv(file=paste0(save_location, "/geneNames.csv"))$x sampleNames <- read.csv(file = paste0(save_location, "/sampleNames.csv"))$x diff --git a/configure b/configure index 3db9f66a..4048b73a 100755 --- a/configure +++ b/configure @@ -1,9 +1,9 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.72 for CoGAPS 3.25.1. +# Generated by GNU Autoconf 2.73 for CoGAPS 3.33.2. # # -# Copyright (C) 1992-1996, 1998-2017, 2020-2023 Free Software Foundation, +# Copyright (C) 1992-1996, 1998-2017, 2020-2026 Free Software Foundation, # Inc. # # @@ -20,7 +20,7 @@ then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. + # contradicts POSIX and common usage. Disable this. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case e in #( @@ -107,7 +107,7 @@ if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then - printf "%s\n" "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + printf '%s\n' "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi @@ -129,10 +129,13 @@ case $- in # (((( *x* ) as_opts=-x ;; * ) as_opts= ;; esac -exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} +case $# in # (( + 0) exec $CONFIG_SHELL $as_opts "$as_myself" ;; + *) exec $CONFIG_SHELL $as_opts "$as_myself" "$@" ;; +esac # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed 'exec'. -printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 +printf '%s\n' "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi # We don't want this to propagate to other subprocesses. @@ -143,7 +146,7 @@ then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which - # is contrary to our usage. Disable this feature. + # contradicts POSIX and common usage. Disable this. alias -g '\${1+\"\$@\"}'='\"\$@\"' setopt NO_GLOB_SUBST else case e in #( @@ -181,7 +184,8 @@ test -x / || exit 1" as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && - test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1" + test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 +test \$(( 1 + 1 )) = 2 || exit 1" if (eval "$as_required") 2>/dev/null then : as_have_required=yes @@ -252,22 +256,25 @@ case $- in # (((( *x* ) as_opts=-x ;; * ) as_opts= ;; esac -exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} +case $# in # (( + 0) exec $CONFIG_SHELL $as_opts "$as_myself" ;; + *) exec $CONFIG_SHELL $as_opts "$as_myself" "$@" ;; +esac # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed 'exec'. -printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2 +printf '%s\n' "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi if test x$as_have_required = xno then : - printf "%s\n" "$0: This script requires a shell more modern than all" - printf "%s\n" "$0: the shells that I found on your system." + printf '%s\n' "$0: This script requires a shell more modern than all" + printf '%s\n' "$0: the shells that I found on your system." if test ${ZSH_VERSION+y} ; then - printf "%s\n" "$0: In particular, zsh $ZSH_VERSION has bugs and should" - printf "%s\n" "$0: be upgraded to zsh 4.3.4 or later." + printf '%s\n' "$0: In particular, zsh $ZSH_VERSION has bugs and should" + printf '%s\n' "$0: be upgraded to zsh 4.3.4 or later." else - printf "%s\n" "$0: Please tell bug-autoconf@gnu.org about your system, + printf '%s\n' "$0: Please tell bug-autoconf@gnu.org about your system, $0: including any error possibly output before this $0: message. Then install a modern shell, or manually run $0: the script under such a shell if you do have one." @@ -327,7 +334,7 @@ as_fn_mkdir_p () as_dirs= while :; do case $as_dir in #( - *\'*) as_qdir=`printf "%s\n" "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *\'*) as_qdir=`printf '%s\n' "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" @@ -336,7 +343,7 @@ $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X"$as_dir" | +printf '%s\n' X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q @@ -419,9 +426,9 @@ as_fn_error () as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - printf "%s\n" "$as_me: error: $2" >&2 + printf '%s\n' "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -448,7 +455,7 @@ as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X/"$0" | +printf '%s\n' X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q @@ -494,7 +501,7 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits s/-\n.*// ' >$as_me.lineno && chmod +x "$as_me.lineno" || - { printf "%s\n" "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } + { printf '%s\n' "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } # If we had to re-execute with $CONFIG_SHELL, we're ensured to have # already done that, so ensure we don't try to do so again and fall @@ -508,29 +515,6 @@ as_cr_alnum=$as_cr_Letters$as_cr_digits exit } - -# Determine whether it's possible to make 'echo' print without a newline. -# These variables are no longer used directly by Autoconf, but are AC_SUBSTed -# for compatibility with existing Makefiles. -ECHO_C= ECHO_N= ECHO_T= -case `echo -n x` in #((((( --n*) - case `echo 'xy\c'` in - *c*) ECHO_T=' ';; # ECHO_T is single tab character. - xy) ECHO_C='\c';; - *) echo `echo ksh88 bug on AIX 6.1` > /dev/null - ECHO_T=' ';; - esac;; -*) - ECHO_N='-n';; -esac - -# For backward compatibility with old third-party macros, we provide -# the shell variables $as_echo and $as_echo_n. New code should use -# AS_ECHO(["message"]) and AS_ECHO_N(["message"]), respectively. -as_echo='printf %s\n' -as_echo_n='printf %s' - rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file @@ -589,6 +573,7 @@ ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` # Initializations. # ac_default_prefix=/usr/local +ac_clean_CONFIG_STATUS= ac_clean_files= ac_config_libobj_dir=. LIBOBJS= @@ -600,8 +585,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='CoGAPS' PACKAGE_TARNAME='cogaps' -PACKAGE_VERSION='3.25.1' -PACKAGE_STRING='CoGAPS 3.25.1' +PACKAGE_VERSION='3.33.2' +PACKAGE_STRING='CoGAPS 3.33.2' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -611,6 +596,7 @@ GAPS_SOURCE_FILES GAPS_LIBS GAPS_CXX_FLAGS GAPS_CPP_FLAGS +SED CXXCPP OBJEXT EXEEXT @@ -619,13 +605,13 @@ CPPFLAGS LDFLAGS CXXFLAGS CXX +ECHO_T +ECHO_N +ECHO_C target_alias host_alias build_alias LIBS -ECHO_T -ECHO_N -ECHO_C DEFS mandir localedir @@ -661,11 +647,12 @@ SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking +enable_gaps_debug enable_debug enable_cpp_tests +enable_checkpoints enable_warnings enable_simd -enable_openmp ' ac_precious_vars='build_alias host_alias @@ -787,7 +774,7 @@ do expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt - ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` + ac_useropt=`printf '%s\n' "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" @@ -813,7 +800,7 @@ do expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: '$ac_useropt'" ac_useropt_orig=$ac_useropt - ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` + ac_useropt=`printf '%s\n' "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" @@ -1026,7 +1013,7 @@ do expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt - ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` + ac_useropt=`printf '%s\n' "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" @@ -1042,7 +1029,7 @@ do expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: '$ac_useropt'" ac_useropt_orig=$ac_useropt - ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'` + ac_useropt=`printf '%s\n' "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" @@ -1086,9 +1073,9 @@ Try '$0 --help' for more information" *) # FIXME: should be removed in autoconf 3.0. - printf "%s\n" "$as_me: WARNING: you should use --build, --host, --target" >&2 + printf '%s\n' "$as_me: WARNING: you should use --build, --host, --target" >&2 expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && - printf "%s\n" "$as_me: WARNING: invalid host type: $ac_option" >&2 + printf '%s\n' "$as_me: WARNING: invalid host type: $ac_option" >&2 : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" ;; @@ -1096,7 +1083,7 @@ Try '$0 --help' for more information" done if test -n "$ac_prev"; then - ac_option=--`echo $ac_prev | sed 's/_/-/g'` + ac_option=--`printf '%s\n' $ac_prev | sed 's/_/-/g'` as_fn_error $? "missing argument to $ac_option" fi @@ -1104,7 +1091,7 @@ if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; - *) printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; + *) printf '%s\n' "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi @@ -1168,7 +1155,7 @@ $as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_myself" : 'X\(//\)[^/]' \| \ X"$as_myself" : 'X\(//\)$' \| \ X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X"$as_myself" | +printf '%s\n' X"$as_myself" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q @@ -1225,7 +1212,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -'configure' configures CoGAPS 3.25.1 to adapt to many kinds of systems. +'configure' configures CoGAPS 3.33.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1287,7 +1274,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of CoGAPS 3.25.1:";; + short | recursive ) echo "Configuration of CoGAPS 3.33.2:";; esac cat <<\_ACEOF @@ -1295,11 +1282,12 @@ Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-debug build debug version of CoGAPS + --enable-gaps-debug build debug version of CoGAPS + --enables-debug build CoGAPS with debug options --enable-cpp-tests turn on C++ unit tests + --enable-checkpoints turn on checkpoint (save/resume) support --enable-warnings compile CoGAPS with warning messages --enable-simd compile with SIMD support if available - --enable-openmp compile with openMP support if available Some influential environment variables: CXX C++ compiler command @@ -1330,9 +1318,9 @@ if test "$ac_init_help" = "recursive"; then case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) - ac_dir_suffix=/`printf "%s\n" "$ac_dir" | sed 's|^\.[\\/]||'` + ac_dir_suffix=/`printf '%s\n' "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. - ac_top_builddir_sub=`printf "%s\n" "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + ac_top_builddir_sub=`printf '%s\n' "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; @@ -1369,7 +1357,7 @@ ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix echo && $SHELL "$ac_srcdir/configure" --help=recursive else - printf "%s\n" "$as_me: WARNING: no configuration information is in $ac_dir" >&2 + printf '%s\n' "$as_me: WARNING: no configuration information is in $ac_dir" >&2 fi || ac_status=$? cd "$ac_pwd" || { ac_status=$?; break; } done @@ -1378,10 +1366,10 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -CoGAPS configure 3.25.1 -generated by GNU Autoconf 2.72 +CoGAPS configure 3.33.2 +generated by GNU Autoconf 2.73 -Copyright (C) 2023 Free Software Foundation, Inc. +Copyright (C) 2026 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -1405,7 +1393,7 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>conftest.err ac_status=$? if test -s conftest.err; then @@ -1413,7 +1401,7 @@ printf "%s\n" "$ac_try_echo"; } >&5 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { test -z "$ac_cxx_werror_flag" || test ! -s conftest.err @@ -1421,7 +1409,7 @@ printf "%s\n" "$ac_try_echo"; } >&5 then : ac_retval=0 else case e in #( - e) printf "%s\n" "$as_me: failed program was:" >&5 + e) printf '%s\n' "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 ;; @@ -1444,7 +1432,7 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err ac_status=$? if test -s conftest.err; then @@ -1452,7 +1440,7 @@ printf "%s\n" "$ac_try_echo"; } >&5 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } > conftest.i && { test -z "$ac_cxx_preproc_warn_flag$ac_cxx_werror_flag" || test ! -s conftest.err @@ -1460,7 +1448,7 @@ printf "%s\n" "$ac_try_echo"; } >&5 then : ac_retval=0 else case e in #( - e) printf "%s\n" "$as_me: failed program was:" >&5 + e) printf '%s\n' "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 ;; @@ -1470,12 +1458,251 @@ fi as_fn_set_status $ac_retval } # ac_fn_cxx_try_cpp + +# ac_fn_cxx_try_run LINENO +# ------------------------ +# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that +# executables *can* be run. +ac_fn_cxx_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf '%s\n' "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf '%s\n' "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; } +then : + ac_retval=0 +else case e in #( + e) printf '%s\n' "$as_me: program exited with status $ac_status" >&5 + printf '%s\n' "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status ;; +esac +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_cxx_try_run + +# ac_fn_cxx_compute_int LINENO EXPR VAR INCLUDES +# ---------------------------------------------- +# Tries to find the compile-time value of EXPR in a program that includes +# INCLUDES, setting VAR accordingly. Returns whether the value could be +# computed +ac_fn_cxx_compute_int () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if test "$cross_compiling" = yes; then + # Depending upon the size, compute the lo and hi bounds. +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main (void) +{ +static int test_array [1 - 2 * !(($2) >= 0)]; +test_array [0] = 0; +return test_array [0]; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_lo=0 ac_mid=0 + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main (void) +{ +static int test_array [1 - 2 * !(($2) <= $ac_mid)]; +test_array [0] = 0; +return test_array [0]; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_hi=$ac_mid; break +else case e in #( + e) as_fn_arith $ac_mid + 1 && ac_lo=$as_val + if test $ac_lo -le $ac_mid; then + ac_lo= ac_hi= + break + fi + as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + done +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main (void) +{ +static int test_array [1 - 2 * !(($2) < 0)]; +test_array [0] = 0; +return test_array [0]; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_hi=-1 ac_mid=-1 + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main (void) +{ +static int test_array [1 - 2 * !(($2) >= $ac_mid)]; +test_array [0] = 0; +return test_array [0]; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_lo=$ac_mid; break +else case e in #( + e) as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val + if test $ac_mid -le $ac_hi; then + ac_lo= ac_hi= + break + fi + as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + done +else case e in #( + e) ac_lo= ac_hi= ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +# Binary search between lo and hi bounds. +while test "x$ac_lo" != "x$ac_hi"; do + as_fn_arith '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo && ac_mid=$as_val + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +int +main (void) +{ +static int test_array [1 - 2 * !(($2) <= $ac_mid)]; +test_array [0] = 0; +return test_array [0]; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_hi=$ac_mid +else case e in #( + e) as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +done +case $ac_lo in #(( +?*) eval "$3=\$ac_lo"; ac_retval=0 ;; +'') ac_retval=1 ;; +esac + else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +static long int longval (void) { return $2; } +static unsigned long int ulongval (void) { return $2; } +#include +#include +int +main (void) +{ + + FILE *f = fopen ("conftest.val", "w"); + if (! f) + return 1; + if (($2) < 0) + { + long int i = longval (); + if (i != ($2)) + return 1; + fprintf (f, "%ld", i); + } + else + { + unsigned long int i = ulongval (); + if (i != ($2)) + return 1; + fprintf (f, "%lu", i); + } + /* Do not output a trailing newline, as this causes \r\n confusion + on some platforms. */ + return ferror (f) || fclose (f) != 0; + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO" +then : + echo >>conftest.val; read $3 config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by CoGAPS $as_me 3.25.1, which was -generated by GNU Autoconf 2.72. Invocation command line was +It was created by CoGAPS $as_me 3.33.2, which was +generated by GNU Autoconf 2.73. Invocation command line was $ $0$ac_configure_args_raw @@ -1535,7 +1762,7 @@ do */) ;; *) as_dir=$as_dir/ ;; esac - printf "%s\n" "PATH: $as_dir" + printf '%s\n' "PATH: $as_dir" done IFS=$as_save_IFS @@ -1570,7 +1797,7 @@ do | -silent | --silent | --silen | --sile | --sil) continue ;; *\'*) - ac_arg=`printf "%s\n" "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; + ac_arg=`printf '%s\n' "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; esac case $ac_pass in 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; @@ -1599,31 +1826,22 @@ done { ac_configure_args0=; unset ac_configure_args0;} { ac_configure_args1=; unset ac_configure_args1;} -# When interrupted or exit'd, cleanup temporary files, and complete -# config.log. We remove comments because anyway the quotes in there -# would cause problems or look ugly. -# WARNING: Use '\'' to represent an apostrophe within the trap. -# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. -trap 'exit_status=$? - # Sanitize IFS. - IFS=" "" $as_nl" - # Save into config.log some information that might help in debugging. - { - echo - - printf "%s\n" "## ---------------- ## -## Cache variables. ## -## ---------------- ##" - echo - # The following way of writing the cache mishandles newlines in values, +# Dump the cache to stdout. It can be in a pipe (this is a requirement). +ac_cache_dump () +{ + # The following way of writing the cache mishandles newlines in values, +# but we know of no workaround that is simple, portable, and efficient. +# So, we kill variables containing newlines. +# Ultrix sh set writes to stderr and can't be redirected directly, +# and sets the high bit in the cache file unless we assign to the vars. ( - for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do + for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do eval ac_val=\$$ac_var case $ac_val in #( *${as_nl}*) case $ac_var in #( - *_cv_*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 -printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; + *_cv_*) { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 +printf '%s\n' "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; esac case $ac_var in #( _ | IFS | as_nl) ;; #( @@ -1632,67 +1850,95 @@ printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} esac ;; esac done + (set) 2>&1 | - case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( + case $as_nl`(ac_space=' '; set) 2>&1` in #( *${as_nl}ac_space=\ *) + # 'set' does not quote correctly, so add quotes: double-quote + # substitution turns \\\\ into \\, and sed turns \\ into \. sed -n \ - "s/'\''/'\''\\\\'\'''\''/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" + "s/'/'\\\\''/g; + s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" ;; #( *) + # 'set' quotes correctly as required by POSIX, so do not add quotes. sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | sort ) - echo +} + +# Print debugging info to stdout. +ac_dump_debugging_info () +{ + echo + + printf '%s\n' "## ---------------- ## +## Cache variables. ## +## ---------------- ##" + echo + ac_cache_dump + echo - printf "%s\n" "## ----------------- ## + printf '%s\n' "## ----------------- ## ## Output variables. ## ## ----------------- ##" + echo + for ac_var in $ac_subst_vars + do + eval ac_val=\$$ac_var + case $ac_val in + *\'*) ac_val=`printf '%s\n' "$ac_val" | sed "s/'/'\\\\\\\\''/g"`;; + esac + printf '%s\n' "$ac_var='$ac_val'" + done | sort + echo + + if test -n "$ac_subst_files"; then + printf '%s\n' "## ------------------- ## +## File substitutions. ## +## ------------------- ##" echo - for ac_var in $ac_subst_vars + for ac_var in $ac_subst_files do eval ac_val=\$$ac_var case $ac_val in - *\'\''*) ac_val=`printf "%s\n" "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; + *\'*) ac_val=`printf '%s\n' "$ac_val" | sed "s/'/'\\\\\\\\''/g"`;; esac - printf "%s\n" "$ac_var='\''$ac_val'\''" + printf '%s\n' "$ac_var='$ac_val'" done | sort echo + fi - if test -n "$ac_subst_files"; then - printf "%s\n" "## ------------------- ## -## File substitutions. ## -## ------------------- ##" - echo - for ac_var in $ac_subst_files - do - eval ac_val=\$$ac_var - case $ac_val in - *\'\''*) ac_val=`printf "%s\n" "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; - esac - printf "%s\n" "$ac_var='\''$ac_val'\''" - done | sort - echo - fi - - if test -s confdefs.h; then - printf "%s\n" "## ----------- ## + if test -s confdefs.h; then + printf '%s\n' "## ----------- ## ## confdefs.h. ## ## ----------- ##" - echo - cat confdefs.h - echo - fi - test "$ac_signal" != 0 && - printf "%s\n" "$as_me: caught signal $ac_signal" - printf "%s\n" "$as_me: exit $exit_status" - } >&5 - rm -f core *.core core.conftest.* && + echo + cat confdefs.h + echo + fi + test "$ac_signal" != 0 && + printf '%s\n' "$as_me: caught signal $ac_signal" + printf '%s\n' "$as_me: exit $exit_status" +} + +# When interrupted or exit'd, cleanup temporary files, and complete +# config.log. +ac_exit_trap () +{ + exit_status= + # Sanitize IFS. + IFS=" "" $as_nl" + # Save into config.log some information that might help in debugging. + ac_dump_debugging_info >&5 + eval "rm -f $ac_clean_CONFIG_STATUS core *.core core.conftest.*" && rm -f -r conftest* confdefs* conf$$* $ac_clean_files && exit $exit_status -' 0 +} + +trap 'ac_exit_trap $?' 0 for ac_signal in 1 2 13 15; do trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal done @@ -1701,21 +1947,21 @@ ac_signal=0 # confdefs.h avoids OS command line length limits that DEFS can exceed. rm -f -r conftest* confdefs.h -printf "%s\n" "/* confdefs.h */" > confdefs.h +printf '%s\n' "/* confdefs.h */" > confdefs.h # Predefined preprocessor variables. -printf "%s\n" "#define PACKAGE_NAME \"$PACKAGE_NAME\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_NAME \"$PACKAGE_NAME\"" >>confdefs.h -printf "%s\n" "#define PACKAGE_TARNAME \"$PACKAGE_TARNAME\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_TARNAME \"$PACKAGE_TARNAME\"" >>confdefs.h -printf "%s\n" "#define PACKAGE_VERSION \"$PACKAGE_VERSION\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_VERSION \"$PACKAGE_VERSION\"" >>confdefs.h -printf "%s\n" "#define PACKAGE_STRING \"$PACKAGE_STRING\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_STRING \"$PACKAGE_STRING\"" >>confdefs.h -printf "%s\n" "#define PACKAGE_BUGREPORT \"$PACKAGE_BUGREPORT\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_BUGREPORT \"$PACKAGE_BUGREPORT\"" >>confdefs.h -printf "%s\n" "#define PACKAGE_URL \"$PACKAGE_URL\"" >>confdefs.h +printf '%s\n' "#define PACKAGE_URL \"$PACKAGE_URL\"" >>confdefs.h # Let the site file select an alternate cache file if it wants to. @@ -1737,12 +1983,12 @@ do ac_site_file=./$ac_site_file ;; esac if test -f "$ac_site_file" && test -r "$ac_site_file"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 -printf "%s\n" "$as_me: loading site script $ac_site_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 +printf '%s\n' "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 . "$ac_site_file" \ - || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} + || { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "failed to load site script $ac_site_file See 'config.log' for more details" "$LINENO" 5; } fi @@ -1752,235 +1998,19 @@ if test -r "$cache_file"; then # Some versions of bash will fail to source /dev/null (special files # actually), so we avoid doing that. DJGPP emulates it as a regular file. if test /dev/null != "$cache_file" && test -f "$cache_file"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 -printf "%s\n" "$as_me: loading cache $cache_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 +printf '%s\n' "$as_me: loading cache $cache_file" >&6;} case $cache_file in [\\/]* | ?:[\\/]* ) . "$cache_file";; *) . "./$cache_file";; esac fi else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 -printf "%s\n" "$as_me: creating cache $cache_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 +printf '%s\n' "$as_me: creating cache $cache_file" >&6;} >$cache_file fi -# Test code for whether the C++ compiler supports C++98 (global declarations) -ac_cxx_conftest_cxx98_globals=' -// Does the compiler advertise C++98 conformance? -#if !defined __cplusplus || __cplusplus < 199711L -# error "Compiler does not advertise C++98 conformance" -#endif - -// These inclusions are to reject old compilers that -// lack the unsuffixed header files. -#include -#include - -// and are *not* freestanding headers in C++98. -extern void assert (int); -namespace std { - extern int strcmp (const char *, const char *); -} - -// Namespaces, exceptions, and templates were all added after "C++ 2.0". -using std::exception; -using std::strcmp; - -namespace { - -void test_exception_syntax() -{ - try { - throw "test"; - } catch (const char *s) { - // Extra parentheses suppress a warning when building autoconf itself, - // due to lint rules shared with more typical C programs. - assert (!(strcmp) (s, "test")); - } -} - -template struct test_template -{ - T const val; - explicit test_template(T t) : val(t) {} - template T add(U u) { return static_cast(u) + val; } -}; - -} // anonymous namespace -' - -# Test code for whether the C++ compiler supports C++98 (body of main) -ac_cxx_conftest_cxx98_main=' - assert (argc); - assert (! argv[0]); -{ - test_exception_syntax (); - test_template tt (2.0); - assert (tt.add (4) == 6.0); - assert (true && !false); -} -' - -# Test code for whether the C++ compiler supports C++11 (global declarations) -ac_cxx_conftest_cxx11_globals=' -// Does the compiler advertise C++ 2011 conformance? -#if !defined __cplusplus || __cplusplus < 201103L -# error "Compiler does not advertise C++11 conformance" -#endif - -namespace cxx11test -{ - constexpr int get_val() { return 20; } - - struct testinit - { - int i; - double d; - }; - - class delegate - { - public: - delegate(int n) : n(n) {} - delegate(): delegate(2354) {} - - virtual int getval() { return this->n; }; - protected: - int n; - }; - - class overridden : public delegate - { - public: - overridden(int n): delegate(n) {} - virtual int getval() override final { return this->n * 2; } - }; - - class nocopy - { - public: - nocopy(int i): i(i) {} - nocopy() = default; - nocopy(const nocopy&) = delete; - nocopy & operator=(const nocopy&) = delete; - private: - int i; - }; - - // for testing lambda expressions - template Ret eval(Fn f, Ret v) - { - return f(v); - } - - // for testing variadic templates and trailing return types - template auto sum(V first) -> V - { - return first; - } - template auto sum(V first, Args... rest) -> V - { - return first + sum(rest...); - } -} -' - -# Test code for whether the C++ compiler supports C++11 (body of main) -ac_cxx_conftest_cxx11_main=' -{ - // Test auto and decltype - auto a1 = 6538; - auto a2 = 48573953.4; - auto a3 = "String literal"; - - int total = 0; - for (auto i = a3; *i; ++i) { total += *i; } - - decltype(a2) a4 = 34895.034; -} -{ - // Test constexpr - short sa[cxx11test::get_val()] = { 0 }; -} -{ - // Test initializer lists - cxx11test::testinit il = { 4323, 435234.23544 }; -} -{ - // Test range-based for - int array[] = {9, 7, 13, 15, 4, 18, 12, 10, 5, 3, - 14, 19, 17, 8, 6, 20, 16, 2, 11, 1}; - for (auto &x : array) { x += 23; } -} -{ - // Test lambda expressions - using cxx11test::eval; - assert (eval ([](int x) { return x*2; }, 21) == 42); - double d = 2.0; - assert (eval ([&](double x) { return d += x; }, 3.0) == 5.0); - assert (d == 5.0); - assert (eval ([=](double x) mutable { return d += x; }, 4.0) == 9.0); - assert (d == 5.0); -} -{ - // Test use of variadic templates - using cxx11test::sum; - auto a = sum(1); - auto b = sum(1, 2); - auto c = sum(1.0, 2.0, 3.0); -} -{ - // Test constructor delegation - cxx11test::delegate d1; - cxx11test::delegate d2(); - cxx11test::delegate d3(45); -} -{ - // Test override and final - cxx11test::overridden o1(55464); -} -{ - // Test nullptr - char *c = nullptr; -} -{ - // Test template brackets - test_template<::test_template> v(test_template(12)); -} -{ - // Unicode literals - char const *utf8 = u8"UTF-8 string \u2500"; - char16_t const *utf16 = u"UTF-8 string \u2500"; - char32_t const *utf32 = U"UTF-32 string \u2500"; -} -' - -# Test code for whether the C compiler supports C++11 (complete). -ac_cxx_conftest_cxx11_program="${ac_cxx_conftest_cxx98_globals} -${ac_cxx_conftest_cxx11_globals} - -int -main (int argc, char **argv) -{ - int ok = 0; - ${ac_cxx_conftest_cxx98_main} - ${ac_cxx_conftest_cxx11_main} - return ok; -} -" - -# Test code for whether the C compiler supports C++98 (complete). -ac_cxx_conftest_cxx98_program="${ac_cxx_conftest_cxx98_globals} -int -main (int argc, char **argv) -{ - int ok = 0; - ${ac_cxx_conftest_cxx98_main} - return ok; -} -" - # Check that the precious variables saved in the cache have kept the same # value. ac_cache_corrupted=false @@ -1991,38 +2021,44 @@ for ac_var in $ac_precious_vars; do eval ac_new_val=\$ac_env_${ac_var}_value case $ac_old_set,$ac_new_set in set,) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&5 -printf "%s\n" "$as_me: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&5 +printf '%s\n' "$as_me: error: '$ac_var' was set to '$ac_old_val' in the previous run" >&2;} ac_cache_corrupted=: ;; ,set) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was not set in the previous run" >&5 -printf "%s\n" "$as_me: error: '$ac_var' was not set in the previous run" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: '$ac_var' was not set in the previous run" >&5 +printf '%s\n' "$as_me: error: '$ac_var' was not set in the previous run" >&2;} ac_cache_corrupted=: ;; ,);; *) if test "x$ac_old_val" != "x$ac_new_val"; then # differences in whitespace do not lead to failure. - ac_old_val_w=`echo x $ac_old_val` - ac_new_val_w=`echo x $ac_new_val` + ac_old_val_w= + for ac_val in x $ac_old_val; do + ac_old_val_w="$ac_old_val_w $ac_val" + done + ac_new_val_w= + for ac_val in x $ac_new_val; do + ac_new_val_w="$ac_new_val_w $ac_val" + done if test "$ac_old_val_w" != "$ac_new_val_w"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: '$ac_var' has changed since the previous run:" >&5 -printf "%s\n" "$as_me: error: '$ac_var' has changed since the previous run:" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: '$ac_var' has changed since the previous run:" >&5 +printf '%s\n' "$as_me: error: '$ac_var' has changed since the previous run:" >&2;} ac_cache_corrupted=: else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&5 -printf "%s\n" "$as_me: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&5 +printf '%s\n' "$as_me: warning: ignoring whitespace changes in '$ac_var' since the previous run:" >&2;} eval $ac_var=\$ac_old_val fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: former value: '$ac_old_val'" >&5 -printf "%s\n" "$as_me: former value: '$ac_old_val'" >&2;} - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: current value: '$ac_new_val'" >&5 -printf "%s\n" "$as_me: current value: '$ac_new_val'" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: former value: '$ac_old_val'" >&5 +printf '%s\n' "$as_me: former value: '$ac_old_val'" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: current value: '$ac_new_val'" >&5 +printf '%s\n' "$as_me: current value: '$ac_new_val'" >&2;} fi;; esac # Pass precious variables to config.status. if test "$ac_new_set" = set; then case $ac_new_val in - *\'*) ac_arg=$ac_var=`printf "%s\n" "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; + *\'*) ac_arg=$ac_var=`printf '%s\n' "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; *) ac_arg=$ac_var=$ac_new_val ;; esac case " $ac_configure_args " in @@ -2032,10 +2068,10 @@ printf "%s\n" "$as_me: current value: '$ac_new_val'" >&2;} fi done if $ac_cache_corrupted; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 -printf "%s\n" "$as_me: error: changes in the environment can compromise the build" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 +printf '%s\n' "$as_me: error: changes in the environment can compromise the build" >&2;} as_fn_error $? "run '${MAKE-make} distclean' and/or 'rm $cache_file' and start over" "$LINENO" 5 fi @@ -2043,6 +2079,23 @@ fi ## Main body of script. ## ## -------------------- ## + +# Determine whether it's possible to make 'echo' print without a newline. +# These variables are no longer used directly by Autoconf, but are AC_SUBSTed +# for compatibility with existing Makefiles. +ECHO_C= ECHO_N= ECHO_T= +case `echo -n x` in #((((( +-n*) + case `echo 'xy\c'` in + *c*) ECHO_T=' ';; # ECHO_T is single tab character. + xy) ECHO_C='\c';; + *) echo `echo ksh88 bug on AIX 6.1` > /dev/null + ECHO_T=' ';; + esac;; +*) + ECHO_N='-n';; +esac + ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' @@ -2061,12 +2114,6 @@ ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_cxx_compiler_gnu - - - - - - ac_ext=cpp ac_cpp='$CXXCPP $CPPFLAGS' ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' @@ -2081,7 +2128,7 @@ if test -z "$CXX"; then do # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CXX+y} then : @@ -2102,7 +2149,7 @@ do for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi done @@ -2114,11 +2161,11 @@ esac fi CXX=$ac_cv_prog_CXX if test -n "$CXX"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 -printf "%s\n" "$CXX" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +printf '%s\n' "$CXX" >&6; } else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf '%s\n' "no" >&6; } fi @@ -2131,7 +2178,7 @@ if test -z "$CXX"; then do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CXX+y} then : @@ -2152,7 +2199,7 @@ do for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CXX="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi done @@ -2164,11 +2211,11 @@ esac fi ac_ct_CXX=$ac_cv_prog_ac_ct_CXX if test -n "$ac_ct_CXX"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 -printf "%s\n" "$ac_ct_CXX" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 +printf '%s\n' "$ac_ct_CXX" >&6; } else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf '%s\n' "no" >&6; } fi @@ -2180,8 +2227,8 @@ done else case $cross_compiling:$ac_tool_warned in yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf '%s\n' "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac CXX=$ac_ct_CXX @@ -2191,7 +2238,7 @@ fi fi fi # Provide some information about the compiler. -printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 set X $ac_compile ac_compiler=$2 for ac_option in --version -v -V -qversion; do @@ -2201,7 +2248,7 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_compiler $ac_option >&5") 2>conftest.err ac_status=$? if test -s conftest.err; then @@ -2211,7 +2258,7 @@ printf "%s\n" "$ac_try_echo"; } >&5 cat conftest.er1 >&5 fi rm -f conftest.er1 conftest.err - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } done @@ -2231,9 +2278,9 @@ ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" # Try to create an executable without -o first, disregard a.out. # It will help us diagnose broken compilers, and finding out an intuition # of exeext. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C++ compiler works" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether the C++ compiler works" >&5 printf %s "checking whether the C++ compiler works... " >&6; } -ac_link_default=`printf "%s\n" "$ac_link" | sed 's/ -o *conftest[^ ]*//'` +ac_link_default=`printf '%s\n' "$ac_link" | sed 's/ -o *conftest[^ ]*//'` # The possible output files: ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" @@ -2254,10 +2301,10 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_link_default") 2>&5 ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : # Autoconf-2.13 could set the ac_cv_exeext variable to 'no'. @@ -2298,29 +2345,29 @@ esac fi if test -z "$ac_file" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -printf "%s\n" "$as_me: failed program was:" >&5 + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf '%s\n' "no" >&6; } +printf '%s\n' "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} +{ { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "C++ compiler cannot create executables See 'config.log' for more details" "$LINENO" 5; } else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } ;; + e) { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf '%s\n' "yes" >&6; } ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C++ compiler default output file name" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for C++ compiler default output file name" >&5 printf %s "checking for C++ compiler default output file name... " >&6; } -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 -printf "%s\n" "$ac_file" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 +printf '%s\n' "$ac_file" >&6; } ac_exeext=$ac_cv_exeext rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out ac_clean_files=$ac_clean_files_save -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 printf %s "checking for suffix of executables... " >&6; } if { { ac_try="$ac_link" case "(($ac_try" in @@ -2328,10 +2375,10 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : # If both 'conftest.exe' and 'conftest' are 'present' (well, observable) @@ -2348,15 +2395,15 @@ for ac_file in conftest.exe conftest conftest.*; do esac done else case e in #( - e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of executables: cannot compile and link See 'config.log' for more details" "$LINENO" 5; } ;; esac fi rm -f conftest conftest$ac_cv_exeext -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 -printf "%s\n" "$ac_cv_exeext" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 +printf '%s\n' "$ac_cv_exeext" >&6; } rm -f conftest.$ac_ext EXEEXT=$ac_cv_exeext @@ -2379,7 +2426,7 @@ _ACEOF ac_clean_files="$ac_clean_files conftest.out" # Check that the compiler produces executables we can run. If not, either # the compiler is broken, or we cross compile. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 printf %s "checking whether we are cross compiling... " >&6; } if test "$cross_compiling" != yes; then { { ac_try="$ac_link" @@ -2388,10 +2435,10 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } if { ac_try='./conftest$ac_cv_exeext' { { case "(($ac_try" in @@ -2399,31 +2446,31 @@ printf "%s\n" "$ac_try_echo"; } >&5 *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_try") 2>&5 ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; }; then cross_compiling=no else if test "$cross_compiling" = maybe; then cross_compiling=yes else - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} + { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error 77 "cannot run C++ compiled programs. If you meant to cross compile, use '--host'. See 'config.log' for more details" "$LINENO" 5; } fi fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 -printf "%s\n" "$cross_compiling" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 +printf '%s\n' "$cross_compiling" >&6; } rm -f conftest.$ac_ext conftest$ac_cv_exeext \ conftest.o conftest.obj conftest.out ac_clean_files=$ac_clean_files_save -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 printf %s "checking for suffix of object files... " >&6; } if test ${ac_cv_objext+y} then : @@ -2447,10 +2494,10 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>&5 ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } then : for ac_file in conftest.o conftest.obj conftest.*; do @@ -2462,11 +2509,11 @@ then : esac done else case e in #( - e) printf "%s\n" "$as_me: failed program was:" >&5 + e) printf '%s\n' "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 -{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} +{ { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of object files: cannot compile See 'config.log' for more details" "$LINENO" 5; } ;; esac @@ -2474,11 +2521,11 @@ fi rm -f conftest.$ac_cv_objext conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 -printf "%s\n" "$ac_cv_objext" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 +printf '%s\n' "$ac_cv_objext" >&6; } OBJEXT=$ac_cv_objext ac_objext=$OBJEXT -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C++" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C++" >&5 printf %s "checking whether the compiler supports GNU C++... " >&6; } if test ${ac_cv_cxx_compiler_gnu+y} then : @@ -2510,8 +2557,8 @@ ac_cv_cxx_compiler_gnu=$ac_compiler_gnu ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 -printf "%s\n" "$ac_cv_cxx_compiler_gnu" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 +printf '%s\n' "$ac_cv_cxx_compiler_gnu" >&6; } ac_compiler_gnu=$ac_cv_cxx_compiler_gnu if test $ac_compiler_gnu = yes; then @@ -2521,7 +2568,7 @@ else fi ac_test_CXXFLAGS=${CXXFLAGS+y} ac_save_CXXFLAGS=$CXXFLAGS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 printf %s "checking whether $CXX accepts -g... " >&6; } if test ${ac_cv_prog_cxx_g+y} then : @@ -2589,8 +2636,8 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ac_cxx_werror_flag=$ac_save_cxx_werror_flag ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 -printf "%s\n" "$ac_cv_prog_cxx_g" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 +printf '%s\n' "$ac_cv_prog_cxx_g" >&6; } if test $ac_test_CXXFLAGS; then CXXFLAGS=$ac_save_CXXFLAGS elif test $ac_cv_prog_cxx_g = yes; then @@ -2606,106 +2653,6 @@ else CXXFLAGS= fi fi -ac_prog_cxx_stdcxx=no -if test x$ac_prog_cxx_stdcxx = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++11 features" >&5 -printf %s "checking for $CXX option to enable C++11 features... " >&6; } -if test ${ac_cv_prog_cxx_cxx11+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) ac_cv_prog_cxx_cxx11=no -ac_save_CXX=$CXX -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_cxx_conftest_cxx11_program -_ACEOF -for ac_arg in '' -std=gnu++11 -std=gnu++0x -std=c++11 -std=c++0x -qlanglvl=extended0x -AA -do - CXX="$ac_save_CXX $ac_arg" - if ac_fn_cxx_try_compile "$LINENO" -then : - ac_cv_prog_cxx_cxx11=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cxx_cxx11" != "xno" && break -done -rm -f conftest.$ac_ext -CXX=$ac_save_CXX ;; -esac -fi - -if test "x$ac_cv_prog_cxx_cxx11" = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } -else case e in #( - e) if test "x$ac_cv_prog_cxx_cxx11" = x -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } -else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_cxx11" >&5 -printf "%s\n" "$ac_cv_prog_cxx_cxx11" >&6; } - CXX="$CXX $ac_cv_prog_cxx_cxx11" ;; -esac -fi - ac_cv_prog_cxx_stdcxx=$ac_cv_prog_cxx_cxx11 - ac_prog_cxx_stdcxx=cxx11 ;; -esac -fi -fi -if test x$ac_prog_cxx_stdcxx = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++98 features" >&5 -printf %s "checking for $CXX option to enable C++98 features... " >&6; } -if test ${ac_cv_prog_cxx_cxx98+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) ac_cv_prog_cxx_cxx98=no -ac_save_CXX=$CXX -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_cxx_conftest_cxx98_program -_ACEOF -for ac_arg in '' -std=gnu++98 -std=c++98 -qlanglvl=extended -AA -do - CXX="$ac_save_CXX $ac_arg" - if ac_fn_cxx_try_compile "$LINENO" -then : - ac_cv_prog_cxx_cxx98=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cxx_cxx98" != "xno" && break -done -rm -f conftest.$ac_ext -CXX=$ac_save_CXX ;; -esac -fi - -if test "x$ac_cv_prog_cxx_cxx98" = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } -else case e in #( - e) if test "x$ac_cv_prog_cxx_cxx98" = x -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } -else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_cxx98" >&5 -printf "%s\n" "$ac_cv_prog_cxx_cxx98" >&6; } - CXX="$CXX $ac_cv_prog_cxx_cxx98" ;; -esac -fi - ac_cv_prog_cxx_stdcxx=$ac_cv_prog_cxx_cxx98 - ac_prog_cxx_stdcxx=cxx98 ;; -esac -fi -fi - ac_ext=cpp ac_cpp='$CXXCPP $CPPFLAGS' ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' @@ -2718,7 +2665,7 @@ ac_cpp='$CXXCPP $CPPFLAGS' ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_cxx_compiler_gnu -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking how to run the C++ preprocessor" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking how to run the C++ preprocessor" >&5 printf %s "checking how to run the C++ preprocessor... " >&6; } if test -z "$CXXCPP"; then if test ${ac_cv_prog_CXXCPP+y} @@ -2785,8 +2732,8 @@ fi else ac_cv_prog_CXXCPP=$CXXCPP fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CXXCPP" >&5 -printf "%s\n" "$CXXCPP" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $CXXCPP" >&5 +printf '%s\n' "$CXXCPP" >&6; } ac_preproc_ok=false for ac_cxx_preproc_warn_flag in '' yes do @@ -2834,8 +2781,8 @@ if $ac_preproc_ok then : else case e in #( - e) { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} as_fn_error $? "C++ preprocessor \"$CXXCPP\" fails sanity check See 'config.log' for more details" "$LINENO" 5; } ;; esac @@ -2863,7 +2810,7 @@ if test -z "$CXX"; then do # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CXX+y} then : @@ -2884,7 +2831,7 @@ do for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then ac_cv_prog_CXX="$ac_tool_prefix$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi done @@ -2896,11 +2843,11 @@ esac fi CXX=$ac_cv_prog_CXX if test -n "$CXX"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 -printf "%s\n" "$CXX" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +printf '%s\n' "$CXX" >&6; } else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf '%s\n' "no" >&6; } fi @@ -2913,7 +2860,7 @@ if test -z "$CXX"; then do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CXX+y} then : @@ -2934,7 +2881,7 @@ do for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CXX="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi done @@ -2946,11 +2893,11 @@ esac fi ac_ct_CXX=$ac_cv_prog_ac_ct_CXX if test -n "$ac_ct_CXX"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 -printf "%s\n" "$ac_ct_CXX" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CXX" >&5 +printf '%s\n' "$ac_ct_CXX" >&6; } else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf '%s\n' "no" >&6; } fi @@ -2962,8 +2909,8 @@ done else case $cross_compiling:$ac_tool_warned in yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf '%s\n' "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac CXX=$ac_ct_CXX @@ -2973,7 +2920,7 @@ fi fi fi # Provide some information about the compiler. -printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 set X $ac_compile ac_compiler=$2 for ac_option in --version -v -V -qversion; do @@ -2983,7 +2930,7 @@ case "(($ac_try" in *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 +printf '%s\n' "$ac_try_echo"; } >&5 (eval "$ac_compiler $ac_option >&5") 2>conftest.err ac_status=$? if test -s conftest.err; then @@ -2993,11 +2940,11 @@ printf "%s\n" "$ac_try_echo"; } >&5 cat conftest.er1 >&5 fi rm -f conftest.er1 conftest.err - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } done -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C++" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C++" >&5 printf %s "checking whether the compiler supports GNU C++... " >&6; } if test ${ac_cv_cxx_compiler_gnu+y} then : @@ -3029,8 +2976,8 @@ ac_cv_cxx_compiler_gnu=$ac_compiler_gnu ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 -printf "%s\n" "$ac_cv_cxx_compiler_gnu" >&6; } +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cxx_compiler_gnu" >&5 +printf '%s\n' "$ac_cv_cxx_compiler_gnu" >&6; } ac_compiler_gnu=$ac_cv_cxx_compiler_gnu if test $ac_compiler_gnu = yes; then @@ -3040,7 +2987,7 @@ else fi ac_test_CXXFLAGS=${CXXFLAGS+y} ac_save_CXXFLAGS=$CXXFLAGS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking whether $CXX accepts -g" >&5 printf %s "checking whether $CXX accepts -g... " >&6; } if test ${ac_cv_prog_cxx_g+y} then : @@ -3069,247 +3016,1476 @@ else case e in #( cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int -main (void) -{ +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + +else case e in #( + e) ac_cxx_werror_flag=$ac_save_cxx_werror_flag + CXXFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + ac_cv_prog_cxx_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_cxx_werror_flag=$ac_save_cxx_werror_flag ;; +esac +fi +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 +printf '%s\n' "$ac_cv_prog_cxx_g" >&6; } +if test $ac_test_CXXFLAGS; then + CXXFLAGS=$ac_save_CXXFLAGS +elif test $ac_cv_prog_cxx_g = yes; then + if test "$GXX" = yes; then + CXXFLAGS="-g -O2" + else + CXXFLAGS="-g" + fi +else + if test "$GXX" = yes; then + CXXFLAGS="-O2" + else + CXXFLAGS= + fi +fi +ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + + +# Check if compiling gaps debug version; actuaooly, it includes broken tests and swithcing on asserts; +# we switched on the asserts unconditionally +# Check whether --enable-gaps-debug was given. +if test ${enable_gaps_debug+y} +then : + enableval=$enable_gaps_debug; case "${enableval}" in + yes) build_gaps_debug=yes ;; + no) build_gaps_debug=no ;; + *) as_fn_error $? "--enable-gaps-debug understads only 'no' or 'yes' values" "$LINENO" 5 ;; + esac +else case e in #( + e) build_gaps_debug=no ;; +esac +fi + + + + +#Switches on debug cpp flags -g -O0 +# Check whether --enable-debug was given. +if test ${enable_debug+y} +then : + enableval=$enable_debug; case "${enableval}" in + yes) build_debug=yes ;; + no) build_debug=no ;; + *) as_fn_error $? "--enable-debug understads only 'no' or 'yes' values" "$LINENO" 5 ;; + esac +else case e in #( + e) build_debug=no ;; +esac +fi + + + +# Check if running C++ tests (this adds to compilation time) +# Check whether --enable-cpp-tests was given. +if test ${enable_cpp_tests+y} +then : + enableval=$enable_cpp_tests; case "${enableval}" in + no) cpp_tests_disable=yes ;; + yes) cpp_tests_disable=no ;; + *) as_fn_error $? "--enable-cpp-tests understads only 'no' or 'yes' values" "$LINENO" 5 ;; + esac +else case e in #( + e) cpp_tests_disable=no ;; +esac +fi + + +# Check if checkpoint (save/resume) support should be compiled in. Checkpoints are +# an emergency/debug feature that serialize state on every run; off by default, +# enable with --enable-checkpoints for package debugging. +# Check whether --enable-checkpoints was given. +if test ${enable_checkpoints+y} +then : + enableval=$enable_checkpoints; case "${enableval}" in + no) cpp_checkpoints_disable=yes ;; + yes) cpp_checkpoints_disable=no ;; + *) as_fn_error $? "--enable-checkpoints understands only 'no' or 'yes' values" "$LINENO" 5 ;; + esac +else case e in #( + e) cpp_checkpoints_disable=yes ;; +esac +fi + + +# Check if compiler warnings should be turned on +# Check whether --enable-warnings was given. +if test ${enable_warnings+y} +then : + enableval=$enable_warnings; warnings=yes +else case e in #( + e) warnings=no ;; +esac +fi + + +# Use SIMD unless requested not to +# Check whether --enable-simd was given. +if test ${enable_simd+y} +then : + enableval=$enable_simd; use_simd=$enableval +else case e in #( + e) use_simd=yes ;; +esac +fi + + +# default CoGAPS specific flags +GAPS_CPP_FLAGS=" -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=0 -D__GAPS_R_BUILD__" +GAPS_CXX_FLAGS= +GAPS_LIBS= + +# checkpoints are off by default; define the disabling macro unless --enable-checkpoints +if test "x$cpp_checkpoints_disable" != "xno" ; then + GAPS_CPP_FLAGS+=" -DGAPS_DISABLE_CHECKPOINTS " +fi + +# get compiler info + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for C++ compiler vendor" >&5 +printf %s "checking for C++ compiler vendor... " >&6; } +if test ${ax_cv_cxx_compiler_vendor+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + vendors=" + intel: __ICC,__ECC,__INTEL_COMPILER + ibm: __xlc__,__xlC__,__IBMC__,__IBMCPP__,__ibmxl__ + pathscale: __PATHCC__,__PATHSCALE__ + clang: __clang__ + cray: _CRAYC + fujitsu: __FUJITSU + sdcc: SDCC,__SDCC + sx: _SX + nvhpc: __NVCOMPILER + portland: __PGI + gnu: __GNUC__ + sun: __SUNPRO_C,__SUNPRO_CC,__SUNPRO_F90,__SUNPRO_F95 + hp: __HP_cc,__HP_aCC + dec: __DECC,__DECCXX,__DECC_VER,__DECCXX_VER + borland: __BORLANDC__,__CODEGEARC__,__TURBOC__ + comeau: __COMO__ + kai: __KCC + lcc: __LCC__ + sgi: __sgi,sgi + microsoft: _MSC_VER + metrowerks: __MWERKS__ + watcom: __WATCOMC__ + tcc: __TINYC__ + unknown: UNKNOWN + " + for ventest in $vendors; do + case $ventest in + *:) + vendor=$ventest + continue + ;; + *) + vencpp="defined("`echo $ventest | sed 's/,/) || defined(/g'`")" + ;; + esac + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + +#if !($vencpp) + thisisanerror; +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + break +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + done + + ax_cv_cxx_compiler_vendor=`echo $vendor | cut -d: -f1` + ;; +esac +fi +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ax_cv_cxx_compiler_vendor" >&5 +printf '%s\n' "$ax_cv_cxx_compiler_vendor" >&6; } + +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5 +printf %s "checking for a sed that does not truncate output... " >&6; } +if test ${ac_cv_path_SED+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ + for ac_i in 1 2 3 4 5 6 7; do + ac_script="$ac_script$as_nl$ac_script" + done + echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed + { ac_script=; unset ac_script;} + if test -z "$SED"; then + ac_path_SED_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_prog in sed gsed + do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_SED="$as_dir$ac_prog$ac_exec_ext" + as_fn_executable_p "$ac_path_SED" || continue +# Check for GNU ac_path_SED and select it if it is found. + # Check for GNU $ac_path_SED +case `"$ac_path_SED" --version 2>&1` in #( +*GNU*) + ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; +#( +*) + ac_count=0 + printf %s 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + printf '%s\n' '' >> "conftest.nl" + "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_SED_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_SED="$ac_path_SED" + ac_path_SED_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_SED_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_SED"; then + as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5 + fi +else + ac_cv_path_SED=$SED +fi + ;; +esac +fi +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 +printf '%s\n' "$ac_cv_path_SED" >&6; } + SED="$ac_cv_path_SED" + rm -f conftest.sed + + + + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: checking for C++ compiler version" >&5 +printf %s "checking for C++ compiler version... " >&6; } +if test ${ax_cv_cxx_compiler_version+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) case $ax_cv_cxx_compiler_vendor in #( + intel) : + if ac_fn_cxx_compute_int "$LINENO" "__INTEL_COMPILER/100" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_INTEL unknown intel compiler version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__INTEL_COMPILER%100)/10" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_INTEL unknown intel compiler version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__INTEL_COMPILER%10)" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_INTEL unknown intel compiler version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + ibm) : + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main (void) +{ + + #if defined(__COMPILER_VER__) + choke me; + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + + if ac_fn_cxx_compute_int "$LINENO" "__xlC__/100" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__xlC__%100" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__xlC_ver__/0x100" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__xlC_ver__%0x100" "_ax_cxx_compiler_version_build" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler build version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch.$_ax_cxx_compiler_version_build" + +else case e in #( + e) + if ac_fn_cxx_compute_int "$LINENO" "__xlC__%1000" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__xlC__/10000)%10" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__xlC__/100000)%10" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_IBM unknown IBM compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; #( + pathscale) : + + if ac_fn_cxx_compute_int "$LINENO" "__PATHCC__" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PATHSCALE unknown pathscale major +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__PATHCC_MINOR__" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PATHSCALE unknown pathscale minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__PATHCC_PATCHLEVEL__" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PATHSCALE unknown pathscale patch level +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + clang) : + + if ac_fn_cxx_compute_int "$LINENO" "__clang_major__" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_CLANG unknown clang major +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__clang_minor__" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_CLANG unknown clang minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__clang_patchlevel__" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) 0 ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + cray) : + + if ac_fn_cxx_compute_int "$LINENO" "_RELEASE" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_CRAY unknown crayc release +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "_RELEASE_MINOR" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_CRAY unknown crayc minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor" + ;; #( + fujitsu) : + + if ac_fn_cxx_compute_int "$LINENO" "__FCC_VERSION" "ax_cv_cxx_compiler_version" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_FUJITSUunknown fujitsu release +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ;; #( + gnu) : + + if ac_fn_cxx_compute_int "$LINENO" "__GNUC__" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_GNU unknown gcc major +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__GNUC_MINOR__" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_GNU unknown gcc minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__GNUC_PATCHLEVEL__" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_GNU unknown gcc patch level +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + sun) : + + + if ac_fn_cxx_compute_int "$LINENO" "!!( + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + < 0x1000)" "_ax_cxx_compiler_version_until59" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun release version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if test "X$_ax_cxx_compiler_version_until59" = X1 +then : + if ac_fn_cxx_compute_int "$LINENO" " + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + % 0x10" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + / 0x10) % 0x10" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + / 0x100)" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + +else case e in #( + e) if ac_fn_cxx_compute_int "$LINENO" " + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + % 0x10" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + / 0x100) % 0x100" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__SUNPRO_CC) + __SUNPRO_CC + #else + __SUNPRO_C + #endif + / 0x1000)" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SUN unknown sun major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ;; +esac +fi + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + hp) : + + + if ac_fn_cxx_compute_int "$LINENO" "!!( + #if defined(__HP_cc) + __HP_cc + #else + __HP_aCC + #endif + <= 1)" "_ax_cxx_compiler_version_untilA0121" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_HP unknown hp release version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if test "X$_ax_cxx_compiler_version_untilA0121" = X1 +then : + ax_cv_cxx_compiler_version="01.21.00" + +else case e in #( + e) if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__HP_cc) + __HP_cc + #else + __HP_aCC + #endif + % 100)" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_HP unknown hp release version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(( + #if defined(__HP_cc) + __HP_cc + #else + __HP_aCC + #endif + / 100)%100)" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_HP unknown hp minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(( + #if defined(__HP_cc) + __HP_cc + #else + __HP_aCC + #endif + / 10000)%100)" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_HP unknown hp major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; +esac +fi + ;; #( + dec) : + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__DECC_VER) + __DECC_VER + #else + __DECCXX_VER + #endif + % 10000)" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_DEC unknown dec release version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(( + #if defined(__DECC_VER) + __DECC_VER + #else + __DECCXX_VER + #endif + / 100000UL)%100)" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_DEC unknown dec minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(( + #if defined(__DECC_VER) + __DECC_VER + #else + __DECCXX_VER + #endif + / 10000000UL)%100)" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_DEC unknown dec major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + borland) : + + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + #if defined(__TURBOC__) + __TURBOC__ + #else + choke me + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + if ac_fn_cxx_compute_int "$LINENO" " + #if defined(__TURBOC__) + __TURBOC__ + #else + choke me + #endif + " "_ax_cxx_compiler_version_turboc_raw" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_BORLAND unknown turboc version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if test $_ax_cxx_compiler_version_turboc_raw -lt 661 || test $_ax_cxx_compiler_version_turboc_raw -gt 1023 +then : + if ac_fn_cxx_compute_int "$LINENO" " + #if defined(__TURBOC__) + __TURBOC__ + #else + choke me + #endif + % 0x100" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_BORLAND unknown turboc minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(__TURBOC__) + __TURBOC__ + #else + choke me + #endif + /0x100)%0x100" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_BORLAND unknown turboc major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="0turboc:$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor" +else case e in #( + e) case $_ax_cxx_compiler_version_turboc_raw in #( + 661) : + ax_cv_cxx_compiler_version="0turboc:1.00" ;; #( + 662) : + ax_cv_cxx_compiler_version="0turboc:1.01" ;; #( + 663) : + ax_cv_cxx_compiler_version="0turboc:2.00" ;; #( + *) : + + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: [_AX_COMPILER_VERSION_BORLAND] unknown turboc version between 0x295 and 0x400 please report bug" >&5 +printf '%s\n' "$as_me: WARNING: [_AX_COMPILER_VERSION_BORLAND] unknown turboc version between 0x295 and 0x400 please report bug" >&2;} + ax_cv_cxx_compiler_version="" + ;; +esac + ;; +esac +fi + +else case e in #( + e) # borlandc + + if ac_fn_cxx_compute_int "$LINENO" " + #if defined(__BORLANDC__) + __BORLANDC__ + #else + __CODEGEARC__ + #endif + " "_ax_cxx_compiler_version_borlandc_raw" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_BORLAND unknown borlandc version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + case $_ax_cxx_compiler_version_borlandc_raw in #( + 512 ) : + ax_cv_cxx_compiler_version="1borlanc:2.00" ;; #( + 1024) : + ax_cv_cxx_compiler_version="1borlanc:3.00" ;; #( + 1024) : + ax_cv_cxx_compiler_version="1borlanc:3.00" ;; #( + 1040) : + ax_cv_cxx_compiler_version="1borlanc:3.1" ;; #( + 1106) : + ax_cv_cxx_compiler_version="1borlanc:4.0" ;; #( + 1280) : + ax_cv_cxx_compiler_version="1borlanc:5.0" ;; #( + 1312) : + ax_cv_cxx_compiler_version="1borlanc:5.02" ;; #( + 1328) : + ax_cv_cxx_compiler_version="2cppbuilder:3.0" ;; #( + 1344) : + ax_cv_cxx_compiler_version="2cppbuilder:4.0" ;; #( + 1360) : + ax_cv_cxx_compiler_version="3borlancpp:5.5" ;; #( + 1361) : + ax_cv_cxx_compiler_version="3borlancpp:5.51" ;; #( + 1378) : + ax_cv_cxx_compiler_version="3borlancpp:5.6.4" ;; #( + 1392) : + ax_cv_cxx_compiler_version="4cppbuilder:2006" ;; #( + 1424) : + ax_cv_cxx_compiler_version="4cppbuilder:2007" ;; #( + 1555) : + ax_cv_cxx_compiler_version="4cppbuilder:2009" ;; #( + 1569) : + ax_cv_cxx_compiler_version="4cppbuilder:2010" ;; #( + 1584) : + ax_cv_cxx_compiler_version="5xe" ;; #( + 1600) : + ax_cv_cxx_compiler_version="5xe:2" ;; #( + 1616) : + ax_cv_cxx_compiler_version="5xe:3" ;; #( + 1632) : + ax_cv_cxx_compiler_version="5xe:4" ;; #( + *) : + + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: [_AX_COMPILER_VERSION_BORLAND] Unknown borlandc compiler version $_ax_cxx_compiler_version_borlandc_raw please report bug" >&5 +printf '%s\n' "$as_me: WARNING: [_AX_COMPILER_VERSION_BORLAND] Unknown borlandc compiler version $_ax_cxx_compiler_version_borlandc_raw please report bug" >&2;} + ;; +esac + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; #( + comeau) : + if ac_fn_cxx_compute_int "$LINENO" "__COMO_VERSION__%100" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_COMEAU unknown comeau compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__COMO_VERSION__/100)%10" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_COMEAU unknown comeau compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor" + ;; #( + kai) : + + if ac_fn_cxx_compute_int "$LINENO" "__KCC_VERSION%100" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_KAI unknown kay compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__KCC_VERSION/100)%10" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_KAI unknown kay compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__KCC_VERSION/1000)%10" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_KAI unknown kay compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + sgi) : + + + if ac_fn_cxx_compute_int "$LINENO" " + #if defined(_COMPILER_VERSION) + _COMPILER_VERSION + #else + _SGI_COMPILER_VERSION + #endif + %10" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SGI unknown SGI compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(_COMPILER_VERSION) + _COMPILER_VERSION + #else + _SGI_COMPILER_VERSION + #endif + /10)%10" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SGI unknown SGI compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi - ; - return 0; -} -_ACEOF -if ac_fn_cxx_try_compile "$LINENO" + if ac_fn_cxx_compute_int "$LINENO" "( + #if defined(_COMPILER_VERSION) + _COMPILER_VERSION + #else + _SGI_COMPILER_VERSION + #endif + /100)%10" "_ax_cxx_compiler_version_major" "" then : else case e in #( - e) ac_cxx_werror_flag=$ac_save_cxx_werror_flag - CXXFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SGI unknown SGI compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi -int -main (void) -{ + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + microsoft) : - ; - return 0; -} -_ACEOF -if ac_fn_cxx_try_compile "$LINENO" + if ac_fn_cxx_compute_int "$LINENO" "_MSC_VER%100" "_ax_cxx_compiler_version_minor" "" then : - ac_cv_prog_cxx_g=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_MICROSOFT unknown microsoft compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; + + if ac_fn_cxx_compute_int "$LINENO" "(_MSC_VER/100)%100" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_MICROSOFT unknown microsoft compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_cxx_werror_flag=$ac_save_cxx_werror_flag ;; + + _ax_cxx_compiler_version_patch=0 + _ax_cxx_compiler_version_build=0 + # special case for version 6 + if test "X$_ax_cxx_compiler_version_major" = "X12" +then : + if ac_fn_cxx_compute_int "$LINENO" "_MSC_FULL_VER%1000" "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) _ax_cxx_compiler_version_patch=0 ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_g" >&5 -printf "%s\n" "$ac_cv_prog_cxx_g" >&6; } -if test $ac_test_CXXFLAGS; then - CXXFLAGS=$ac_save_CXXFLAGS -elif test $ac_cv_prog_cxx_g = yes; then - if test "$GXX" = yes; then - CXXFLAGS="-g -O2" - else - CXXFLAGS="-g" - fi -else - if test "$GXX" = yes; then - CXXFLAGS="-O2" - else - CXXFLAGS= - fi + fi -ac_prog_cxx_stdcxx=no -if test x$ac_prog_cxx_stdcxx = xno + # for version 7 + if test "X$_ax_cxx_compiler_version_major" = "X13" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++11 features" >&5 -printf %s "checking for $CXX option to enable C++11 features... " >&6; } -if test ${ac_cv_prog_cxx_cxx11+y} + if ac_fn_cxx_compute_int "$LINENO" "_MSC_FULL_VER%1000" "_ax_cxx_compiler_version_patch" "" then : - printf %s "(cached) " >&6 + else case e in #( - e) ac_cv_prog_cxx_cxx11=no -ac_save_CXX=$CXX -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_cxx_conftest_cxx11_program -_ACEOF -for ac_arg in '' -std=gnu++11 -std=gnu++0x -std=c++11 -std=c++0x -qlanglvl=extended0x -AA -do - CXX="$ac_save_CXX $ac_arg" - if ac_fn_cxx_try_compile "$LINENO" -then : - ac_cv_prog_cxx_cxx11=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cxx_cxx11" != "xno" && break -done -rm -f conftest.$ac_ext -CXX=$ac_save_CXX ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_MICROSOFT unknown microsoft compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi -if test "x$ac_cv_prog_cxx_cxx11" = xno + +fi + # for version > 8 + if test $_ax_cxx_compiler_version_major -ge 14 then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } -else case e in #( - e) if test "x$ac_cv_prog_cxx_cxx11" = x + if ac_fn_cxx_compute_int "$LINENO" "_MSC_FULL_VER%10000" "_ax_cxx_compiler_version_patch" "" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } + else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_cxx11" >&5 -printf "%s\n" "$ac_cv_prog_cxx_cxx11" >&6; } - CXX="$CXX $ac_cv_prog_cxx_cxx11" ;; -esac -fi - ac_cv_prog_cxx_stdcxx=$ac_cv_prog_cxx_cxx11 - ac_prog_cxx_stdcxx=cxx11 ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_MICROSOFT unknown microsoft compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + + fi -if test x$ac_prog_cxx_stdcxx = xno + if test $_ax_cxx_compiler_version_major -ge 15 then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++98 features" >&5 -printf %s "checking for $CXX option to enable C++98 features... " >&6; } -if test ${ac_cv_prog_cxx_cxx98+y} + if ac_fn_cxx_compute_int "$LINENO" "_MSC_BUILD" "_ax_cxx_compiler_version_build" "" then : - printf %s "(cached) " >&6 + else case e in #( - e) ac_cv_prog_cxx_cxx98=no -ac_save_CXX=$CXX -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_cxx_conftest_cxx98_program -_ACEOF -for ac_arg in '' -std=gnu++98 -std=c++98 -qlanglvl=extended -AA -do - CXX="$ac_save_CXX $ac_arg" - if ac_fn_cxx_try_compile "$LINENO" -then : - ac_cv_prog_cxx_cxx98=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cxx_cxx98" != "xno" && break -done -rm -f conftest.$ac_ext -CXX=$ac_save_CXX ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_MICROSOFT unknown microsoft compiler build version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi -if test "x$ac_cv_prog_cxx_cxx98" = xno + +fi + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch.$_ax_cxx_compiler_version_build" + ;; #( + metrowerks) : + if ac_fn_cxx_compute_int "$LINENO" "__MWERKS__%0x100" "_ax_cxx_compiler_version_patch" "" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } + else case e in #( - e) if test "x$ac_cv_prog_cxx_cxx98" = x + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_METROWERKS unknown metrowerks compiler patch version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "(__MWERKS__/0x100)%0x10" "_ax_cxx_compiler_version_minor" "" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } + else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cxx_cxx98" >&5 -printf "%s\n" "$ac_cv_prog_cxx_cxx98" >&6; } - CXX="$CXX $ac_cv_prog_cxx_cxx98" ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_METROWERKS unknown metrowerks compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi - ac_cv_prog_cxx_stdcxx=$ac_cv_prog_cxx_cxx98 - ac_prog_cxx_stdcxx=cxx98 ;; + + if ac_fn_cxx_compute_int "$LINENO" "(__MWERKS__/0x1000)%0x10" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_METROWERKS unknown metrowerks compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + watcom) : + if ac_fn_cxx_compute_int "$LINENO" "__WATCOMC__%100" "_ax_cxx_compiler_version_minor" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_WATCOM unknown watcom compiler minor version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac fi -ac_ext=cpp -ac_cpp='$CXXCPP $CPPFLAGS' -ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + if ac_fn_cxx_compute_int "$LINENO" "(__WATCOMC__/100)%100" "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_WATCOM unknown watcom compiler major version +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor" + ;; #( + nvhpc) : -# Check if compiling debug version -# Check whether --enable-debug was given. -if test ${enable_debug+y} + if ac_fn_cxx_compute_int "$LINENO" "__NVCOMPILER_MAJOR__" "_ax_cxx_compiler_version_major" "" then : - enableval=$enable_debug; build_debug=yes + else case e in #( - e) build_debug=no ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_NVHPC unknown nvhpc major +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + if ac_fn_cxx_compute_int "$LINENO" "__NVCOMPILER_MINOR__" "_ax_cxx_compiler_version_minor" "" +then : -# Check if running C++ tests (this adds to compilation time) -# Check whether --enable-cpp-tests was given. -if test ${enable_cpp_tests+y} +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_NVHPC unknown nvhpc minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__NVCOMPILER_PATCHLEVEL__" "_ax_cxx_compiler_version_patch" "" then : - enableval=$enable_cpp_tests; case "${enableval}" in - no) cpp_tests_disable=yes ;; - yes) cpp_tests_disable=no ;; - *) as_fn_error $? "--enable-cpp-tests understads only 'no' value" "$LINENO" 5 ;; - esac + else case e in #( - e) cpp_tests_disable=no ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_NVHPC unknown nvhpc patch level +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + portland) : -# Check if compiler warnings should be turned on -# Check whether --enable-warnings was given. -if test ${enable_warnings+y} + if ac_fn_cxx_compute_int "$LINENO" "__PGIC__" "_ax_cxx_compiler_version_major" "" then : - enableval=$enable_warnings; warnings=yes + else case e in #( - e) warnings=no ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PORTLAND unknown pgi major +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + if ac_fn_cxx_compute_int "$LINENO" "__PGIC_MINOR__" "_ax_cxx_compiler_version_minor" "" +then : -# Use SIMD unless requested not to -# Check whether --enable-simd was given. -if test ${enable_simd+y} +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PORTLAND unknown pgi minor +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi + + if ac_fn_cxx_compute_int "$LINENO" "__PGIC_PATCHLEVEL__" "_ax_cxx_compiler_version_patch" "" then : - enableval=$enable_simd; use_simd=$enableval + else case e in #( - e) use_simd=yes ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_PORTLAND unknown pgi patch level +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + tcc) : -# default CoGAPS specific flags -GAPS_CPP_FLAGS=" -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=0 -DGAPS_DISABLE_CHECKPOINTS -D__GAPS_R_BUILD__" -GAPS_CXX_FLAGS= -GAPS_LIBS= + ax_cv_cxx_compiler_version=`tcc -v | $SED 's/^[ ]*tcc[ ]\+version[ ]\+\([0-9.]\+\).*/\1/g'` + ;; #( + sdcc) : -# get compiler info -AX_COMPILER_VENDOR -AX_COMPILER_VERSION + if ac_fn_cxx_compute_int "$LINENO" "/* avoid parse error with comments */ + #if(defined(__SDCC_VERSION_MAJOR)) + __SDCC_VERSION_MAJOR + #else + SDCC/100 + #endif + " "_ax_cxx_compiler_version_major" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SDCC unknown sdcc major +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi -# set openmp flags, disable only if requested -# Check whether --enable-openmp was given. -if test ${enable_openmp+y} + if ac_fn_cxx_compute_int "$LINENO" "/* avoid parse error with comments */ + #if(defined(__SDCC_VERSION_MINOR)) + __SDCC_VERSION_MINOR + #else + (SDCC%100)/10 + #endif + " "_ax_cxx_compiler_version_minor" "" then : - enableval=$enable_openmp; use_openmp=$enableval + else case e in #( - e) use_openmp=yes ;; + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SDCC unknown sdcc minor +See 'config.log' for more details" "$LINENO" 5; } ;; esac fi + if ac_fn_cxx_compute_int "$LINENO" " + /* avoid parse error with comments */ + #if(defined(__SDCC_VERSION_PATCH)) + __SDCC_VERSION_PATCH + #elsif(defined(_SDCC_VERSION_PATCHLEVEL)) + __SDCC_VERSION_PATCHLEVEL + #else + SDCC%10 + #endif + " "_ax_cxx_compiler_version_patch" "" +then : + +else case e in #( + e) { { printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf '%s\n' "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "_AX_COMPILER_VERSION_SDCC unknown sdcc patch level +See 'config.log' for more details" "$LINENO" 5; } ;; +esac +fi -AX_OPENMP -if test "x$use_openmp" != "xno" ; then - GAPS_CXX_FLAGS+=" $OPENMP_CXXFLAGS " - GAPS_LIBS+=" $OPENMP_CXXFLAGS " + ax_cv_cxx_compiler_version="$_ax_cxx_compiler_version_major.$_ax_cxx_compiler_version_minor.$_ax_cxx_compiler_version_patch" + ;; #( + *) : + ax_cv_cxx_compiler_version="" ;; +esac + ;; +esac fi +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: result: $ax_cv_cxx_compiler_version" >&5 +printf '%s\n' "$ax_cv_cxx_compiler_version" >&6; } + + +# OpenMP removed together with the asynchronous sampler (it broke MCMC balance). echo "building on $ax_cv_cxx_compiler_vendor compiler version $ax_cv_cxx_compiler_version" -# set compile flags for debug build +# set -g -O0 for debug options if test "x$build_debug" = "xyes" ; then + echo "Building CoGAPS with debug information" + GAPS_CPP_FLAGS+=" -g -O0 " +fi + +# set GAPS_DEBUG for debug build +if test "x$build_gaps_debug" = "xyes" ; then echo "Building Debug Version of CoGAPS" GAPS_CPP_FLAGS+=" -DGAPS_DEBUG " fi @@ -3338,10 +4514,7 @@ GAPS_SOURCE_FILES+=" GapsStatistics.o" GAPS_SOURCE_FILES+=" RcppExports.o" GAPS_SOURCE_FILES+=" test-runner.o" GAPS_SOURCE_FILES+=" atomic/Atom.o" -GAPS_SOURCE_FILES+=" atomic/ConcurrentAtom.o" GAPS_SOURCE_FILES+=" atomic/AtomicDomain.o" -GAPS_SOURCE_FILES+=" atomic/ConcurrentAtomicDomain.o" -GAPS_SOURCE_FILES+=" atomic/ProposalQueue.o" GAPS_SOURCE_FILES+=" data_structures/HashSets.o" GAPS_SOURCE_FILES+=" data_structures/HybridMatrix.o" GAPS_SOURCE_FILES+=" data_structures/HybridVector.o" @@ -3365,14 +4538,23 @@ GAPS_SOURCE_FILES+=" math/VectorMath.o" # add c++ tests to source list if test "x$cpp_tests_disable" != "xyes" ; then echo "Enabling C++ Unit Tests" - GAPS_CPP_FLAGS+=" -DGAPS_CPP_UNIT_TESTS " - GAPS_SOURCE_FILES+=" cpp_tests/testVector.o" + GAPS_CPP_FLAGS+=" -DGAPS_VCPP_UNIT_TESTS " + GAPS_SOURCE_FILES+=" cpp_tests/testAtomicDomain.o" + GAPS_SOURCE_FILES+=" cpp_tests/testDenseGibbsSampler.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseGibbsSampler.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSamplerHighLevel.o" + GAPS_SOURCE_FILES+=" cpp_tests/testFileParsers.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHashSets.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHybridMatrix.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHybridVector.o" + GAPS_SOURCE_FILES+=" cpp_tests/testMathHelpers.o" + GAPS_SOURCE_FILES+=" cpp_tests/testMatrix.o" + GAPS_SOURCE_FILES+=" cpp_tests/testRandom.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseMatrix.o" GAPS_SOURCE_FILES+=" cpp_tests/testSparseVector.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHashSets.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHybridMatrix.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHybridVector.o" - GAPS_SOURCE_FILES+=" cpp_tests/testSparseIterator.o" -# GAPS_SOURCE_FILES+=" cpp_tests/testConcurrentAtomicDomain.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseIterator.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSerialization.o" + GAPS_SOURCE_FILES+=" cpp_tests/testVector.o" fi # commented files in the if above are to be reviwed @@ -3383,6 +4565,12 @@ fi +echo "Writing src/Makevars" +echo Echoing GAPS_CPP_FLAGS: +echo $GAPS_CPP_FLAGS +echo Echoing GAPS_CXX_FLAGS: +echo $GAPS_CXX_FLAGS + # create makefile, output configure file ac_config_files="$ac_config_files src/Makevars" @@ -3402,44 +4590,7 @@ cat >confcache <<\_ACEOF _ACEOF -# The following way of writing the cache mishandles newlines in values, -# but we know of no workaround that is simple, portable, and efficient. -# So, we kill variables containing newlines. -# Ultrix sh set writes to stderr and can't be redirected directly, -# and sets the high bit in the cache file unless we assign to the vars. -( - for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do - eval ac_val=\$$ac_var - case $ac_val in #( - *${as_nl}*) - case $ac_var in #( - *_cv_*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 -printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; - esac - case $ac_var in #( - _ | IFS | as_nl) ;; #( - BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( - *) { eval $ac_var=; unset $ac_var;} ;; - esac ;; - esac - done - - (set) 2>&1 | - case $as_nl`(ac_space=' '; set) 2>&1` in #( - *${as_nl}ac_space=\ *) - # 'set' does not quote correctly, so add quotes: double-quote - # substitution turns \\\\ into \\, and sed turns \\ into \. - sed -n \ - "s/'/'\\\\''/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" - ;; #( - *) - # 'set' quotes correctly as required by POSIX, so do not add quotes. - sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" - ;; - esac | - sort -) | +ac_cache_dump | sed ' /^ac_cv_env_/b end t clear @@ -3451,8 +4602,8 @@ printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} if diff "$cache_file" confcache >/dev/null 2>&1; then :; else if test -w "$cache_file"; then if test "x$cache_file" != "x/dev/null"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 -printf "%s\n" "$as_me: updating cache $cache_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 +printf '%s\n' "$as_me: updating cache $cache_file" >&6;} if test ! -f "$cache_file" || test -h "$cache_file"; then cat confcache >"$cache_file" else @@ -3466,8 +4617,8 @@ printf "%s\n" "$as_me: updating cache $cache_file" >&6;} fi fi else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 -printf "%s\n" "$as_me: not updating unwritable cache $cache_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 +printf '%s\n' "$as_me: not updating unwritable cache $cache_file" >&6;} fi fi rm -f confcache @@ -3518,7 +4669,7 @@ U= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' - ac_i=`printf "%s\n" "$ac_i" | sed "$ac_script"` + ac_i=`printf '%s\n' "$ac_i" | sed "$ac_script"` # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR # will be set to the directory where LIBOBJS objects are built. as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" @@ -3531,13 +4682,21 @@ LTLIBOBJS=$ac_ltlibobjs : "${CONFIG_STATUS=./config.status}" +case $CONFIG_STATUS in #( + -*) : + CONFIG_STATUS=./$CONFIG_STATUS ;; #( + */*) : + ;; #( + *) : + CONFIG_STATUS=./$CONFIG_STATUS ;; +esac + ac_write_fail=0 -ac_clean_files_save=$ac_clean_files -ac_clean_files="$ac_clean_files $CONFIG_STATUS" -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 -printf "%s\n" "$as_me: creating $CONFIG_STATUS" >&6;} +ac_clean_CONFIG_STATUS='"$CONFIG_STATUS"' +{ printf '%s\n' "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 +printf '%s\n' "$as_me: creating $CONFIG_STATUS" >&6;} as_write_fail=0 -cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 +cat >"$CONFIG_STATUS" <<_ASEOF || as_write_fail=1 #! $SHELL # Generated by $as_me. # Run this file to recreate the current configuration. @@ -3551,7 +4710,7 @@ ac_cs_silent=false SHELL=\${CONFIG_SHELL-$SHELL} export SHELL _ASEOF -cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ASEOF || as_write_fail=1 ## -------------------- ## ## M4sh Initialization. ## ## -------------------- ## @@ -3563,7 +4722,7 @@ then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. + # contradicts POSIX and common usage. Disable this. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case e in #( @@ -3650,7 +4809,7 @@ if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then - printf "%s\n" "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 + printf '%s\n' "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi @@ -3666,9 +4825,9 @@ as_fn_error () as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + printf '%s\n' "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - printf "%s\n" "$as_me: error: $2" >&2 + printf '%s\n' "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -3763,7 +4922,7 @@ as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X/"$0" | +printf '%s\n' X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q @@ -3785,29 +4944,6 @@ as_cr_Letters=$as_cr_letters$as_cr_LETTERS as_cr_digits='0123456789' as_cr_alnum=$as_cr_Letters$as_cr_digits - -# Determine whether it's possible to make 'echo' print without a newline. -# These variables are no longer used directly by Autoconf, but are AC_SUBSTed -# for compatibility with existing Makefiles. -ECHO_C= ECHO_N= ECHO_T= -case `echo -n x` in #((((( --n*) - case `echo 'xy\c'` in - *c*) ECHO_T=' ';; # ECHO_T is single tab character. - xy) ECHO_C='\c';; - *) echo `echo ksh88 bug on AIX 6.1` > /dev/null - ECHO_T=' ';; - esac;; -*) - ECHO_N='-n';; -esac - -# For backward compatibility with old third-party macros, we provide -# the shell variables $as_echo and $as_echo_n. New code should use -# AS_ECHO(["message"]) and AS_ECHO_N(["message"]), respectively. -as_echo='printf %s\n' -as_echo_n='printf %s' - rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file @@ -3849,7 +4985,7 @@ as_fn_mkdir_p () as_dirs= while :; do case $as_dir in #( - *\'*) as_qdir=`printf "%s\n" "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( + *\'*) as_qdir=`printf '%s\n' "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" @@ -3858,7 +4994,7 @@ $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X"$as_dir" | +printf '%s\n' X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q @@ -3911,19 +5047,19 @@ as_tr_sh="eval sed '$as_sed_sh'" # deprecated exec 6>&1 -## ----------------------------------- ## -## Main body of $CONFIG_STATUS script. ## -## ----------------------------------- ## +## ------------------------------------- ## +## Main body of "$CONFIG_STATUS" script. ## +## ------------------------------------- ## _ASEOF -test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 +test $as_write_fail = 0 && chmod +x "$CONFIG_STATUS" || ac_write_fail=1 -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by CoGAPS $as_me 3.25.1, which was -generated by GNU Autoconf 2.72. Invocation command line was +This file was extended by CoGAPS $as_me 3.33.2, which was +generated by GNU Autoconf 2.73. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -3942,13 +5078,13 @@ esac -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 # Files that config.status was made for. config_files="$ac_config_files" _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 ac_cs_usage="\ '$as_me' instantiates files and other configuration actions from templates according to the current configuration. Unless the files @@ -3972,25 +5108,29 @@ $config_files Report bugs to the package provider." _ACEOF -ac_cs_config=`printf "%s\n" "$ac_configure_args" | sed "$ac_safe_unquote"` -ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\''/g"` -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +ac_cs_config=`printf '%s\n' "$ac_configure_args" | sed "$ac_safe_unquote"` +ac_cs_config_escaped=`printf '%s\n' "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\''/g"` +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -CoGAPS config.status 3.25.1 -configured by $0, generated by GNU Autoconf 2.72, +CoGAPS config.status 3.33.2 +configured by $0, generated by GNU Autoconf 2.73, with options \\"\$ac_cs_config\\" -Copyright (C) 2023 Free Software Foundation, Inc. +Copyright (C) 2026 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." ac_pwd='$ac_pwd' srcdir='$srcdir' -test -n "\$AWK" || AWK=awk +test -n "\$AWK" || { + awk '' >$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 # The default lists apply if the user does not specify any file. ac_need_defaults=: while test $# != 0 @@ -4018,21 +5158,21 @@ do -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) ac_cs_recheck=: ;; --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) - printf "%s\n" "$ac_cs_version"; exit ;; + printf '%s\n' "$ac_cs_version"; exit ;; --config | --confi | --conf | --con | --co | --c ) - printf "%s\n" "$ac_cs_config"; exit ;; + printf '%s\n' "$ac_cs_config"; exit ;; --debug | --debu | --deb | --de | --d | -d ) debug=: ;; --file | --fil | --fi | --f ) $ac_shift case $ac_optarg in - *\'*) ac_optarg=`printf "%s\n" "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + *\'*) ac_optarg=`printf '%s\n' "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; --he | --h | --help | --hel | -h ) - printf "%s\n" "$ac_cs_usage"; exit ;; + printf '%s\n' "$ac_cs_usage"; exit ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil | --si | --s) ac_cs_silent=: ;; @@ -4056,32 +5196,32 @@ if $ac_cs_silent; then fi _ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 if \$ac_cs_recheck; then set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion shift - \printf "%s\n" "running CONFIG_SHELL=$SHELL \$*" >&6 + \printf '%s\n' "running CONFIG_SHELL=$SHELL \$*" >&6 CONFIG_SHELL='$SHELL' export CONFIG_SHELL exec "\$@" fi _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 exec 5>>config.log { echo sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX ## Running $as_me. ## _ASBOX - printf "%s\n" "$ac_log" + printf '%s\n' "$ac_log" } >&5 _ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 # Handling of arguments. for ac_config_target in $ac_config_targets @@ -4159,13 +5299,13 @@ _ACEOF echo "_ACEOF" } >conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 -ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` +ac_delim_num=`echo "$ac_subst_vars" | sed -n '$='` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 - ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` + ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | sed -n '$='` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then @@ -4176,7 +5316,7 @@ for ac_last_try in false false false false false :; do done rm -f conf$$subs.sh -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && _ACEOF sed -n ' @@ -4221,9 +5361,9 @@ t delim N s/\n// } -' >>$CONFIG_STATUS || ac_write_fail=1 +' >>"$CONFIG_STATUS" || ac_write_fail=1 rm -f conf$$subs.awk -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 _ACAWK cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && for (key in S) S_is_set[key] = 1 @@ -4252,7 +5392,7 @@ cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && _ACAWK _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" else @@ -4284,7 +5424,7 @@ s/^[^=]*=[ ]*$// }' fi -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 fi # test -n "$CONFIG_FILES" @@ -4327,7 +5467,7 @@ do esac || as_fn_error 1 "cannot find input file: '$ac_f'" "$LINENO" 5;; esac - case $ac_f in *\'*) ac_f=`printf "%s\n" "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac + case $ac_f in *\'*) ac_f=`printf '%s\n' "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" done @@ -4335,17 +5475,17 @@ do # use $as_me), people would be surprised to read: # /* config.h. Generated by config.status. */ configure_input='Generated from '` - printf "%s\n" "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' + printf '%s\n' "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' `' by configure.' if test x"$ac_file" != x-; then configure_input="$ac_file. $configure_input" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 -printf "%s\n" "$as_me: creating $ac_file" >&6;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 +printf '%s\n' "$as_me: creating $ac_file" >&6;} fi # Neutralize special characters interpreted by sed in replacement strings. case $configure_input in #( *\&* | *\|* | *\\* ) - ac_sed_conf_input=`printf "%s\n" "$configure_input" | + ac_sed_conf_input=`printf '%s\n' "$configure_input" | sed 's/[\\\\&|]/\\\\&/g'`;; #( *) ac_sed_conf_input=$configure_input;; esac @@ -4362,7 +5502,7 @@ $as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$ac_file" : 'X\(//\)[^/]' \| \ X"$ac_file" : 'X\(//\)$' \| \ X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || -printf "%s\n" X"$ac_file" | +printf '%s\n' X"$ac_file" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q @@ -4386,9 +5526,9 @@ printf "%s\n" X"$ac_file" | case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) - ac_dir_suffix=/`printf "%s\n" "$ac_dir" | sed 's|^\.[\\/]||'` + ac_dir_suffix=/`printf '%s\n' "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. - ac_top_builddir_sub=`printf "%s\n" "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` + ac_top_builddir_sub=`printf '%s\n' "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; @@ -4424,7 +5564,7 @@ ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 # If the template does not know about datarootdir, expand it. # FIXME: This hack should be removed a few years after 2.60. ac_datarootdir_hack=; ac_datarootdir_seen= @@ -4441,10 +5581,10 @@ ac_sed_dataroot=' case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in *datarootdir*) ac_datarootdir_seen=yes;; *@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 -printf "%s\n" "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 +printf '%s\n' "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} _ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 ac_datarootdir_hack=' s&@datadir@&$datadir&g s&@docdir@&$docdir&g @@ -4458,11 +5598,11 @@ _ACEOF # Neutralize VPATH when '$srcdir' = '.'. # Shell code in configure.ac might set extrasub. # FIXME: do we really want to maintain this feature? -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<_ACEOF || ac_write_fail=1 ac_sed_extra="$ac_vpsub $extrasub _ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 +cat >>"$CONFIG_STATUS" <<\_ACEOF || ac_write_fail=1 :t /@[a-zA-Z_][a-zA-Z_0-9]*@/!b s|@configure_input@|$ac_sed_conf_input|;t t @@ -4484,9 +5624,9 @@ test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ "$ac_tmp/out"`; test -z "$ac_out"; } && - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable 'datarootdir' + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&5 -printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable 'datarootdir' +printf '%s\n' "$as_me: WARNING: $ac_file contains a reference to the variable 'datarootdir' which seems to be undefined. Please make sure it is defined" >&2;} rm -f "$ac_tmp/stdin" @@ -4506,7 +5646,7 @@ done # for ac_tag as_fn_exit 0 _ACEOF -ac_clean_files=$ac_clean_files_save +ac_clean_CONFIG_STATUS= test $ac_write_fail = 0 || as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 @@ -4522,19 +5662,26 @@ test $ac_write_fail = 0 || # need to make the FD available again. if test "$no_create" != yes; then ac_cs_success=: + case $CONFIG_STATUS in #( + -*) : + ac_no_opts=-- ;; #( + *) : + ac_no_opts= ;; +esac ac_config_status_args= test "$silent" = yes && ac_config_status_args="$ac_config_status_args --quiet" exec 5>/dev/null - $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false + $SHELL $ac_no_opts "$CONFIG_STATUS" $ac_config_status_args || + ac_cs_success=false exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. $ac_cs_success || as_fn_exit 1 fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 -printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} + { printf '%s\n' "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 +printf '%s\n' "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi diff --git a/configure.ac b/configure.ac index 6b6df1ee..f7f1fc06 100644 --- a/configure.ac +++ b/configure.ac @@ -9,17 +9,44 @@ AC_LANG(C++) AC_REQUIRE_CPP AC_PROG_CXX -# Check if compiling debug version -AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[build debug version of CoGAPS])], [build_debug=yes], [build_debug=no]) +# Check if compiling gaps debug version; actuaooly, it includes broken tests and swithcing on asserts; +# we switched on the asserts unconditionally +AC_ARG_ENABLE(gaps-debug, [AS_HELP_STRING([--enable-gaps-debug],[build debug version of CoGAPS])], + [case "${enableval}" in + yes) build_gaps_debug=yes ;; + no) build_gaps_debug=no ;; + *) AC_MSG_ERROR([--enable-gaps-debug understads only 'no' or 'yes' values]) ;; + esac],[build_gaps_debug=no]) + + + +#Switches on debug cpp flags -g -O0 +AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enables-debug],[build CoGAPS with debug options])], + [case "${enableval}" in + yes) build_debug=yes ;; + no) build_debug=no ;; + *) AC_MSG_ERROR([--enable-debug understads only 'no' or 'yes' values]) ;; + esac],[build_debug=no]) + # Check if running C++ tests (this adds to compilation time) AC_ARG_ENABLE(cpp-tests, [AS_HELP_STRING([--enable-cpp-tests],[turn on C++ unit tests])], [case "${enableval}" in no) cpp_tests_disable=yes ;; yes) cpp_tests_disable=no ;; - *) AC_MSG_ERROR([--enable-cpp-tests understads only 'no' value]) ;; - esac],[cpp_tests_disable=no]) - + *) AC_MSG_ERROR([--enable-cpp-tests understads only 'no' or 'yes' values]) ;; + esac],[cpp_tests_disable=no]) + +# Check if checkpoint (save/resume) support should be compiled in. Checkpoints are +# an emergency/debug feature that serialize state on every run; off by default, +# enable with --enable-checkpoints for package debugging. +AC_ARG_ENABLE(checkpoints, [AS_HELP_STRING([--enable-checkpoints],[turn on checkpoint (save/resume) support])], + [case "${enableval}" in + no) cpp_checkpoints_disable=yes ;; + yes) cpp_checkpoints_disable=no ;; + *) AC_MSG_ERROR([--enable-checkpoints understands only 'no' or 'yes' values]) ;; + esac],[cpp_checkpoints_disable=yes]) + # Check if compiler warnings should be turned on AC_ARG_ENABLE(warnings, [AS_HELP_STRING([--enable-warnings],[compile CoGAPS with warning messages])], [warnings=yes], [warnings=no]) @@ -28,28 +55,31 @@ AC_ARG_ENABLE(simd, [AS_HELP_STRING([--enable-simd],[compile with SIMD support i [use_simd=$enableval], [use_simd=yes]) # default CoGAPS specific flags -GAPS_CPP_FLAGS=" -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=0 -DGAPS_DISABLE_CHECKPOINTS -D__GAPS_R_BUILD__" +GAPS_CPP_FLAGS=" -DBOOST_MATH_PROMOTE_DOUBLE_POLICY=0 -D__GAPS_R_BUILD__" GAPS_CXX_FLAGS= GAPS_LIBS= +# checkpoints are off by default; define the disabling macro unless --enable-checkpoints +if test "x$cpp_checkpoints_disable" != "xno" ; then + GAPS_CPP_FLAGS+=" -DGAPS_DISABLE_CHECKPOINTS " +fi + # get compiler info AX_COMPILER_VENDOR AX_COMPILER_VERSION -# set openmp flags, disable only if requested -AC_ARG_ENABLE(openmp, [AS_HELP_STRING([--enable-openmp],[compile with openMP support if available])], - [use_openmp=$enableval], [use_openmp=yes]) - -AX_OPENMP -if test "x$use_openmp" != "xno" ; then - GAPS_CXX_FLAGS+=" $OPENMP_CXXFLAGS " - GAPS_LIBS+=" $OPENMP_CXXFLAGS " -fi +# OpenMP removed together with the asynchronous sampler (it broke MCMC balance). echo "building on $ax_cv_cxx_compiler_vendor compiler version $ax_cv_cxx_compiler_version" -# set compile flags for debug build +# set -g -O0 for debug options if test "x$build_debug" = "xyes" ; then + echo "Building CoGAPS with debug information" + GAPS_CPP_FLAGS+=" -g -O0 " +fi + +# set GAPS_DEBUG for debug build +if test "x$build_gaps_debug" = "xyes" ; then echo "Building Debug Version of CoGAPS" GAPS_CPP_FLAGS+=" -DGAPS_DEBUG " fi @@ -78,10 +108,7 @@ GAPS_SOURCE_FILES+=" GapsStatistics.o" GAPS_SOURCE_FILES+=" RcppExports.o" GAPS_SOURCE_FILES+=" test-runner.o" GAPS_SOURCE_FILES+=" atomic/Atom.o" -GAPS_SOURCE_FILES+=" atomic/ConcurrentAtom.o" GAPS_SOURCE_FILES+=" atomic/AtomicDomain.o" -GAPS_SOURCE_FILES+=" atomic/ConcurrentAtomicDomain.o" -GAPS_SOURCE_FILES+=" atomic/ProposalQueue.o" GAPS_SOURCE_FILES+=" data_structures/HashSets.o" GAPS_SOURCE_FILES+=" data_structures/HybridMatrix.o" GAPS_SOURCE_FILES+=" data_structures/HybridVector.o" @@ -105,14 +132,23 @@ GAPS_SOURCE_FILES+=" math/VectorMath.o" # add c++ tests to source list if test "x$cpp_tests_disable" != "xyes" ; then echo "Enabling C++ Unit Tests" - GAPS_CPP_FLAGS+=" -DGAPS_CPP_UNIT_TESTS " - GAPS_SOURCE_FILES+=" cpp_tests/testVector.o" + GAPS_CPP_FLAGS+=" -DGAPS_VCPP_UNIT_TESTS " + GAPS_SOURCE_FILES+=" cpp_tests/testAtomicDomain.o" + GAPS_SOURCE_FILES+=" cpp_tests/testDenseGibbsSampler.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseGibbsSampler.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSamplerHighLevel.o" + GAPS_SOURCE_FILES+=" cpp_tests/testFileParsers.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHashSets.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHybridMatrix.o" + GAPS_SOURCE_FILES+=" cpp_tests/testHybridVector.o" + GAPS_SOURCE_FILES+=" cpp_tests/testMathHelpers.o" + GAPS_SOURCE_FILES+=" cpp_tests/testMatrix.o" + GAPS_SOURCE_FILES+=" cpp_tests/testRandom.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseMatrix.o" GAPS_SOURCE_FILES+=" cpp_tests/testSparseVector.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHashSets.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHybridMatrix.o" - GAPS_SOURCE_FILES+=" cpp_tests/testHybridVector.o" - GAPS_SOURCE_FILES+=" cpp_tests/testSparseIterator.o" -# GAPS_SOURCE_FILES+=" cpp_tests/testConcurrentAtomicDomain.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSparseIterator.o" + GAPS_SOURCE_FILES+=" cpp_tests/testSerialization.o" + GAPS_SOURCE_FILES+=" cpp_tests/testVector.o" fi # commented files in the if above are to be reviwed @@ -123,6 +159,12 @@ AC_SUBST(GAPS_CXX_FLAGS) AC_SUBST(GAPS_LIBS) AC_SUBST(GAPS_SOURCE_FILES) +echo "Writing src/Makevars" +echo Echoing GAPS_CPP_FLAGS: +echo $GAPS_CPP_FLAGS +echo Echoing GAPS_CXX_FLAGS: +echo $GAPS_CXX_FLAGS + # create makefile, output configure file AC_CONFIG_FILES([src/Makevars]) AC_OUTPUT diff --git a/data/GIST.RData b/data/GIST.RData index 90781cc4..97a11ec5 100755 Binary files a/data/GIST.RData and b/data/GIST.RData differ diff --git a/data/modsimdata.rda b/data/modsimdata.rda index 045ff01e..f7dc7ede 100644 Binary files a/data/modsimdata.rda and b/data/modsimdata.rda differ diff --git a/data/modsimresult.rda b/data/modsimresult.rda index 7f553b63..eb81d6e6 100644 Binary files a/data/modsimresult.rda and b/data/modsimresult.rda differ diff --git a/dev-notes/132-LLM-assisted-solved-issues.md b/dev-notes/132-LLM-assisted-solved-issues.md new file mode 100644 index 00000000..af005eea --- /dev/null +++ b/dev-notes/132-LLM-assisted-solved-issues.md @@ -0,0 +1,687 @@ +# LLM-Assisted Solved Issues — Branch `132-uncertainty-improvements` + +Branch tracks GitHub issue #132 ("Uncertainty improvements"). +The issues below were diagnosed and solved with LLM assistance. +Issue numbering continues from the manually-fixed set (`132-manually-fixed-issues.md`, +issues 1–8); cross-references such as "issue 9" refer to that shared numbering. + +--- + +## 9. SIMD NaN in `alphaParameters` on Mac Intel (SSE4/AVX) + +Full root-cause analysis is in `dev-notes/simd-issue.md`. + +The bug manifested only on Mac Intel, where Apple Clang enables SSE4.1 (or AVX) +by default. On this platform the SIMD loop in `alphaParameters()` increments +its index by 4 (or 8) per step, so the last iteration reads past the last real +data element into the alignment-padding positions that `Vector` allocates but +does not initialise to anything meaningful. + +The uncertainty matrix `mSMatrix` is computed by `gaps::pmax(mDMatrix, factor, +mLambda)`, which before the fix filled only the real elements. The padding +positions were left at zero — the default from the `Vector` constructor. When +the SIMD loop read one of those padding positions, both the numerator (`pMat`) +and the denominator (`pS * pS`) were zero, giving `0/0 = NaN`. That NaN +propagated through the horizontal sum of the SIMD register and poisoned the +returned `AlphaParameters`. A poisoned `AlphaParameters` causes `gibbsMass()` +to return an empty result, so every `sampleBirth()` call was silently rejected. +Consequently PSampler never accumulated atoms: `MyMatrix` stayed all-zero, the +AP product stayed all-zero, and chi-square never decreased no matter how many +iterations were run. + +The fix adds `padSIMD(float val)` to `Vector` and `Matrix`, which fills only the +padding positions (beyond `mSize`) with the given value. Both `gaps::pmax` +overloads call `padSIMD(min_threshold)` before returning. Padding lanes now +hold `mLambda > 0`, so the ratio there is `0 / mLambda^2 = 0` — no NaN, no +contribution to the accumulator. + +Fix commit: `bfd5e09d` + +--- + +## 10. `SIMD_PAD` insufficient on ARM — compiler auto-vectorization heap corruption + +### Affected platform + +Apple Silicon (M1/M2/M3/M4). Does **not** affect Intel Mac or Linux. + +### Root cause + +On ARM there is no SSE/AVX, so `SIMD_INC = 1` and the `SIMD_PAD` macro +allocated only one extra element per `Vector`: + +```c +SIMD_PAD(n) = 1 + 1*(n/1) = n + 1 +``` + +The hand-written SIMD loops in `alphaParameters()` and `updateAPMatrix()` were +safe with `SIMD_INC = 1` — they iterate one float at a time, exactly `n` times. +However, when the package was built with optimisation (`-O2` / `-O3`, the R +default in `install_local`), Clang's auto-vectoriser replaced those scalar loops +with 4-wide ARM NEON instructions. The auto-vectorised code processes floats in +batches of 4, so the last batch for a vector of 25 elements starts at index 24 +and reads (or writes) indices 24, 25, 26, 27 — but only 26 elements were +allocated. Indices 26 and 27 are past the end of the `std::vector` storage. + +The writes in `updateAPMatrix` (`pAP.store(ap + i)`) were particularly +destructive: they silently overwrote adjacent heap memory, which could corrupt +the data of a neighbouring `Vector` — for example a column of `mSMatrix`. When +`chiSq()` subsequently checked `GAPS_ASSERT(mSMatrix(i,j) > 0.f)`, the +corrupted value failed the assert and the sampler terminated with "CoGAPS +terminated". + +The bug was invisible with `devtools::load_all()` because that build retains +the existing `src/Makevars` which had `-g -O0` (debug flags from a previous +configure run), suppressing all auto-vectorisation. `devtools::install_local()` +runs `configure` from scratch, produces a release `Makevars` without `-O0`, and +the bug manifested immediately. + +### Relationship to the SSE4/AVX fix (issue 9) + +Issue 9 addressed padding values (zeros → `mLambda`). This issue addresses +padding size: even with correct values in padding, there was simply not enough +padding for the auto-vectoriser to stay within bounds. Both fixes are required +for correctness across all platforms. + +### Fix + +Changed `SIMD_PAD` in `Vector.cpp` to always use an effective width of 8, +regardless of `SIMD_INC`: + +```c +#define SIMD_PAD_INC 8 +#define SIMD_PAD(x) (SIMD_PAD_INC + SIMD_PAD_INC * ((x) / SIMD_PAD_INC)) +``` + +`SIMD_PAD(25)` now allocates 32 elements on every platform, leaving 7 padding +positions (indices 25–31). The auto-vectoriser on ARM reads/writes at most up +to index 27 — safely within bounds. The `padSIMD(mLambda)` calls already +present in `gaps::pmax` fill all padding positions with a positive value, so +no NaN can arise there either. + +Fix commit: `60f94063` + +--- + +## 11. `AtomicDomain::move()` leaves a stale map iterator in the atom + +### Problem + +`AtomicDomain::move()` replaced the map entry for an atom (erase old key, insert new key) but never updated `atom->mIterator` to point to the new map node: + +```cpp +std::pair newpair(newPos, atom->iterator()->second); +mAtomMap.erase(atom->pos()); // atom->mIterator is now dangling +atom->updatePos(newPos); +mAtomMap.insert(newpair); // atom->mIterator NOT updated — still dangling +``` + +After `move()`, `atom->mIterator` pointed to a deleted `std::map` node. If the +same atom was subsequently passed to `erase()`, the line + +```cpp +mAtomMap.erase(atom->iterator()); // UB: dangling iterator +``` + +caused undefined behaviour. In practice this corrupted the red-black tree, +which meant the next `insert()` placed the atom at a wrong position in the +ordering. Corrupted neighbour links eventually caused + +``` +GAPS_ASSERT(a <= b) // lbound+1 > rbound-1 +``` + +to fire inside `uniform64()` in `SingleThreadedGibbsSampler::move()`, terminating +with "CoGAPS terminated". + +The bug triggered reliably with GIST data (1363 × 9, 9 patterns) but not with +the smaller random-matrix test, because only the GIST run produced a `move()` +followed immediately by an `erase()` on the same atom within the first few +thousand iterations. + +### Fix + +Capture the iterator returned by `mAtomMap.insert()` and store it in the atom: + +```cpp +size_t storageIdx = atom->iterator()->second; +mAtomMap.erase(atom->pos()); +atom->updatePos(newPos); +atom->setIterator( + mAtomMap.insert(std::pair(newPos, storageIdx)).first); +``` + +`AtomicDomain` is a friend of `Atom`, so it may call the private `setIterator()`. + +A regression test `[atomicdomain][movethenerase]` was added to +`src/cpp_tests/testAtomicDomain.cpp`: it inserts four atoms, moves one, erases +the same atom, and verifies the map contains exactly the expected three keys. + +### Changed files + +| File | Change | +|---|---| +| `src/atomic/AtomicDomain.cpp` | `move()` calls `atom->setIterator()` after insert | +| `src/cpp_tests/testAtomicDomain.cpp` | regression test `[atomicdomain][movethenerase]` | + +--- + +## 12. `gaps::min/max(SparseVector)` segfault on an empty (all-zero) sparse vector + +### Problem + +`gaps::min(const SparseVector&)` and `gaps::max(const SparseVector&)` read the +first element through the iterator **before** checking `atEnd()`: + +```cpp +float gaps::max(const SparseVector &v) +{ + SparseIterator<1> it(v); + float mx = get<1>(it); // reads mData[0] with no bounds check + while (!it.atEnd()) { ... } + return mx; +} +``` + +For an **empty** sparse vector (one with no stored non-zero elements — i.e. an +all-zero row or column), the iterator is immediately `atEnd()` and `get<1>(it)` +dereferences `mData[0]` of an empty backing array → read at address `0x0` +(`SparseVector::getIthElement`), segfault. + +This is reachable in normal use, not just tests: `SparseNormalModel`'s constructor +runs `if (gaps::max(mDMatrix) > 50.f)` (`SparseNormalModel.h:86`) on **every** +`sparseOptimization=TRUE` run. Any all-zero gene (row) or sample (column) in the +data — common in sparse single-cell matrices — yields an empty `SparseVector` and +crashes construction before sampling starts. + +Related to issue 4 (`min`/`max` initialisation), whose fix did not cover the +empty-`SparseVector` case. + +### Diagnosis + +The crash surfaced only after `testSamplerHighLevel` was added to the build (it +constructs a `SparseNormalModel` on dummy data whose row 0 / column 0 are all +zero). Root-caused with a standalone AddressSanitizer harness compiled at `-O2`: +the backtrace pointed at `SparseVector::getIthElement` ← `gaps::max(SparseVector)` +← `SparseNormalModel` constructor. + +### Fix + +Guard the empty case — a sparse vector's absent elements are `0`, and CoGAPS data +is non-negative, so the min/max of an all-zero vector is `0.f`: + +```cpp +SparseIterator<1> it(v); +if (it.atEnd()) return 0.f; // empty (all-zero) sparse vector +float mx = get<1>(it); +``` + +Applied to both `gaps::min` and `gaps::max`. Non-empty behaviour is unchanged. + +A regression test `[sparsevector][emptyminmax]` was added to +`src/cpp_tests/testSparseVector.cpp`: it checks `min`/`max` return `0.f` on an +empty `SparseVector` (no crash) and are unaffected for a non-empty one. + +### Changed files + +| File | Change | +|---|---| +| `src/math/VectorMath.cpp` | `atEnd()` guard in `gaps::min`/`gaps::max` for `SparseVector` | +| `src/cpp_tests/testSparseVector.cpp` | regression test `[sparsevector][emptyminmax]` | + +--- + +## 13. `SingleThreadedGibbsSampler` deserialization used `<<` (write) instead of `>>` (read) + +### Problem + +The sampler's `operator>>` (`src/gibbs_sampler/SingleThreadedGibbsSampler.h`) read +the `DataModel` base with `>>` but then **chained `<<` (write)** for the sampler's +own members: + +```cpp +Archive& operator>>(Archive &ar, SingleThreadedGibbsSampler &s) +{ + operator>>(ar, static_cast(s)) << s.mDomain << s.mNumBins + << s.mBinLength << s.mNumPatterns << s.mdDomainLength << s.mAlpha; // BUG: << + return ar; +} +``` + +So restoring a sampler from a checkpoint did **not** reload the atomic domain or the +bin geometry — those fields were written back into the read archive instead. A run +resumed from a checkpoint started with an empty atomic domain and diverged from an +uninterrupted run. + +Found while auditing the code after the async removal. It is a latent bug in the +shipped package because checkpoints are disabled by default +(`-DGAPS_DISABLE_CHECKPOINTS`, `checkpointsEnabled() == FALSE`), but it is a genuine +correctness bug and would bite anyone building with checkpoints enabled. + +### Fix + +Use `>>` for the whole chain, matching the write path: + +```cpp +operator>>(ar, static_cast(s)) >> s.mDomain >> s.mNumBins + >> s.mBinLength >> s.mNumPatterns >> s.mdDomainLength >> s.mAlpha; +``` + +Regression test `[serialization][gibbssampler-roundtrip]` in +`testSerialization.cpp`: build a dense sampler, `sync` + `extraInitialization` + +`update(500)` to populate the domain, serialize, deserialize into a fresh sampler, +and require the restored `nAtoms()` (and `MyMatrix` sum) to match the original. +Before the fix the restored domain stayed empty (`nAtoms() == 0`). + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/SingleThreadedGibbsSampler.h` | `operator>>` chain `<<` → `>>` | +| `src/cpp_tests/testSerialization.cpp` | regression test `[serialization][gibbssampler-roundtrip]` | +--- + +## 14. `gaps::nonZeroMean` divided by zero (NaN) on an all-zero matrix + +### Problem + +`gaps::nonZeroMean(const Matrix&)` and `(const SparseMatrix&)` +(`src/math/MatrixMath.cpp`) returned `sum / nNonZeroes` with no guard for +`nNonZeroes == 0`. For an all-zero data matrix (or a subset that contains no +positive entries) this is `0 / 0 = NaN`. + +It feeds model initialization directly: `DenseNormalModel.h` / `SparseNormalModel.h` +compute `meanD = nonZeroMean(mDMatrix)` then +`mLambda = alpha * sqrt(nPatterns() / meanD)` and `mMaxGibbsMass /= mLambda`. A NaN +`meanD` poisons `mLambda`, `mMaxGibbsMass`, and every subsequent sample. + +### Fix + +Guard the empty case (a matrix with no positive entries has non-zero mean `0`): + +```cpp +if (nNonZeroes == 0) return 0.f; // all-zero matrix: avoid 0/0 = NaN +return sum / static_cast(nNonZeroes); +``` + +Applied to both the dense and sparse overloads. Regression test +`[matrix][nonzeromean-empty]` in `testMatrix.cpp` checks `nonZeroMean` of an all-zero +`Matrix` returns `0.f` (a NaN would fail the equality). + +### Changed files + +| File | Change | +|---|---| +| `src/math/MatrixMath.cpp` | `nNonZeroes == 0` guard in both `nonZeroMean` overloads | +| `src/cpp_tests/testMatrix.cpp` | regression test `[matrix][nonzeromean-empty]` | + +--- + +## 15. Dense/hybrid `gaps::min`/`max`/`whichMax` read element 0 on an empty container + +### Problem + +The dense and hybrid reductions `gaps::min`/`max(Vector)`, +`gaps::min`/`max(HybridVector)`, `gaps::whichMax(Vector)` +(`src/math/VectorMath.cpp`) initialised the accumulator with `v[0]` with no size +check, and the templated `gaps::min`/`max(MatrixType)` (`src/math/MatrixMath.h`) +called `getCol(0)` with no `nCol() > 0` check. On an empty vector / zero-column +matrix this is an out-of-bounds read. + +This is the same empty-container class fixed for the `SparseVector` overloads in +issue 12; the dense/hybrid/matrix siblings were left unguarded (and issue 4, which +changed the accumulator init from `0` to `v[0]`, is what introduced the +empty-vector deref). `v[0]` currently reads SIMD padding rather than crashing on +this platform, but it is a genuine OOB on a vector with no padding. + +### Fix + +Return `0.f` (index `0` for `whichMax`) for empty input, before any indexing: + +```cpp +if (v.size() == 0) return 0.f; // Vector / HybridVector min/max +if (v.size() == 0) return 0; // whichMax +if (mat.nCol() == 0) return 0.f; // min/max(MatrixType) +``` + +Regression tests: `[vector][emptyminmax]` in `testVector.cpp` (empty `Vector` +min/max/whichMax → 0) and the "zero-column matrix" section of `[matrix][minmax-empty]` +in `testMatrix.cpp`. + +### Changed files + +| File | Change | +|---|---| +| `src/math/VectorMath.cpp` | empty-size guard in dense/hybrid `min`/`max`/`whichMax` | +| `src/math/MatrixMath.h` | `nCol() == 0` guard in `min`/`max(MatrixType)` | +| `src/cpp_tests/testVector.cpp` | regression test `[vector][emptyminmax]` | +| `src/cpp_tests/testMatrix.cpp` | regression test `[matrix][minmax-empty]` (zero-column section) | + +--- + +## 16. `SparseVector` deserialization dropped all stored values + +### Problem + +`operator>>(Archive&, SparseVector&)` read the stored (non-zero) values with a loop +bounded by the **destination's** current size: + +```cpp +for (unsigned i = 0; i < vec.mData.size(); ++i) // BUG: destination's count + ar >> vec.mData[i]; +``` + +`SparseVector` stores only its non-zero entries in `mData`; the count of non-zeros +is not written to the archive. When deserializing into a freshly-constructed +`SparseVector(size)` (empty `mData`), the loop ran zero times, so **none** of the +serialized values were read back — the restored vector was all-zero (and the +archive read pointer was left misaligned for whatever followed). + +`SparseMatrix` serialization delegates to this operator per column, so it inherited +the same defect. Both are unused in the current checkpoint path (`SparseNormalModel` +serializes its `HybridMatrix` `mMatrix`, not the `SparseMatrix` `mDMatrix`), which +is why it was never noticed — the same "untested serialization operator" class as +issue 13. + +Found while filling the empty `SparseVector`/`SparseMatrix` serialization test +stubs: the round-trip into an empty destination failed. (The pre-existing +`SparseMatrix` stub would have passed even unfixed, because a test that rebuilds the +read target from the same data gives it the right `mData` sizes by accident.) + +### Fix + +The number of stored values equals the popcount of the bit flags, which are read +first. Derive it and `resize` `mData` before reading (no archive-format change): + +```cpp +unsigned nNonZeroes = 0; +for (unsigned i = 0; i < vec.mIndexBitFlags.size(); ++i) + nNonZeroes += __builtin_popcountll(vec.mIndexBitFlags[i]); +vec.mData.resize(nNonZeroes); +for (unsigned i = 0; i < nNonZeroes; ++i) + ar >> vec.mData[i]; +``` + +Regression tests `[serialization][sparsevector]` and `[serialization][sparsematrix]` +in `testSerialization.cpp` round-trip into an empty / all-zero destination and +compare the dense reconstruction element-wise. + +### Changed files + +| File | Change | +|---|---| +| `src/data_structures/SparseVector.cpp` | derive non-zero count from bit flags, `resize` + read `mData` | +| `src/cpp_tests/testSerialization.cpp` | filled `[serialization][sparsevector]` and `[serialization][sparsematrix]` (read into empty target) | + +--- + +## 17. `sparseOptimization` inconsistent with the dense sampler (uncertainty model) + +### Problem + +`sparseOptimization=TRUE` produced results inconsistent with the dense sampler: +the two paths implement the **same** statistical model, so the sparse version is +meant to be an optimization giving the same answer — but they had diverged. + +Both compute the same alpha statistics for the Gibbs update, +`s = Σⱼ P²/S²`, `s_mu = Σⱼ P(D−AP)/S²`, differing only in the uncertainty `S`: + +- **Dense** stores `mSMatrix` explicitly and divides by `S²`. +- **Sparse** never stores `S`; it bakes the assumption into the algebra + (`mZ1` base = the `S=1` sum, per-non-zero corrections replace `S=1` with `S=d`), + scaled by `mBeta = 100`. Since `1/factor² = 1/0.1² = 100 = mBeta`, the sparse + model's *effective* uncertainty is `S = factor·D` for observed data and + `factor` for zeros. + +Two independent defects made the effective `S` differ: + +1. **Dense zero-floor regression (this branch, from issue 2).** Issue 2 changed + the dense uncertainty from `pmax(mDMatrix, 0.1f)` (i.e. `max(0.1·D, 0.1)`, floor + `0.1`) to `pmax(mDMatrix, factor, mLambda)` (floor `mLambda ≈ 0.006`), intending + `mLambda` as an "atom-size" floor. But `mLambda` is the atom-mass scale + (used for `mMaxGibbsMass`), not an uncertainty. For zero data this made dense + `S = mLambda ≈ 0.006` (weight `1/S² ≈ 27000`) versus the sparse `S = 0.1` + (weight `100`) — a ~230× divergence on every zero entry. + (The issue 2 note also misdiagnosed master as "S = constant 1 via `pad(1.f)`"; + in fact master's `Vector::pad` loops from `mSize`, so it only touches padding — + master's real `S` was already `max(0.1·D, 0.1)`.) + +2. **Missing floor in sparse (pre-existing, from master).** Sparse used `S = d` + for *every* non-zero entry with no floor, so `S = factor·d`. For fractional + data `0 < d < 1` this gives `S < factor`, i.e. a tiny measurement is treated as + more precise than a zero (weight `1/(factor·d)²` blows up). Master's dense + floored at `0.1` but master's sparse did not, so dense and sparse already + mismatched on fractional data even on master. + +Symptom on GIST (continuous data): dense-vs-sparse `A·P` reconstruction was +uncorrelated; the alpha statistics diverged by ~230× (caught by the revived +`[sparsegibbs]` consistency test, issue-independent). + +### Fix + +Use one uncertainty model in both — relative error floored at `factor`: +``` +S[i,j] = max(factor · D[i,j], factor) factor = 0.1 +``` +so zeros and any `D < 1` get `S = 0.1`, and `D ≥ 1` get `S = 0.1·D`. + +- **Dense** (`DenseNormalModel.h`): revert to `gaps::pmax(mDMatrix, factor, factor)` + (floor = `factor`, not `mLambda`). +- **Sparse** (`SparseNormalModel.cpp`): floor the raw uncertainty at 1 + (`S = max(d, 1)`, effective `factor·max(d,1)`) in all three `alphaParameters` + functions. `d` plays a dual role (it is both the data value `D` and, previously, + the uncertainty `S`); the floor lowers only the uncertainty while the data `d` + is kept in the residual term (`s_mu += v·d·invS2 + v(1−invS2)·AP`, + `invS2 = 1/max(d,1)²`). + +### Verification + +- `[sparsegibbs]` was revived (ported from the removed `GibbsSampler` API + to `SingleThreadedGibbsSampler<...NormalModel>` via a test-only `ExposedSampler` + subclass that surfaces the protected `alphaParameters`) and its data extended to + **continuous values including `0 < D < 1`**. It asserts sparse and dense + `alphaParameters` (1D, 2D, symmetry, with-change) match to `TEST_APPROX`; now + passes (8404 assertions). +- End-to-end on GIST, permutation-invariant metrics confirm equivalence: `A·P` + reconstruction correlation dense-vs-sparse = `0.999`, equal to the + dense-vs-dense (different-seed) control; `meanChiSq` for sparse lands within the + dense-run range. (Raw `featureLoadings` correlation is meaningless here — NMF is + only identifiable up to pattern permutation, so even two dense runs correlate at + ≈ `−0.1`.) + +Note: this deliberately changes dense/sparse numerics (the pre-fix behaviour was +wrong), so the async-removal parity baseline no longer applies to these paths. + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/DenseNormalModel.h` | uncertainty floor `mLambda` → `factor` (`pmax(mDMatrix, factor, factor)`) | +| `src/gibbs_sampler/SparseNormalModel.cpp` | floor `S = max(d, 1)` in the three `alphaParameters` functions | +| `src/cpp_tests/testSparseGibbsSampler.cpp` | revived `[sparsegibbs]` dense-vs-sparse consistency test; continuous (fractional) data | + +## 18. `SparseNormalModel::chiSq()` segfaults when called before `sync()` + +### Problem + +`chiSq()` computes the fit `Σ (D − A·P)² / S²`, so it needs the *other* factor +matrix (`A` needs `P`, and vice versa). Each sampler receives that pointer, +`mOtherMatrix`, only when `sync()` is called; the constructor leaves it `NULL`. + +`SparseNormalModel::chiSq()` dereferences `mOtherMatrix` directly +(`mOtherMatrix->getRow(i)` inside the dot products), so calling `chiSq()` on a +freshly-constructed, not-yet-`sync()`ed sampler is a NULL dereference — an +immediate segfault. + +`DenseNormalModel::chiSq()` is **not** affected: it reads its cached, zero- +initialised `mAPMatrix` instead of `mOtherMatrix`, so before `sync()` it simply +returns the "no fit" value (`A·P = 0`). The two models therefore disagreed on +whether `chiSq()`-before-`sync()` is legal — dense tolerated it, sparse crashed. + +Not reachable from the production run loop (which always `sync()`s during +initialization), but a real robustness defect for anyone using these classes as a +library, and it surfaced while writing the sampler unit tests. + +### Diagnosis + +AddressSanitizer backtrace: crash in `HybridMatrix::getRow` ← +`SparseNormalModel::chiSq()`, on the first `chiSq()` call, before any `sync()`. +The cause is the un-set `mOtherMatrix` (NULL) being dereferenced. + +### Fix + +Guard `SparseNormalModel::chiSq()` against `mOtherMatrix == NULL` and return the +same "no fit" value dense returns. With `A·P = 0`, every dot product is zero: the +`Σ (A·P)²` loop contributes nothing and each stored data value contributes its +`(D − 0)² / D² = 1` term, so the result is `(#non-zero entries) · mBeta`. + +For data with all `D ≥ 1` this equals dense's `Σ D²/S² = Σ (D / 0.1D)² = 100` +per entry, i.e. the two models return the identical no-fit chiSq (verified in the +regression test on `D = i + j + 1`, giving `100 · nRow · nCol`). + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/SparseNormalModel.cpp` | `chiSq()` returns the no-fit value when `mOtherMatrix == NULL` instead of dereferencing it | +| `src/cpp_tests/testSparseGibbsSampler.cpp` | regression: `chiSq()` before `sync()` does not crash and equals dense's no-fit chiSq | + +## 19. `SparseNormalModel::chiSq()` did not floor the uncertainty (inconsistent with dense and with issue #17) + +### Problem + +Issue #17 unified the uncertainty model on `S = max(factor·D, factor)`, +`factor = 0.1`, and floored it in all three `alphaParameters` functions — but +**not in `chiSq()`**, which was left computing the residual with the unfloored +`S = D` (`dsq = get<1>(it)²`). So for continuous data with `0 < D < 1` the sparse +`chiSq` used `S = D < 0.1` where the dense `chiSq` (and the sparse +`alphaParameters`) used the floor `S = 0.1`. Result: `meanChiSq` reported for a +`sparseOptimization=TRUE` run diverged from the equivalent dense run on fractional +data. Diagnostic only (goodness-of-fit / annealing readout — the *sampling* +decisions use the already-floored `alphaParameters`), but a real correctness gap +and a landmine if `chiSq` were ever used more widely. + +### Root cause / design note + +The sparse model deliberately does **not** materialise an `S` matrix (that would +defeat its whole purpose — avoiding dense `nRow×nCol` storage for single-cell +data — and would break the `mZ1`/`mZ2` lookup-table algebra, which depends on the +zero-entry uncertainty being a single constant that factors out into +`mBeta = 1/factor² = 100`). Instead every calculation computes `1/S²` on the fly +from the data value. The floor therefore has to be applied identically at each +call site; issue #17 did three of the four, and `chiSq()` was the straggler. + +### Fix + +Extract the one uncertainty model into a single file-local helper and call it +from all four sites: +```cpp +static inline float invSSq(float d) // = 1/max(D,1)^2 (the factor is in mBeta) +{ + float sraw = gaps::max(d, 1.f); + return 1.f / (sraw * sraw); +} +``` +- The three `alphaParameters` functions now call `invSSq(d_val)` instead of the + inlined `1/max(d,1)²` (no numeric change — just de-duplication). +- `chiSq()`'s non-zero correction becomes the floored residual: the first loop + adds `A·P²` at the zero weight, and each stored entry corrects it to + `(D − A·P)² · invSSq(D)` via `D²·invS2 − 2·D·A·P·invS2 + A·P²·(invS2 − 1)` + (algebraically identical to the old `1 + dot(dot − 2d − dsq·dot)/dsq` when + `D ≥ 1`, so integer/count data is unchanged). +- The `mOtherMatrix == NULL` no-fit branch (issue #18) likewise became + `D² · invSSq(D)` so it stays consistent with the floored main path for `D < 1`. + +`mZ1`/`mZ2` need no change: they encode the all-entries-are-zero baseline at the +constant `S = factor`, which the per-non-zero corrections adjust — already +consistent with the floor. + +### Verification + +Extended the `[sparsegibbs]` consistency test (continuous data with `0 < D < 1`, +identical `A`/`P` on a sparse and a dense sampler) to also assert +`sparse.chiSq() == Approx(dense.chiSq())` for both the A- and P-samplers. Fails +before the fix (sparse used `S = D` on the sub-1 entries), passes after. Full cpp +suite green. + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/SparseNormalModel.cpp` | new `invSSq()` helper; `chiSq()` (both branches) and all three `alphaParameters` now floor via it | +| `src/cpp_tests/testSparseGibbsSampler.cpp` | consistency test also compares sparse vs dense `chiSq()` on fractional data | + +--- + +## 20. User-supplied `uncertainty=` was silently discarded (`pad()` overwrote the whole matrix) + +### Problem + +The `uncertainty=` argument of `CoGAPS()` had no effect whatsoever on a dense run. +Two runs differing only in the uncertainty matrix returned bit-identical results: + +```r +r1 <- CoGAPS(D, nPatterns=3, nIterations=200, uncertainty=0.1*as.matrix(D), seed=1) +r2 <- CoGAPS(D, nPatterns=3, nIterations=200, uncertainty=10 *as.matrix(D), seed=1) +getMeanChiSq(r1) == getMeanChiSq(r2) # TRUE -- 704.3173 in both cases +``` + +A hundredfold change in the uncertainty left the reported chi-square untouched, +i.e. the sampler was fitting with `S = 1` everywhere regardless of what the caller +passed. + +### Root cause + +`DenseNormalModel::setUncertainty()` loaded the caller's matrix and then called +`pad()`: + +```cpp +mSMatrix = Matrix(unc, transpose, subsetRows, params.dataIndicesSubset); +mSMatrix.pad(1.f); // so that SIMD operations don't divide by zero +``` + +`Vector::pad(val)` fills **every** allocated element, not just the SIMD padding +lanes (`for (i = 0; i < mData.size(); ++i)`), so the line above replaced the whole +uncertainty matrix with 1.0f. The comment shows the intent was the padding lanes +only; the correct method is `padSIMD()`, which fills `[mSize, mData.size())` and +was added in this branch for the SIMD-NaN fix (issue #9). + +The bug predates the branch — it is present in `master` too, on both the default +and the user-supplied path. Issue #17 removed the `pad(1.f)` from the *default* +path (replacing it with `gaps::pmax(mDMatrix, factor, factor)`) and so fixed that +half without the user-supplied half being noticed. + +### Fix + +```cpp +mSMatrix = Matrix(unc, transpose, subsetRows, params.dataIndicesSubset); +// Only the SIMD padding may be overwritten -- pad() would set *every* element +// to 1.f and so discard the uncertainty the caller passed in. Padding lanes +// get 1.f so that the SIMD loops divide by 1, not by 0. +mSMatrix.padSIMD(1.f); +``` + +`padSIMD(1.f)` keeps the SIMD-safety property (padding lanes divide by 1, never by +0 -- see issue #9) while leaving the caller's values intact. + +### Verification + +The same two runs now differ as they must (13338.07 vs 109.872). The chi-square +consistency test gained a case that recomputes the reported `getMeanChiSq()` +against an explicitly passed uncertainty matrix; it fails before the fix +(442 vs 103450) and passes after. That test came from `master` +(`test_chisq.R`), where it was written against the pre-#17 code, and is one of the +things the master merge brought in. + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/DenseNormalModel.h` | `setUncertainty()` uses `padSIMD(1.f)` instead of `pad(1.f)` | +| `tests/testthat/test_chisq.R` | added the explicit-uncertainty round-trip case | + +### Note + +`SparseNormalModel` is unaffected: `sparseOptimization=TRUE` rejects a +user-supplied uncertainty in `checkInputs()`, so it only ever uses the built-in +model. diff --git a/dev-notes/132-manually-fixed-issues.md b/dev-notes/132-manually-fixed-issues.md new file mode 100644 index 00000000..b7bbd68b --- /dev/null +++ b/dev-notes/132-manually-fixed-issues.md @@ -0,0 +1,247 @@ +# Manually Fixed Issues — Branch `132-uncertainty-improvements` + +Branch tracks GitHub issue #132 ("Uncertainty improvements"). +All fixes below were made manually by @favorov. + +--- + +## 1. `pmax` signature: separate `factor` and `min_threshold` + +### Problem + +The original `gaps::pmax(v, p)` prototype took a single float used simultaneously as +a multiplicative scaling factor **and** as the minimum-value threshold. The two roles +are conceptually independent and the caller had no way to set them separately. + +### Fix + +Added a three-argument overload: + +```cpp +Vector pmax(const Vector &v, float factor, float min_threshold); +Matrix pmax(const Matrix &m, float factor, float min_threshold); +``` + +The two-argument versions are kept for back-compatibility (they default +`min_threshold = factor`). Parameters changed from pass-by-value to +`const &` and the result is now returned as a newly created object +(see §2 below). + +### Changed files + +| File | Change | +|---|---| +| `src/math/VectorMath.h` | new three-arg overload; `const Vector &` params | +| `src/math/VectorMath.cpp` | implementation of new overload | +| `src/math/MatrixMath.h` | same for Matrix | +| `src/math/MatrixMath.cpp` | same for Matrix | + +Commits: `daf2cd94`, `eb767c3b`, `d8590ef`, `bd11860` + +--- + +## 2. Data-driven uncertainty in `DenseNormalModel` + +### Problem + +The uncertainty matrix `mSMatrix` was initialised with `mSMatrix.pad(1.f)` — a +single constant for every element regardless of the data. This makes the +chi-square denominator unresponsive to actual data magnitude. + +### Fix + +Replaced the constant pad with a data-driven computation: + +```cpp +float factor = 0.1f; +mSMatrix = gaps::pmax(mDMatrix, factor, mLambda); +// mSMatrix[i,j] = max(mDMatrix[i,j] * factor, mLambda) +``` + +`mLambda` is set before `mSMatrix` is computed and represents the expected atom +size, so the minimum uncertainty is now data-scale-aware. + +### Changed files + +| File | Change | +|---|---| +| `src/gibbs_sampler/DenseNormalModel.h` | replace `pad(1.f)` with `gaps::pmax(mDMatrix, factor, mLambda)` | + +Commit: `3efffbc1` + +--- + +## 3. `elementSq` / `pmax` mutation bug + +### Problem + +`Vector elementSq(Vector v)` and `Vector pmax(Vector v, float p)` accepted vectors +by value, mutated the local copy, and returned it. The intent was to return a new +object, but the pass-by-value idiom is fragile and was already causing confusion +with callers that expected the original to be unchanged. + +### Fix + +Changed signatures to `const Vector &` and constructed the result explicitly +before returning. + +Commits: `d8590ef`, `bd11860` + +--- + +## 4. `gaps::min` / `gaps::max` initialised with `0` instead of first element + +### Problem + +```cpp +float mn = 0.f; // wrong: should be v[0] +for (...) mn = (v[i] < mn) ? v[i] : mn; +``` + +For any vector whose every element is positive, `gaps::min()` returned `0` instead +of the true minimum. Symmetrically, `gaps::max()` returned `0` for all-negative +vectors. This corrupted any downstream computation that depends on the true +data range. + +Affected overloads: `min(Vector)`, `min(HybridVector)`, `min(SparseVector)`, +`max(Vector)`, `max(HybridVector)`. + +### Fix + +Initialise accumulators with the first element (`v[0]` / `get<1>(it)`) and start +the loop at index 1. + +### Changed files + +| File | Change | +|---|---| +| `src/math/VectorMath.cpp` | all five overloads corrected | +| `src/math/MatrixMath.h` | `min`/`max` for Matrix corrected | + +Commits: `db47de2f`, `5741cfc8` + +--- + +## 5. `Vector::pad` started at `mSize` instead of `0` + +### Problem + +```cpp +void Vector::pad(float val) +{ + for (unsigned i = mSize; i < mData.size(); ++i) // wrong start + mData[i] = val; +} +``` + +`pad` was supposed to overwrite **all** allocated elements with `val`. Starting at +`mSize` left the real elements (`[0, mSize)`) unchanged, so calls like +`mSMatrix.pad(1.f)` silently left the matrix full of zeros. + +### Fix + +Changed loop start to `0`. + +### Changed files + +| File | Change | +|---|---| +| `src/data_structures/Vector.cpp` | loop start `mSize → 0` | + +Commit: `88fc63e9` + +--- + +## 6. `AtomicDomain`: removed `MutableMap`; fixed `erase` and `move` + +### Problem + +`MutableMap` was used to update the position key of an atom in-place via +`updateKey()`. This operation is inherently unsafe (it bypasses std::map's +ordering invariant). Additionally, `erase()` had two bugs: + +1. When the last atom in `mAtoms` was swapped into the erased slot, the neighbour + atoms' stored indices were not updated, leaving dangling left/right index references. +2. `dst.mIndex = index` assignment was redundant (the assert already verified + equality); however the map entry was not being updated to reflect the + post-swap storage index. + +### Fix + +- Replaced `mAtomMap.updateKey(...)` in `move()` with an explicit erase + insert: + ```cpp + std::pair newpair(newPos, atom->iterator()->second); + mAtomMap.erase(atom->pos()); + atom->updatePos(newPos); + mAtomMap.insert(newpair); + ``` +- After copying the last atom into the erased slot, updated the left/right + neighbour references: + ```cpp + if (dst.hasLeft()) mAtoms[dst.leftIndex()].setRightIndex(index); + if (dst.hasRight()) mAtoms[dst.rightIndex()].setLeftIndex(index); + ``` +- Changed `GAPS_ASSERT(size() > 0)` in `erase()` to + `GAPS_ASSERT_MSG(size() > 0, "empty AtomicDomain tries to erase an atom")`. +- Removed the now-unused `GAPS_ASSERT(size() > 0)` from `randomFreePosition()` + (called when domain may legitimately be empty). + +### Changed files + +| File | Change | +|---|---| +| `src/atomic/AtomicDomain.cpp` | `erase`, `move` rewritten; asserts updated | +| `src/atomic/AtomicDomain.h` | removed `MutableMap`; public accessor added | + +Commits: `9dadfc36`, `89100aaf`, `3ebef56c`, `e3f1b601`, `9dadfc36` + +--- + +## 7. All atom index types converted to `size_t` + +### Problem + +Atom indices were inconsistently typed: `int` in some places, `unsigned` in others, +and `size_t` implied by `std::vector` — making comparisons and casts unsafe and +triggering signed/unsigned warnings. + +### Fix + +Converted all atom storage indices to `size_t` throughout `Atom.h`, +`AtomicDomain.h`, `AtomicDomain.cpp`. `random32` → `random64` where an +unsigned 64-bit random position is needed. + +Commit: `fdfcd37e` + +--- + +## 8. `static_cast` of near-max `double` on old Mac Intel Clang + +### Problem + +`SingleThreadedGibbsSampler` stored domain length as `double mDomainLength` and +converted it back to `uint64_t` via `static_cast`. Old Apple Clang (x86-64) +converts `double` values very close to `UINT64_MAX` incorrectly — the cast +overflows to `0`. This silently corrupted the atomic domain length calculation. + +Documented with a standalone reproducer, kept (with an explanation) in +`dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp`. + +### Fix + +Added `uint64_t AtomicDomain::DomainLength() const` accessor that returns +`mDomainLength` directly as `uint64_t`. Callers now use `mDomain.DomainLength()` +instead of the double round-trip. The `double mdDomainLength` field in +`SingleThreadedGibbsSampler` was renamed (`mdDomainLength`) to make it distinct +from the safe integer accessor. + +### Changed files + +| File | Change | +|---|---| +| `src/atomic/AtomicDomain.h` | added `DomainLength()` inline accessor | +| `src/atomic/AtomicDomain.cpp` | minor cleanup | +| `src/gibbs_sampler/SingleThreadedGibbsSampler.h` | use `mDomain.DomainLength()` | +| `dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp` | standalone reproducer (own `main()`, never compiled by the build) | + +Commits: `7f469a30`, `ac8a2e1a` diff --git a/dev-notes/README.md b/dev-notes/README.md new file mode 100644 index 00000000..7c3907c9 --- /dev/null +++ b/dev-notes/README.md @@ -0,0 +1,56 @@ +# CoGAPS developer notes — index + +Nothing in this directory is part of the package: `dev-notes/` is excluded by +`.Rbuildignore`, so it reaches neither the tarball nor `R CMD check` nor +Bioconductor. It is kept in the repository so that the reasoning behind the code +travels with it. + +Two files stand apart from the branch work and are deliberately in Russian, which +is why they live in `rus/` — everything an outside reader sees stays in English: +[`rus/agent-rules-rus.md`](rus/agent-rules-rus.md), the working agreements with +the maintainer, and [`rus/plan-rus.md`](rus/plan-rus.md), what is queued next. The assistant-facing +description of the project — how it is built, tested and laid out — is +`../CLAUDE.md`. Everything else below belongs to branch +`132-uncertainty-improvements`, which tracks GitHub issue #132 ("Uncertainty +improvements"). + +## Defect journals + +Fixed defects share one issue numbering across the first two files: 1–8 were fixed +manually, 9 onwards with LLM assistance. A cross-reference such as "issue 8" always +refers to that shared numbering. + +| File | What it is | +|---|---| +| [`132-manually-fixed-issues.md`](132-manually-fixed-issues.md) | Issues 1–8, fixed by hand by @favorov: the `pmax` signature split, data-driven uncertainty in `DenseNormalModel`, the `static_cast` overflow, and so on. | +| [`132-LLM-assisted-solved-issues.md`](132-LLM-assisted-solved-issues.md) | Issues 9–20, diagnosed and fixed with LLM assistance — SIMD padding NaN, `SIMD_PAD` on ARM, the stale iterator in `AtomicDomain::move()`, checkpoint deserialization, empty-container guards, the two models brought onto one uncertainty formula (17–19), and `Vector::pad()` silently discarding a user-supplied `uncertainty=` matrix (20). The longest file here and the best entry point for why the C++ looks the way it does. | +| [`master-issues.md`](master-issues.md) | Defects inherited from `master` and fixed here even though they have nothing to do with uncertainty — found while merging `master` in (2026-08) and while covering exported functions that no test had ever called. Records where they came from, so a reviewer does not have to wonder. | + +## Removal of the asynchronous sampler + +Read in this order; each file was written before the next step was taken. + +| File | What it is | +|---|---| +| [`remove-async-plan-eng.md`](remove-async-plan-eng.md) | The spec: why the OpenMP sampler breaks MCMC detailed balance, and the full list of edits its removal requires. | +| [`async-removal.md`](async-removal.md) | The retrospective report of executing that spec. | +| [`after_async-removed-plan.md`](after_async-removed-plan.md) | The audit that followed: does the C++ suite still compile, which test cases are empty stubs, and a ranked list of latent bugs in the surviving code. Issues 9 onwards grow out of this list. | + +## Reference write-ups + +| File | What it is | +|---|---| +| [`uncertainty-model-eng.md`](uncertainty-model-eng.md) | How measurement uncertainty `S` enters the sampler, and how `DenseNormalModel` and `SparseNormalModel` compute the same statistics from two very different representations. Written once issues 17–19 had unified them onto one formula. | +| [`simd-issue.md`](simd-issue.md) | The standalone root-cause analysis behind issue 9: SIMD padding zeros produce `NaN` in `alphaParameters()`, which silences every `sampleBirth()` call on Mac Intel. | + +## Open questions + +| File | What it is | +|---|---| +| [`annotation-weights-sampling-issue-eng.md`](annotation-weights-sampling-issue-eng.md) | Annotation-weighted subset sampling for distributed CoGAPS is incorrect. Deliberately **not** fixed on this branch; the report keeps the evidence and the options for whoever takes it on. | + +## Reproducer + +| Directory | What it is | +|---|---| +| [`static-cast-uint64-reproducer/`](static-cast-uint64-reproducer/) | A standalone program for issue 8: `static_cast` of a `double` near `UINT64_MAX` overflows to `0` on old Apple Clang. Has its own `main()`, is not listed in `configure.ac`, and is never compiled by the build — it lives here rather than in `src/cpp_tests/`, where it looked like a unit test. | diff --git a/dev-notes/after_async-removed-plan.md b/dev-notes/after_async-removed-plan.md new file mode 100644 index 00000000..46cf09a0 --- /dev/null +++ b/dev-notes/after_async-removed-plan.md @@ -0,0 +1,152 @@ +# Post-async-removal: test-coverage audit & suspicious-code plan + +**Branch:** `132-uncertainty-improvements` +**Date:** 2026-07-07 +**Context:** after the async sampler removal (`async-removal.md`), a review of the +C++ test suite and the surviving code for latent bugs and coverage gaps. + +--- + +## 1. Do all C++ tests compile? — Yes + +All 15 test files in the build list (`configure.ac` `GAPS_SOURCE_FILES`) compile +and pass: **46 Catch test cases**. The only file outside the build is +`dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp` — a standalone reproducer with its +own `main` (intentional, from the `static_cast` overflow investigation). + +**Caveat:** 6 registered TEST_CASEs are **empty stubs** (`{}`) — they "pass" while +asserting nothing (see §3). + +--- + +## 2. Suspicious code — real latent bugs (ranked) + +| ID | Location | Defect | Severity | Trigger | +|----|----------|--------|----------|---------| +| **B1** | `gibbs_sampler/SingleThreadedGibbsSampler.h:265` | `operator>>` reads the `DataModel` with `>>` but then chains `<<` (write) for `mDomain`, `mNumBins`, `mBinLength`, `mNumPatterns`, `mdDomainLength`, `mAlpha`. Checkpoint **restore does not reload the atomic domain / bin geometry** — it writes them into the read archive instead. Compare the correct write path at line 257. | High\* | any checkpoint save→restore | +| **B2** | `math/MatrixMath.cpp:54` (dense), `:72` (sparse) | `nonZeroMean` returns `sum / nNonZeroes` with no guard for `nNonZeroes == 0` → `0/0 = NaN`. Feeds `DenseNormalModel.h:95` / `SparseNormalModel.h:82`: `meanD = nonZeroMean(mDMatrix); mLambda = alpha*sqrt(nPatterns()/meanD)` → poisons `mLambda`, `mMaxGibbsMass`, and every sample. | Med-High | all-zero data matrix or subset | +| **B3** | `math/VectorMath.cpp:7,17,40,50,74` | Dense/hybrid `min`/`max`/`whichMax(Vector/HybridVector)` dereference `v[0]` with no size check. Same empty-container class just fixed for the *sparse* overloads (issue 12); the dense/hybrid ones were left unguarded. `MatrixMath.h:37,49` `min/max(MatrixType)` also call `getCol(0)` with no `nCol()>0` check. | Med | `gaps::max(Vector(0))`, 0-column matrix | +| B4 | `atomic/AtomicDomain.cpp:57,60` | `randomFreePosition` uses `uniform64(1, mDomainLength)` inclusive of `mDomainLength`; a returned `pos == mDomainLength` gives `pos/mBinLength == nElements` → one-past-last row → OOB in `changeMatrix`/`updateAPMatrix`. `move()` (`:197`) correctly uses `rbound - 1`. | Med | `uniform64` returns exactly `mDomainLength` (~1/2^64) | +| B5 | `gibbs_sampler/SingleThreadedGibbsSampler.h:241-243` | `exchange()` computes `mass.value()` **before** the `hasValue()` guard. Benign today only because `OptionalFloat::value()` returns `0.f` when empty and `&&` short-circuits; wrong evaluation order. | Med-Low | `sampleExchange` returns empty `OptionalFloat` | +| B6 | `math/MatrixMath.cpp:19,35`; `MatrixMath.h:72` | `float size = mat.nRow() * mat.nCol();` multiplies two 32-bit `unsigned` in 32-bit arithmetic before widening → overflow for matrices > ~4.3e9 cells → wrong `sparsity`/`mean`. | Med | very large data (e.g. 50k × 100k) | + +**\* B1 scope:** checkpoints are **disabled in the default build** +(`-DGAPS_DISABLE_CHECKPOINTS` in `configure.ac:48` and `Makevars.win`; +`checkpointsEnabled() == FALSE`). B1 is therefore **latent** — not reachable by +users of the shipped package — but it is a genuine correctness bug and a one-token +fix (`<<` → `>>`), and a serialization round-trip test would catch it. + +Additional robustness concerns noted (lower priority): `exponential()` +(`math/Random.cpp:178`) can yield `+inf` when `uniform()` returns exactly 0; +`mMaxGibbsMass /= mLambda` divides by zero if `alpha == 0`; `mean()`/`sparsity()` +divide by `nRow*nCol` with no empty-matrix guard. + +**Verified OK (looked suspicious, but safe):** `SparseIterator<1>` underflow of +`mSparseIndex` on empty is guarded by `atEnd()` before any deref; `AtomicDomain` +erase/move iterator fix-ups (the previously-fixed area) are self-consistent. + +--- + +## 3. Test-coverage gaps + +### 3.1 Empty / stub TEST_CASEs (registered, assert nothing) — all in `testSerialization.cpp` + +| Line | Tag | +|------|-----| +| 114 | `HybridVector Serialization` | +| 118 | `SparseVector Serialization` | +| 165 | `HybridMatrix Serialization` | +| 169 | `SparseMatrix Serialization` | +| 248 | `GapsParameters Serialization` — checkpoint/restart relies on this; we just kept its fields for format compatibility | +| 252 | `GapsStatistics Serialization` | + +Serialization is the checkpoint/resume mechanism; 6 of 12 serializable types have a +registered-but-empty test. + +### 3.2 No live coverage at all +- `Random.h`: `truncNormal`, `truncGammaUpper`, `poisson`, `exponential` — the core + proposal distributions of the Gibbs sampler. (Covered only under `#if 0` in + `testRandom.cpp:127-204`.) +- `Math.h`: `d_gamma`/`p_gamma`/`q_gamma`/`d_norm`/`p_norm`/`q_norm` — only under + `#if 0` (`testRandom.cpp:206-214`). +- `AlphaParameters` dense-vs-sparse consistency (the numerical heart of the sampler) + — only under `#if 0` (`testSparseGibbsSampler.cpp:40-248`). +- `VectorMath`/`MatrixMath`: `whichMax`, `elementSq`, `pmax`, `dot_diff`, + `sparsity`, `nonZeroMean`, `mean` — no dedicated tests. + +### 3.3 Shallow tests +- `testSamplerHighLevel.cpp:53` "Sampler Update" — constructs but never calls + `update()` (needs `sync()`+`extraInitialization()` first); no assertions. +- `testSparseGibbsSampler.cpp:8` — only checks initial `chiSq()`; never calls + `update()`. The dense test asserts chiSq decreases; the sparse one does not. + +### 3.4 Dead code +- `gibbs_sampler/SparseNegativeBinomialModel.{h,cpp}` — 0-byte files, referenced + nowhere, not compiled. Delete or mark WIP. + +### 3.5 Disabled tests worth reviving (`#if 0`) +- `testRandom.cpp:127-214` — poisson/exponential means + gamma/norm exact values + (High; needs port from old `gaps::random::` global API to `GapsRng`). +- `testSparseGibbsSampler.cpp:40-248` — dense-vs-sparse `alphaParameters` equality + (High; needs port to `SingleThreadedGibbsSampler` API). +- `testSerialization.cpp:256-295` — `AtomicDomain` round-trip (Med-High; needs a + friend/accessor for `mAtoms`/`mDomainLength`). + +--- + +## 4. Plan + +**Phase 1 (now): fix confirmed bugs B1 + B2 + B3 with regression tests.** +- B1: `<<` → `>>` in the sampler deserialization operator; regression = a sampler + serialize→deserialize round-trip. (Also fills the empty serialization stubs + conceptually.) +- B2: guard `nonZeroMean` for `nNonZeroes == 0` (return 0); test on an all-zero + matrix (dense + sparse). +- B3: guard dense/hybrid `min`/`max`/`whichMax` (and `min/max(Matrix)`) for empty + input; test size-0 vector / 0-column matrix. +- Document each as an entry in `132-LLM-assisted-solved-issues.md` (fixed bugs). + +**Phase 2 (coverage).** +- ~~Fill the 6 empty serialization TEST_CASEs.~~ **DONE** (2026-07-07) — all 6 filled + with round-trip tests; uncovered and fixed issue 16 (`SparseVector` deserialize + dropped all stored values). Also removed the now-confirmed-unnecessary + `maxThreads`/`asynchronousUpdates` `GapsParameters` fields (they were never in the + serialized format — see `async-removal.md` §4.2). +- Revive `testRandom` distribution tests and the `AlphaParameters` dense-vs-sparse + consistency test against the current API. +- Make `testSamplerHighLevel`/`testSparseGibbsSampler` actually run `update()` and + assert chiSq decreases. + +**Checkpoints — RESOLVED (2026-07-08).** Policy: checkpoints stay **off by default** +(they are an emergency/debug save-resume feature that serializes on every run), but +must be correct and cleanly enableable. Previously `-DGAPS_DISABLE_CHECKPOINTS` was +hard-coded in `configure.ac`; added a proper `--enable-checkpoints` toggle (variable +`cpp_checkpoints_disable`, default `yes`) mirroring `--enable-cpp-tests`. The macro +is now added only when checkpoints are disabled. Verified: default build → +`checkpointsEnabled() == FALSE`; `--configure-args="--enable-checkpoints yes"` → +`TRUE`, the (previously dead) checkpoint code compiles, and a `checkpointOutFile` +→ `checkpointInFile` save/resume round-trip runs and returns a finite result +(exercises the issue 13 `>>` fix). cpp unit tests stay **on by default** +(`cpp_tests_disable=no`) — no runtime overhead, like testthat. + +Note: `src/Makevars.win` keeps `-DGAPS_DISABLE_CHECKPOINTS` hard-coded (Windows has +no `configure`), so Windows defaults to checkpoints-off too; a Windows dev enables +them by editing `Makevars.win`. + +**Phase 3 (cleanup): decide the fate of `SparseNegativeBinomialModel.{h,cpp}`.** + +**Phase 4 (R test infrastructure) — after the phases above.** +- Sort out the native R `testthat` tests (`tests/testthat/test_*.R`): the suite is + currently a bit murky — review what's actually exercised, what's stale/skipped, + and tidy it into a coherent, informative set. +- Wire the C++ Catch unit tests into `testthat`: make `run_catch_unit_tests()` run + as part of the R test suite (there is a `test_cpp.R` today — confirm/fix it so the + cpp tests are actually invoked and reported through testthat). + +**Tomorrow's starting point:** revive one of the High-value `#if 0` tests — the +`testRandom` distribution tests (poisson/exponential/truncNormal/gamma/norm, port +from the old `gaps::random::` global API to `GapsRng`) or the +`testSparseGibbsSampler` dense-vs-sparse `alphaParameters` consistency test (port +from `GibbsSampler`/`` to +`SingleThreadedGibbsSampler`/``). Reviving may +surface real bugs, as filling the serialization stubs did (issue 16). diff --git a/dev-notes/annotation-weights-sampling-issue-eng.md b/dev-notes/annotation-weights-sampling-issue-eng.md new file mode 100644 index 00000000..ea01dfd0 --- /dev/null +++ b/dev-notes/annotation-weights-sampling-issue-eng.md @@ -0,0 +1,167 @@ +# Annotation-weighted subset sampling — problem report and options + +Status: **open design issue**, deliberately left out of branch +`132-uncertainty-improvements` (which is about the uncertainty model). This +report documents the problem so the decision and evidence are not lost. + +Surfaced while tidying the R `testthat` suite (Phase 4): the test +`test_subset_data.R` "subsetting data with annotation weights" is entirely +commented out, with a TODO. Investigating why it is commented out revealed a +genuine correctness gap in the feature it was meant to cover. + +--- + +## 1. What the feature is + +For **distributed** CoGAPS (`GWCoGAPS` / `scCoGAPS`, i.e. `distributed = +"genome-wide"` or `"single-cell"`), the data is partitioned into `nSets` +subsets that are factored in parallel and then stitched back together. + +`setAnnotationWeights(params, annotation, weights)` lets the user bias that +partitioning. Every gene (genome-wide) or sample (single-cell) carries a +category label from `annotation`; every category carries a number in `weights`. +Subsets are then drawn so that heavily-weighted categories are over-represented, +instead of the default uniform partition. + +```r +weight <- c(A = 1, B = 2, C = 3) +anno <- sample(names(weight), nrow(data), replace = TRUE) +params <- setAnnotationWeights(CogapsParams(), anno, weight) +res <- CoGAPS(data, params, distributed = "genome-wide", ...) +``` + +The two values are stored in the `CogapsParams` slots `samplingAnnotation` and +`samplingWeight` (`R/class-CogapsParams.R`), set together by +`setAnnotationWeights` (`R/methods-CogapsParams.R`). + +--- + +## 2. How it works today + +`createSets()` (`R/SubsetData.R`) dispatches to `sampleWithAnnotationWeights()` +whenever `samplingAnnotation` is set: + +```r +sampleWithAnnotationWeights <- function(allParams, setSize) +{ + weight <- allParams$gaps@samplingWeight + weight <- weight[order(names(weight))] + groups <- sort(unique(allParams$gaps@samplingAnnotation)) + + lapply(1:allParams$gaps@nSets, function(i) + { + # 1. draw setSize category tickets, proportional to weight + groupCount <- sample(groups, size = setSize, replace = TRUE, prob = weight) + # 2. for each category, draw that many member indices WITH REPLACEMENT + sort(unlist(sapply(groups, function(g) + { + groupNdx <- which(allParams$gaps@samplingAnnotation == g) + sample(groupNdx, size = sum(groupCount == g), replace = TRUE) + }))) + }) +} +``` + +Both draws use `replace = TRUE`. That is the root of the problem: step 2 can pick +the same gene several times within one subset, and nothing links subsets, so a +gene may land in several subsets or in none. + +--- + +## 3. The defects (empirically confirmed) + +Reproduction — GIST (1363 genes), three categories A/B/C with weights 1/2/3, +`distributed = "genome-wide"`, `nSets = 4`: + +| observation | value | should be | +|---|---|---| +| duplicates **within** each subset | ~75 of 340 (**~22 %**) | 0 (a gene at most once per subset) | +| unique genes covered across all subsets | **597 of 1363** (~44 %) | all genes represented | +| `nrow(res@featureLoadings)` | **1360** (= 4 × 340, i.e. counts duplicates) | 1363 (one row per gene) | + +So a weighted run (a) puts the same gene in a subset multiple times, (b) leaves +~56 % of genes unsampled, and (c) emits duplicated genes as **separate rows** in +`featureLoadings` instead of collapsing them. The output matrix is therefore +mis-shaped (rows ≠ genes) and its rows are not unique. + +This matches the original author's TODO, verbatim from the commented test: + +> address how weighted sampling works with duplicates, do we need to allow +> passing a value for setSize in this case? we should collapse down using the +> mean; prevent multiple copies from being in the same set + +## 4. Why the existing test is commented out + +The commented test asserts the *fixed* behaviour: + +```r +expect_equal(nrow(result@featureLoadings), nrow(GIST.matrix)) # 1363 +expect_equal(sum(sapply(sets, length)), nrow(result@featureLoadings)) # full coverage +``` + +Against the current implementation the first assertion is false (`1360 ≠ 1363`), +so the test cannot pass without changing the feature. It was commented out rather +than fixed — leaving a silent gap: the `sampleWithAnnotationWeights` path has +**zero** test coverage. + +--- + +## 5. Options + +### A. Delete the stub, file the design gap as a tracked issue *(recommended for this branch)* +Remove the commented-out test and open an issue capturing §3 (the quantified +defect) and §6 (a fix sketch). Rationale: the test describes behaviour the +feature does not yet provide; making it pass is a feature change out of scope for +the uncertainty branch. A commented-out test is worse than none — it reads as +"covered" while testing nothing. +- Pro: keeps branch 132 focused; the gap is recorded, not lost. +- Con: the sampling path stays untested until the feature is fixed. + +### B. Add a characterization test of the *current* behaviour +`sampleWithAnnotationWeights` has no tests at all. Add one that asserts only what +is true today — it runs without error, returns `nSets` subsets, and the +high-weight category is over-represented relative to a uniform draw — with an +explicit comment (and a filed issue) that duplicate handling is a known gap. +- Pro: real regression coverage of an untested code path, cheaply. +- Con: risks blessing buggy behaviour; must be clearly labelled as + characterization, not correctness. + +### C. Fix the feature, then test it *(out of scope here)* +Implement the intended semantics and test the fixed behaviour. This is a feature +change to the distributed subsetting and result assembly, not test tidying, and +belongs in its own branch/PR — not in the uncertainty branch. + +**Recommendation:** **A** for branch 132 (delete + issue). If we want to avoid +losing coverage entirely, **B** is an acceptable middle ground. **C** is a +separate piece of work. + +--- + +## 6. Sketch of a real fix (for the issue / option C) + +Not implemented here; recorded so the issue is actionable. +1. **Dedupe within a subset.** Draw category tickets as now, but draw member + indices **without replacement** per category (`replace = FALSE`), capping the + count at the category size. Decide the policy when a category is exhausted + (spill to other categories, or shrink the subset). +2. **Collapse duplicates by mean.** If a gene still ends up represented more than + once (across the stitched result), average its rows rather than emitting + copies — as the TODO says ("collapse down using the mean"). +3. **Coverage / `setSize`.** Clarify the contract: is weighted sampling meant to + cover every gene (a weighted *partition*) or to draw a weighted *sample* + (coverage < 100 % by design)? The current output shape (rows ≠ genes) implies + the former was intended. Possibly expose `setSize` so the caller controls it. +4. Re-enable the test to assert the chosen contract (unique rows, expected shape, + category proportions within tolerance). + +--- + +## 7. Code map + +| what | location | +|---|---| +| params slots `samplingAnnotation` / `samplingWeight` | `R/class-CogapsParams.R` (slot docs ~27–29) | +| `setAnnotationWeights` generic / method | `R/class-CogapsParams.R`; `R/methods-CogapsParams.R` | +| weighted draw (the defect) | `R/SubsetData.R` — `sampleWithAnnotationWeights()` | +| dispatch into it | `R/SubsetData.R` — `createSets()` (the `samplingAnnotation` branch) | +| commented-out test (the stub) | `tests/testthat/test_subset_data.R` — "subsetting data with annotation weights" | diff --git a/dev-notes/async-removal.md b/dev-notes/async-removal.md new file mode 100644 index 00000000..26db7af0 --- /dev/null +++ b/dev-notes/async-removal.md @@ -0,0 +1,240 @@ +# Async sampler removal — implementation report + +**Branch:** `132-uncertainty-improvements` +**Date:** 2026-07-06 +**Spec:** `remove-async-plan-eng.md` (this report is the retrospective record of +executing that spec) + +--- + +## 1. Summary + +The asynchronous (OpenMP multi-threaded) Gibbs sampler was **completely removed** +from CoGAPS. It broke MCMC detailed balance: proposals were generated and applied +in parallel out of a `ProposalQueue`, so the Markov chain was not sampling from the +intended posterior. CoGAPS now always runs the sequential +`SingleThreadedGibbsSampler`. + +The change is **behaviour-preserving for all users**: the async dispatch had +already been commented out, so every run already fell through to the sequential +sampler. A four-configuration parity check (below) confirms bit-for-bit identical +results before and after removal. + +Scope: **8 files deleted, 28 modified** (1589 insertions / 369 deletions; the large +`configure` insertion count is from regenerating it — see §7). + +--- + +## 2. Rationale + +- The async sampler's `ProposalQueue` batched conflict-free proposals and applied + them in parallel (`#pragma omp parallel for`). Overlapping accept/reject decisions + within a batch violate detailed balance. +- The path was already dead: `chooseSampler` in `GapsRunner.cpp` had its async + branch commented out, always constructing `SingleThreadedGibbsSampler`. +- The async classes still compiled into the package and were exercised by tests, so + a clean removal was still required. +- OpenMP existed **only** to support the async sampler. Distributed CoGAPS + (GWCoGAPS / scCoGAPS) parallelises at the R process level via `BiocParallel`, not + C++ threads, and is unaffected. + +--- + +## 3. Files deleted (async-only) + +| File | Role | +|------|------| +| `src/gibbs_sampler/AsynchronousGibbsSampler.h` | the async sampler (was already non-compiling in isolation) | +| `src/atomic/ProposalQueue.{h,cpp}` | conflict-free proposal batching for parallel apply | +| `src/atomic/ConcurrentAtomicDomain.{h,cpp}` | OpenMP-thread-safe atom domain | +| `src/atomic/ConcurrentAtom.{h,cpp}` | atom type for the concurrent domain | +| `src/cpp_tests/testConcurrentAtomicDomain.cpp` | test for the concurrent domain | + +Kept (sequential structures, shared by the surviving sampler): `src/atomic/Atom.*`, +`src/atomic/AtomicDomain.*`, `DenseNormalModel`, `SparseNormalModel`. + +--- + +## 4. C++ changes + +### 4.1 Core driver — `src/GapsRunner.cpp` +- Removed the async `#include`, `#include `, and the `if (asynchronousUpdates)` + branch in `chooseSampler`. +- `updateSampler` no longer passes `maxThreads` into `update()`/`sync()`. +- Deleted `calculateNumberOfThreads()` (used `omp_get_max_threads()`) and its call. +- Dropped `result.averageQueueLengthA/P = ...getAverageQueueLength()`. + +### 4.2 `GapsParameters` — fields removed + +> **Update (2026-07-07):** the fields `bool asynchronousUpdates` and +> `unsigned maxThreads` were initially **kept** on the belief they were part of the +> checkpoint serialization format. That was wrong: `GapsParameters::operator<>` +> serialize only 11 fields (`seed, nGenes, nSamples, nPatterns, nIterations, alphaA, +> alphaP, maxGibbsMassA, maxGibbsMassP, useSparseOptimization, checkpointInterval`) +> and never touched `maxThreads`/`asynchronousUpdates`. So the format was never +> affected, and both fields (plus their `print()` lines and the forced assignments +> in `Cogaps.cpp`) were subsequently **removed** entirely. The R arguments +> `nThreads`/`asynchronousUpdates` remain as deprecated no-ops (§4.8). Verified by +> the new `[serialization][gapsparameters]` round-trip test and unchanged parity. + +### 4.3 Samplers / data structures (§ "variant B" OpenMP strip) +- `SingleThreadedGibbsSampler`: removed `getAverageQueueLength()`; `update(nSteps, + nThreads)` → `update(nSteps)`. +- `DenseNormalModel::sync` / `SparseNormalModel::sync`: dropped the `nThreads` + parameter; removed the `#pragma omp parallel for` from the dense AP-transpose loop. +- `HybridVector::add`/`set`: removed the `#pragma omp atomic` directives (thread + safety with no threads = dead overhead). + +### 4.4 `GapsResult.h` +- Removed `averageQueueLengthA` / `averageQueueLengthP` (async queue diagnostics, + not read by any downstream R code). + +### 4.5 OpenMP infrastructure (variant B — full removal) +- `src/utils/GlobalConfig.h`: removed the `#ifdef _OPENMP → #define __GAPS_OPENMP__` + block and the "Compiled with OpenMP" status line (now "OpenMP: disabled"). +- `src/Cogaps.cpp`: removed `compiledWithOpenMPSupport_cpp()`. +- Build files: see §7. + +--- + +## 5. R layer + +- **Deprecated stubs (variant chosen):** `nThreads` and `asynchronousUpdates` remain + arguments of `CoGAPS` / `scCoGAPS` / `GWCoGAPS` for backward compatibility but are + ignored. A `warning` fires **only** on a non-default value + (`nThreads != 1 || isTRUE(asynchronousUpdates)`), so default calls stay silent. + `CoGAPS`'s `asynchronousUpdates` default flipped `TRUE → FALSE`. +- **`compiledWithOpenMPSupport()` (variant B1):** the exported R function is kept + (public API preserved) but now returns `FALSE` directly; the C++ `_cpp` backend + and its RcppExports entry were removed. +- `R/HelperFunctions.R`: removed the now-dead "can't run multi-threaded and + distributed" warning. +- `R/DistributedCogaps.R`: `callInternalCoGAPS` still sets `asynchronousUpdates <- + FALSE` / `nThreads <- 1` on the inner param list — kept (harmless, sets the + ignored fields to quiet values). +- roxygen `@param` / `@return` updated; `man/*.Rd` regenerated. +- `RcppExports.{cpp,R}` regenerated (`Rcpp::compileAttributes`). + +Distributed mode (GWCoGAPS / scCoGAPS) is unaffected — it parallelises over data +subsets with `BiocParallel::bplapply`, each worker running the sequential sampler. + +--- + +## 6. Tests + +- `testConcurrentAtomicDomain.cpp` deleted. +- `testSamplerHighLevel.cpp`: async sampler variants removed (only the two + `SingleThreadedGibbsSampler` variants remain; the "Sampler Update" case is a + construction smoke test — calling `update()` there would need `sync()` + + `extraInitialization()` first, or `mOtherMatrix` is NULL → segfault). +- `testSerialization.cpp`: the async-only "ProposalQueue Serialization" case (which + also used a stale API) removed; the other serialization cases stay. +- `testDenseGibbsSampler.cpp`: `update(100, 1)` callers updated to `update(100)`. +- R tests: `test_seed_consistency.R` and `test_top_level.R` had their `nThreads` + variants removed (they tested multi-thread determinism that no longer exists); + seed-consistency coverage for the dense and sparse samplers is retained. +- `inst/scripts/debugRuns.R`: `asynchronousUpdates` / `nThreads` invocations removed. + +C++ Catch suite: **46/46 test cases pass** after removal (unchanged count — the +async test was never in the build list). + +--- + +## 7. Build system — `configure` regenerated (not hand-patched) + +`configure.ac` edits: removed the `AC_ARG_ENABLE(openmp)` / `AX_OPENMP` block and its +`OPENMP_CXXFLAGS` injection; removed the three async object files +(`atomic/ConcurrentAtom.o`, `atomic/ConcurrentAtomicDomain.o`, `atomic/ProposalQueue.o`). +`src/Makevars.win` had the same three object files removed. + +The generated `configure` was **regenerated from `configure.ac`**, not hand-edited. +Because `automake`/`aclocal` is not installed (only `autoconf` + `autoconf-archive`), +the standard `autoreconf` fails on `aclocal`. Worked around it by running `autoconf` +against a temporary `aclocal.m4` that `m4_include`s the three archive macros +(`ax_openmp.m4`, `ax_compiler_vendor.m4`, `ax_compiler_version.m4`); the temp +`aclocal.m4` is **not** left in the repo. The regenerated `configure` is cleaner than +a hand patch: zero `openmp`/`OPENMP` references (including the residual +`enable_openmp` boilerplate), zero async objects, and the `AX_COMPILER_*` macros now +expand properly (they were literal no-ops in the previously committed `configure`). +This accounts for the large `configure` diff. Build validated end-to-end (RC 0). + +> For a standard `autoreconf` workflow later: `brew install automake`. + +--- + +## 8. Verification + +All run against the actually rebuilt, async-removed package (an early check +accidentally ran against a stale install because a build had silently failed on a +`update(x,y)` caller — caught and fixed). + +- **Build:** `R CMD INSTALL --preclean` → RC 0, no unresolved symbols. +- **Symbol audit:** `grep -rn "Asynchronous\|ProposalQueue\|Concurrent" src/` → empty; + no `__GAPS_OPENMP__` / `#pragma omp` / `omp.h` remain. +- **C++ tests:** 46/46 Catch cases pass (823904 assertions). +- **Parity (§11 of the spec):** `CoGAPS` on `GIST.matrix`, `seed=42`, + `nIterations=1000`, compared before/after with `identical()`: + + | Config | featureLoadings | sampleFactors | + |--------|:---:|:---:| + | dense | identical | identical | + | sparse (`sparseOptimization=TRUE`) | identical | identical | + | uncertainty (`GIST.uncertainty`) | identical | identical | + | genome-wide distributed (`nSets=2`, `SerialParam`) | identical | identical | + + (CoGAPS was first confirmed bit-reproducible for a fixed seed, so `identical()` + is a valid criterion.) +- **Deprecation behaviour:** `compiledWithOpenMPSupport()` → `FALSE`; a default + `CoGAPS()` call emits 0 deprecation warnings; `CoGAPS(..., nThreads=4)` emits + exactly 1 and still returns a `CogapsResult`. +- **Affected R tests:** `test_seed_consistency.R` (4 passed) and `test_top_level.R` + (36 passed), 0 warnings, after the `nThreads` cleanup. + +--- + +## 9. Related bug found and fixed during this work + +Enabling `testSamplerHighLevel` in the build surfaced a **real, latent product bug** +(unrelated to async): `gaps::min/max(const SparseVector&)` dereferenced the first +element before checking `atEnd()`, segfaulting on an empty (all-zero) sparse vector — +reachable on any `sparseOptimization=TRUE` run whose data has an all-zero gene or +sample (`SparseNormalModel` calls `gaps::max(mDMatrix)` at construction). Root-caused +with a standalone AddressSanitizer harness. Fixed (empty → `0.f`) with a regression +test. This is documented separately as **issue 12** in +`132-LLM-assisted-solved-issues.md` and was landed as its own commit +(`dab57f2b`), together with the test-build enablement commit (`989db34f`) that +predates the async removal proper. + +--- + +## 10. Notes / caveats + +- `man/*.Rd` were regenerated with the local **roxygen2 8.0.0**, while `DESCRIPTION` + pins `RoxygenNote: 7.3.2`. The doc *content* is correct; `DESCRIPTION` and + `NAMESPACE` were reverted to avoid roxygen version churn and to keep + `#import(fgsea)` commented out. Regenerating with 7.3.2 later is optional (format + only). +- `GapsParameters` deliberately keeps the two inert fields to preserve checkpoint + binary compatibility. A future release could drop the deprecated R arguments and + those fields together (a breaking change). +- `std::thread` / `std::async` / `std::mutex` / `std::atomic` were never used — all + parallelism was OpenMP pragmas, now gone. + +--- + +## 11. Changeset (uncommitted at time of writing) + +Deleted (8): the five async source pairs/headers listed in §3. + +Modified (28): `src/GapsRunner.cpp`, `src/GapsParameters.h`, `src/GapsResult.h`, +`src/Cogaps.cpp`, `src/RcppExports.cpp`, `src/Makevars.win`, +`src/gibbs_sampler/{SingleThreadedGibbsSampler.h, DenseNormalModel.{h,cpp}, +SparseNormalModel.{h,cpp}}`, `src/data_structures/HybridVector.cpp`, +`src/utils/GlobalConfig.h`, +`src/cpp_tests/{testDenseGibbsSampler.cpp, testSamplerHighLevel.cpp, +testSerialization.cpp}`, `configure`, `configure.ac`, +`R/{CoGAPS.R, HelperFunctions.R, RcppExports.R}`, `man/*.Rd` (4), +`inst/scripts/debugRuns.R`, `tests/testthat/{test_seed_consistency.R, +test_top_level.R}`. + +Ready to commit. diff --git a/dev-notes/master-issues.md b/dev-notes/master-issues.md new file mode 100644 index 00000000..d7a067d2 --- /dev/null +++ b/dev-notes/master-issues.md @@ -0,0 +1,126 @@ +# Defects inherited from master, fixed on this branch + +Branch `132-uncertainty-improvements` is about the uncertainty model. It also +carries a handful of fixes that have nothing to do with uncertainty. This file +records why, so that a reviewer does not have to wonder where they came from. + +All of them were found while merging `master` into the branch (2026-08) and while +writing tests for exported functions that had never been called from any test — +21 of the 37 exported functions had no coverage at all. None of them originate on +this branch: the branch point is `8a21f281` (merge of PR #131), and the defects +are either older than that or came in from `master` afterwards. + +A longer, Russian-language write-up of items 1–5 was prepared separately for the +`master` maintainer; this file is the version that belongs with the code. + +| # | What | Origin | Fixed in | +|---|------|--------|----------| +| 1 | `scCoGAPS()` / `GWCoGAPS()` unusable | `24448b6e` on master | `661a0828` | +| 2 | `patternMarkers` lp test asserted nothing | `24448b6e` on master | `4e74b1bf` | +| 3 | `binaryA()` unusable | older than `8a21f281` | `40eb0f58` | +| 4 | `fromCSV()` returned the wrong slot types | older than `8a21f281` | `40eb0f58` | +| 5 | `show(CogapsParams)` errored | older than `8a21f281` | `40eb0f58` | + +--- + +## 1. `scCoGAPS()` / `GWCoGAPS()` failed on every call + +`24448b6e` ("require nPatterns always") made `nPatterns` a mandatory argument of +the `CogapsParams` initializer, replacing `.Object@nPatterns <- 7`. `CoGAPS()` +was updated to match — `params = new("CogapsParams", nPatterns = nPatterns)` — +but the two deprecated wrappers kept `params = new("CogapsParams")`, which can no +longer be constructed. + +The non-obvious part: passing `nPatterns` did not help either. It goes into `...` +and would reach `CoGAPS()`, but the first statement of each wrapper is +`params@distributed <- ...`, which forces the lazy default and fails first. So +the wrappers worked only with an explicitly built `params` object, and the error +message pointed nowhere useful. + +Fixed by giving both wrappers the same signature treatment as `CoGAPS()`. +`tests/testthat/test_deprecated_wrappers.R` now calls them with and without an +explicit `params` — they had no test coverage and no runnable example, which is +why this went unnoticed. + +## 2. The `patternMarkers` lp test stopped testing anything + +The same commit raised `nPatterns` from 5 to 7 in `test_patternMarkers.R` but +left the `lp` vectors at length 5. `patternMarkers()` then warned "lp length must +equal the number of columns of the Amatrix" and took the invalid-lp path, so the +case meant to cover a well-formed `lp` never exercised it. `expect_no_error()` +does not catch a warning, so the test stayed green while the noise accounted for +2727 of the suite's 2730 warnings. + +Fixed by padding the two vectors that are supposed to be valid to length 7. The +middle call keeps its length-4 vector: it asserts the length warning on purpose. + +## 3. `binaryA()` failed on every call + +```r +binA <- ifelse(calcZ(object) > threshold, 1, 0) +``` + +`calcZ()` has no default for `whichMatrix`, so every call died with +`argument "whichMatrix" is missing`. Fixed by passing `"featureLoadings"` — the +function is named binary**A** and draws a "Heatmap of Standardized Feature +Matrix". + +Still open, left alone deliberately: the name promises a binary matrix, but the +function returns whatever `mtext()` returns. It is a plotting function; changing +its contract is a separate decision. + +## 4. `fromCSV()` returned data.frames where matrices were expected + +All four matrix slots were read with `read.csv()` and handed to +`new("CogapsResult", ...)`. `CogapsResult` extends `LinearEmbeddingMatrix`, whose +slots are matrices, so a `toCSV()`/`fromCSV()` round-trip produced an object with +the wrong slot types. Values and dimnames did survive; only the type was wrong. +Fixed by wrapping the reads in `as.matrix()`. + +## 5. `show()` on `CogapsParams` errored with a checkpoint file set + +```r +cat("checkpointInFile ", checkpointInFile, "\n") +``` + +Missing `object@`, so R looked for a global variable and printing any params +object with `checkpointInFile` set raised `object 'checkpointInFile' not found`. +Fixed by qualifying it. + +This one had been visible all along in `R CMD check` as the "no visible binding +for global variable ‘checkpointInFile’" NOTE — nobody had gone through the NOTEs. + +--- + +## Known gaps left open + +Not defects introduced here, and not fixed here either. Recorded so the next +person does not have to rediscover them. + +- **`nPatterns` has no default any more.** Dropping it broke `new("CogapsParams")` + and `scCoGAPS(data)`, both of which worked in 3.27. The old default of 7 was + arbitrary, but its removal is a backwards-compatibility break worth a conscious + decision. +- **`configure` on master leaves `AX_COMPILER_VENDOR` / `AX_COMPILER_VERSION` + unexpanded.** They end up in `configure` as literal shell commands: it still + runs, but prints `command not found`, leaves `$ax_cv_cxx_compiler_vendor` empty + and thereby makes `--enable-warnings` a silent no-op. Regenerating needs + `aclocal` before `autoconf`; see `src/README.md`. The branch's `configure` is + correct. +- **`--enable-warnings` fails the build under `-Werror`** — on + `-Wcast-function-type-mismatch` raised inside Rcpp's own `routines.h` with + newer clang. No CoGAPS source file produces a warning. +- **Windows builds no C++ tests.** `src/Makevars.win` is maintained by hand and + lists no `cpp_tests` objects, so the Catch suite is empty there. + `tests/testthat/test_cpp.R` now skips with an explanation instead of passing + vacuously. +- **`R CMD check` still reports one NOTE** — `std::cout` / `printf` in the + compiled code (`GapsPrint.h` and Catch2). Clearing it means routing C++ output + through `Rprintf` throughout. +- **12 exported functions still have no test**, down from 21: `MANOVA`, + `buildReport`, `compiledWithOpenMPSupport`, `findConsensusMatrix`, + `getClusteredPatterns`, `getCorrelationToMeanPattern`, `getParam`, + `getRetinaSubset`, `getUnmatchedPatterns`, `plotPatternMarkers`, + `plotResiduals`, `setAnnotationWeights`. +- **Weighted subset sampling is still broken** — separate report in + `annotation-weights-sampling-issue-eng.md`. diff --git a/dev-notes/remove-async-plan-eng.md b/dev-notes/remove-async-plan-eng.md new file mode 100644 index 00000000..84e2e6ac --- /dev/null +++ b/dev-notes/remove-async-plan-eng.md @@ -0,0 +1,364 @@ +# Spec: Complete removal of the asynchronous sampler from CoGAPS + +**Branch:** `132-uncertainty-improvements` +**Date:** 2026-07-04 + +## 1. Goal and rationale + +The asynchronous (multi-threaded) sampler breaks MCMC detailed balance: +proposals are generated and processed in parallel from a queue, which makes the +chain incorrect from the standpoint of Markov-chain theory. The decision is to +**completely remove** the asynchronous path, leaving only the sequential +`SingleThreadedGibbsSampler`. + +Current state: async is already partially disabled — in `src/GapsRunner.cpp` the +`#include` (line 8) and the dispatcher call (lines 72–73) are commented out, so +`chooseSampler` always falls through to `SingleThreadedGibbsSampler`. However, the +async classes are still compiled (listed in Makevars) and still covered by tests, +so a clean removal requires the edits below. + +## 2. Scope of work + +The removal touches three groups: +- **A.** Files that exist only for async → delete. +- **B.** Files that reference async → edit. +- **C.** Auxiliary parallelism (OpenMP) that only makes sense when + `maxThreads > 1` → strip as dead code. + +--- + +## 3. Files to DELETE (async-only) + +| File | Reason | +|------|--------| +| `src/gibbs_sampler/AsynchronousGibbsSampler.h` | the async sampler itself (`#pragma omp parallel for`, line 105) | +| `src/atomic/ProposalQueue.h` | queue of conflict-free `AtomicProposal`s for parallel processing | +| `src/atomic/ProposalQueue.cpp` | queue implementation | +| `src/atomic/ConcurrentAtomicDomain.h` | "OpenMP thread-safe" atom domain | +| `src/atomic/ConcurrentAtomicDomain.cpp` | implementation | +| `src/atomic/ConcurrentAtom.h` | atom type used only by `ConcurrentAtomicDomain` | +| `src/atomic/ConcurrentAtom.cpp` | implementation | +| `src/cpp_tests/testConcurrentAtomicDomain.cpp` | test exclusively for `ConcurrentAtomicDomain` | + +**Keep (these are sync structures, NOT async):** `src/atomic/Atom.{h,cpp}`, +`src/atomic/AtomicDomain.{h,cpp}` — used by `SingleThreadedGibbsSampler`. +`DenseNormalModel` / `SparseNormalModel` (the base DataModels) contain no async +references and are shared by both samplers. + +--- + +## 4. Files to EDIT + +### 4.1. `src/GapsRunner.cpp` +- Remove the commented `#include "gibbs_sampler/AsynchronousGibbsSampler.h"` (line 8). +- In `chooseSampler` (lines 65–78) remove the `if (params.asynchronousUpdates)` + branch (lines 69–74) entirely; keep the direct call to `SingleThreadedGibbsSampler`. +- In `updateSampler` (lines 201–222) remove passing `params.maxThreads` into + `update()` / `sync()` (lines 207, 210, 216, 219) — the calls become single-threaded. +- Remove `calculateNumberOfThreads` (lines 352–364) and its call (line 441), + as well as `#include ` (lines 21–23), if no longer used. +- Remove the assignment `result.averageQueueLengthA/P = ...getAverageQueueLength()` + (lines 474–475) — see §4.5. + +### 4.2. `src/GapsParameters.h` — DO NOT change the struct + +**Decision: leave the `GapsParameters` struct untouched.** The fields +`bool asynchronousUpdates` (line 63) and `unsigned maxThreads` (line 44) +**remain** — this preserves the binary serialization/checkpoint format unchanged. +The fields become "dead" (they no longer affect sampler selection, since the async +dispatcher is removed) but neutralized: +- `maxThreads(1)` — already the default (line 89), keep it. +- `asynchronousUpdates` — change the default `true` (line 108) → **`false`** + (the only edit in this file), so the flag is not misleading. + +### 4.3. `src/GapsParameters.cpp` +- The prints of `maxThreads` (line 17) and `asynchronousUpdates` (line 23) — + **keep** (the fields still exist, the output is harmless). + +### 4.4. `src/Cogaps.cpp` +- Reading the R parameters `nThreads`→`maxThreads` (line 85) and + `asynchronousUpdates` (line 102) — hard-fix to `1` / `false`, **not letting R + override them**: + ```cpp + params.maxThreads = 1; // async removed — always single-threaded + params.asynchronousUpdates = false; // async removed + ``` + (or, if R stops passing these keys — just drop the reads and rely on the defaults + from §4.2). Reconcile with the R-API decision (§4.8). +- Remove the return to R of `averageQueueLengthA/P` (lines 177–178). +- `compiledWithOpenMPSupport_cpp()` (lines 233–240) — removed as part of variant B + (see §5.3–5.4). + +### 4.5. `src/GapsResult.h` +- Remove the fields `float averageQueueLengthA` / `averageQueueLengthP` + (lines 34–35) — this is async queue-length diagnostics. + +### 4.6. Build +- `src/Makevars` (line 4, `OBJECTS`): remove `atomic/ConcurrentAtom.o`, + `atomic/ConcurrentAtomicDomain.o`, `atomic/ProposalQueue.o`. +- `src/Makevars.win` (lines 13, 15, 16): remove the same three object files. +- Check `src/Makevars.in` in case the object files are listed there too. + +### 4.7. C++ tests +- `src/cpp_tests/testSamplerHighLevel.cpp`: remove + `#include "../gibbs_sampler/AsynchronousGibbsSampler.h"` (line 7) and + `INIT_SAMPLER(..., AsynchronousGibbsSampler, ...)` (lines 45–46, 61–62). +- `src/cpp_tests/testSerialization.cpp`: remove `#include "../atomic/ProposalQueue.h"` + (line 7) and the "ProposalQueue Serialization" test case (lines 298–347). + +### 4.8. R layer — deprecated stubs with a warning + +**Decision:** the arguments `asynchronousUpdates` and `nThreads` **remain** in the +signatures of `CoGAPS` / `scCoGAPS` / `GWCoGAPS` (backward compatibility with old +scripts and Bioconductor), but are **functionally ignored** — a run is always +single-threaded sequential (C++ forces `maxThreads=1` / `asynchronousUpdates=false`, +§4.4). A `warning` is emitted **only on an attempt to use a non-default value**, so +that default calls stay silent. + +- `R/CoGAPS.R`: + - Change the default `asynchronousUpdates=TRUE` → **`FALSE`** (line 92), otherwise + a plain `CoGAPS(data)` would warn every time. `nThreads=1` — keep the default. + - Add a deprecation warning at the top of the body: + ```r + if (!identical(nThreads, 1) || isTRUE(asynchronousUpdates)) + warning("'nThreads' and 'asynchronousUpdates' are deprecated and ignored; ", + "CoGAPS now always runs single-threaded (async broke MCMC balance)") + ``` + - The arguments no longer affect the result; the forced `1`/`false` go to C++. +- `R/HelperFunctions.R`: the warning about `nThreads` (lines 235–236) — + remove/subsume it into the new deprecation warning (do not duplicate). +- `R/DistributedCogaps.R`: lines 32–33 (`asynchronousUpdates <- FALSE` / + `nThreads <- 1`) — **keep** (they set the ignored fields to the "quiet" values; + the `callInternalCoGAPS` path calls C++ directly and does not trigger the warning). +- roxygen: mark both parameters as *deprecated* in `@param`; regenerate the + `.Rd` (man/) and `NAMESPACE` if needed. + +> Later (in a separate release) the stubs can be removed entirely — then the +> arguments, the forcing in `Cogaps.cpp`, and the test edits go away. + +### 4.9. Impact on the distributed mode (GWCoGAPS / scCoGAPS) + +**Conclusion: removing async does not break the distributed mode.** GWCoGAPS +(genome-wide) and scCoGAPS (single-cell) **do not use** async — each worker already +runs the purely sequential `SingleThreadedGibbsSampler`. + +Their parallelism lives at a **different level** — above the MCMC chain, not inside it: +- `R/DistributedCogaps.R` splits the data into subsets (`createSets`, line 56) — + by rows (genes) for genome-wide, by columns (cells) for single-cell. +- `BiocParallel::bplapply(..., BPPARAM=...)` (lines 60–61, 68–72, 97–101) launches + an **independent CoGAPS on each subset in a separate worker process** + (default `MulticoreParam`). +- The results are stitched: `findConsensusMatrix` → a second pass with a fixed + matrix → `stitchTogether`. + +This is parallelism at the level of **R processes** (independent, correct sequential +chains over disjoint data subsets) — it **does not break** detailed balance and is +in no way tied to OpenMP/async in C++. + +Comparison of the two kinds of parallelism: + +| | Async (removed) | Distributed (GWCoGAPS/scCoGAPS) | +|---|---|---| +| Level | inside a single MCMC chain | across chains, different data subsets | +| Mechanism | OpenMP (`ProposalQueue`, `ConcurrentAtomicDomain`) in C++ | `BiocParallel::bplapply` in R | +| MCMC correctness | **breaks** detailed balance | correct | +| Inner sampler | `AsynchronousGibbsSampler` | `SingleThreadedGibbsSampler` | + +**The only edit here:** in `callInternalCoGAPS` (`R/DistributedCogaps.R`) remove +lines 32–33 that forcibly disable async: +```r +allParams$asynchronousUpdates <- FALSE +allParams$nThreads <- 1 +``` +After removing the `asynchronousUpdates`/`nThreads` fields (§4.8) these lines would +reference non-existent parameters. Reconcile with the decision in §4.8 +(full removal vs. deprecated stubs). Do **not** touch the subset-splitting logic, +`bplapply`/`BPPARAM`, or the pattern matching/stitching. + +### 4.10. Auto-generated +- `src/RcppExports.cpp` — **do not edit by hand**, it is regenerated from the R + layer (`Rcpp::compileAttributes`). + +--- + +## 5. Complete OpenMP removal (variant B) + +**Decision: strip OpenMP entirely.** After async removal the only remaining pragmas +are thread safety without threads (dead overhead), and GWCoGAPS/scCoGAPS do not +depend on OpenMP (their parallelism is `BiocParallel` at the R process level, §4.9). + +### 5.1. Pragmas and threading calls in C++ +- `src/gibbs_sampler/DenseNormalModel.cpp:26` — remove + `#pragma omp parallel for num_threads(nThreads)` in `sync()`, keeping a plain loop. +- `src/data_structures/HybridVector.cpp` — remove `#pragma omp atomic` (lines 60, + 65, 77, 82) in `add()` / `set()` and the "can be called from multiple concurrent + OpenMP threads" comments. +- `src/GapsRunner.cpp` — remove `#include ` (lines 21–23), the function + `calculateNumberOfThreads` (`omp_get_max_threads`, lines 352–364) and its call + (line 441) (already in §4.1). +- Remove the `nThreads` parameter from the `sync()` signatures: + `DenseNormalModel.h:61`, `DenseNormalModel.cpp:20`, `SparseNormalModel.h:27`, + `SparseNormalModel.cpp:27`; and from `SingleThreadedGibbsSampler::update()` + (`SingleThreadedGibbsSampler.h:45, 118`). + +### 5.2. Build infrastructure and macros +- `src/utils/GlobalConfig.h` — remove the `#ifdef _OPENMP / #define __GAPS_OPENMP__` + block (lines 12–14) and the "Compiled with OpenMP" status line in `configReport` + (lines 47–51, keep only the SIMD report or replace with "OpenMP: disabled"). +- `configure.ac` (lines 56–64) — remove `AC_ARG_ENABLE(openmp)`, `AX_OPENMP` and the + addition of `OPENMP_CXXFLAGS` to `GAPS_CXX_FLAGS`/`GAPS_LIBS`. **Regenerate + `configure`** (`autoreconf`/`autoconf`) — the `configure` file is auto-generated, + do not edit by hand (the corresponding blocks around lines ~653, 1288, 2921–2933 + disappear on regeneration). Optionally remove `m4/ax_openmp.m4` if it is no longer + needed anywhere. +- `src/Makevars.win` — contains no OpenMP flags (PKG_CXXFLAGS/LIBS are empty), no + flag edits needed (only the object list from §4.6). +- `src/Makevars` — generated from `Makevars.in` via `@GAPS_CXX_FLAGS@`; after editing + `configure.ac`, `-fopenmp` stops flowing in automatically. + +### 5.3. R export of OpenMP status — see §5.4 (needs a decision) +- `src/Cogaps.cpp::compiledWithOpenMPSupport_cpp()` (lines 233–240) and the + `#ifdef __GAPS_OPENMP__` inside it. +- `src/RcppExports.cpp` — the `_CoGAPS_compiledWithOpenMPSupport_cpp` entry + (auto-generated, regenerated). +- `R/CoGAPS.R:34–37` — the exported wrapper `compiledWithOpenMPSupport()`. +- `NAMESPACE:13` — `export(compiledWithOpenMPSupport)`. +- `R/CoGAPS.R:105–112` — the `if (!compiledWithOpenMPSupport()) { ... }` block — + **remove** in any case (replaced by the deprecation warning from §4.8). + +> Note: `std::thread` / `std::async` / `std::mutex` / `std::atomic` are absent from +> the code — all parallelism was solely on OpenMP pragmas. + +### 5.4. Public `compiledWithOpenMPSupport()` — DECIDED: B1 (stub) +`compiledWithOpenMPSupport()` is an **exported public function** (in `NAMESPACE`, +with an example in the docs). **Decision — B1:** keep the R wrapper so as not to +break the public API, but have it return `FALSE` (there is no OpenMP anymore — an +honest answer): +```r +#' @return FALSE (OpenMP support removed; CoGAPS runs single-threaded) +compiledWithOpenMPSupport <- function() FALSE +``` +- Remove the C++ side: `compiledWithOpenMPSupport_cpp()` in `Cogaps.cpp` and the + `_CoGAPS_compiledWithOpenMPSupport_cpp` entry in `RcppExports.cpp` (regenerated) and + `R/RcppExports.R:20–22`. +- `NAMESPACE:13` `export(compiledWithOpenMPSupport)` — **keep**. +- Update the roxygen `@return` (now always `FALSE`). + +--- + +## 6. The `getAverageQueueLength` method + +`SingleThreadedGibbsSampler::getAverageQueueLength()` (`SingleThreadedGibbsSampler.h:92–96`) +returns `0.f` — it is a stub for an interface needed only for async diagnostics. +After removing the `averageQueueLengthA/P` fields (§4.5) and their calls in +GapsRunner (§4.1), the method can be removed entirely. + +--- + +## 7. Classes removed entirely (reference) + +| Class | Where defined | Who used it (all goes away) | +|-------|---------------|-----------------------------| +| `AsynchronousGibbsSampler` | `AsynchronousGibbsSampler.h` | `GapsRunner.cpp` (commented), `testSamplerHighLevel.cpp` | +| `ProposalQueue` + `struct AtomicProposal` | `ProposalQueue.{h,cpp}` | `AsynchronousGibbsSampler.h`, `ConcurrentAtomicDomain.h` (friend), `testSerialization.cpp` | +| `ConcurrentAtomicDomain` | `ConcurrentAtomicDomain.{h,cpp}` | `AsynchronousGibbsSampler.h`, `ProposalQueue.{h,cpp}`, `testConcurrentAtomicDomain.cpp` | +| `ConcurrentAtom` (+ neighborhood) | `ConcurrentAtom.{h,cpp}` | `ConcurrentAtomicDomain.{h,cpp}`, `ProposalQueue.h`, `AsynchronousGibbsSampler.h` (debug) | + +--- + +## 8. Recommended order of execution + +1. Delete the files from §3. +2. Edit `GapsRunner.cpp`, `GapsParameters.{h,cpp}`, `GapsResult.h`, + `Cogaps.cpp` (§4.1–4.5). +3. Strip the parallelism and `nThreads` parameters (§5, §6). +4. Update `Makevars` / `Makevars.win` / `Makevars.in` (§4.6). +5. Update the C++ tests (§4.7). +6. Update the R layer (including `DistributedCogaps.R`, §4.8–4.9) and regenerate + `RcppExports.cpp` (§4.10). +7. Rebuild the package and run the C++ tests (`cpp_tests`) and the R tests. + +## 9. Acceptance criteria + +- [ ] The project builds without errors or warnings about unresolved symbols + (`ConcurrentAtom*`, `ProposalQueue`, `AsynchronousGibbsSampler`). +- [ ] `grep -rn "Asynchronous\|ProposalQueue\|Concurrent" src/` returns no matches + (except possibly comments in history). +- [ ] All C++ tests pass; the removed async tests are absent from the build. +- [ ] The R functions `CoGAPS/scCoGAPS/GWCoGAPS` work; the `asynchronousUpdates`/ + `nThreads` parameters are either removed or marked deprecated + (per the decision in §4.8). +- [ ] **Parity passed** per the §11 protocol — results match the baseline captured + before removal bit-for-bit (async was already disabled, and the pragmas being + removed are FP-neutral, so exact equality is expected, not "within tolerance"). + +## 10. Open questions + +1. ~~**R compatibility:**~~ **DECIDED** — deprecated stubs with a `warning` only for + a non-default value (`nThreads != 1` or `asynchronousUpdates=TRUE`), see §4.8. +2. ~~**OpenMP:**~~ **DECIDED** — variant **B** (complete OpenMP removal, §5) + + sub-variant **B1** for the public `compiledWithOpenMPSupport()` (stub → `FALSE`, + §5.4). OpenMP infrastructure is **not needed** for GWCoGAPS/scCoGAPS — their + parallelism is at the R process level (`BiocParallel`), not C++ threads; inside + workers single-threading is even desirable (otherwise N×T = oversubscription). +3. ~~**Result format:**~~ **DECIDED** — `averageQueueLengthA/P` are removed from + `GapsResult` (async diagnostics, downstream does not read them), see §4.5. + +**All open questions are closed.** Additionally recorded: +- Branch: work continues in `132-uncertainty-improvements` (not a separate branch). +- `GapsParameters` struct is left unchanged: `maxThreads=1`, `asynchronousUpdates=false`. +- Verification: mandatory parity protocol, see §11. + +--- + +## 11. Parity-check protocol (mandatory) + +Goal — prove that the cleanup did not touch the sequential path (insurance against +interacting bugs). Async is already disabled, and the pragmas being removed are +FP-neutral (`omp parallel for` with 1 thread = the same loop order; `omp atomic` +does not change values) — so **exact bit-for-bit equality is expected**. + +### 11.1. Capture the baseline BEFORE the edits +On the current `HEAD` (before any changes) build the package and run a matrix of +configurations with a fixed seed, saving the results to RDS: +```r +library(CoGAPS) +data(GIST) # GIST.matrix +cfg <- function(tag, ...) { + r <- CoGAPS(GIST.matrix, seed=42, nIterations=1000, messages=FALSE, ...) + saveRDS(list(fl=r@featureLoadings, sf=r@sampleFactors, + chi=r@metadata$meanChiSq), sprintf("parity_before_%s.rds", tag)) +} +cfg("dense") # DenseNormalModel +cfg("sparse", sparseOptimization=TRUE) # SparseNormalModel +cfg("unc", uncertainty=GIST.uncertainty) # uncertainty-matrix path +# distributed (DistributedCogaps.R and the R arguments are affected): +rsc <- scCoGAPS(GIST.matrix, seed=42, nIterations=1000, messages=FALSE, + nSets=2, BPPARAM=BiocParallel::SerialParam()) +saveRDS(list(fl=rsc@featureLoadings, sf=rsc@sampleFactors), "parity_before_scc.rds") +``` +> `SerialParam()` for the distributed run — so the result is deterministic and does +> not depend on the number of workers/scheduler. + +### 11.2. Capture the same AFTER the edits +Rebuild the package after all changes, run the same calls, save to +`parity_after_*.rds`. + +### 11.3. Compare +```r +for (tag in c("dense","sparse","unc","scc")) { + a <- readRDS(sprintf("parity_before_%s.rds", tag)) + b <- readRDS(sprintf("parity_after_%s.rds", tag)) + stopifnot(identical(a$fl, b$fl), identical(a$sf, b$sf)) +} +``` +Criterion — `identical()` (exact match). Any discrepancy = the cleanup touched the +numerical path → investigate before committing to the PR. + +### 11.4. Plus standard checks +- `cpp_tests` (Catch) — all pass. +- `R CMD check` / `devtools::test()` — accounting for the edits to + `test_seed_consistency.R`, `test_top_level.R`, `inst/scripts/debugRuns.R` + (§4.7-related). +- Check both build paths where things diverged before (see issues 9–10): + `devtools::install_local()` (release, `-O2`), not only `load_all()`. diff --git a/dev-notes/rus/agent-rules-rus.md b/dev-notes/rus/agent-rules-rus.md new file mode 100644 index 00000000..2fc4640e --- /dev/null +++ b/dev-notes/rus/agent-rules-rus.md @@ -0,0 +1,22 @@ +# Правила совместной работы + +## Git + +- **Коммиты только по явной команде** («делай коммит»). Не коммитить по своей инициативе. +- Комментарии к коммитам — на английском, кратко. Длинное описание — только если попросят. +- **Не переписывать историю** (`rebase -i`, `force push`) после начала ревью на Bioconductor. +- **Разрешение на коммит не переносится на следующий шаг** + +## Стиль кода + +- lint +- Без лишних комментариев — только там, где нужно пояснение. +- Все комментарии на английском + +## Общение + +- Не делать ничего «на всякий случай» без спроса. +- Если что-то неясно — спросить, а не угадывать. +- Мы разговариваем по-русски. Все документы для внешнего читатателя, например, комментарии, README, коммиты - по-английски. +- Я называю тебя "модель, ты". Называй меня тоже ты - так проще. Ты вербализуешься как сущность женского рода, я мужского. + diff --git a/dev-notes/rus/plan-rus.md b/dev-notes/rus/plan-rus.md new file mode 100644 index 00000000..3f954d44 --- /dev/null +++ b/dev-notes/rus/plan-rus.md @@ -0,0 +1,68 @@ +# План: что делать дальше + +Список задач, поставленных на 11 августа 2026, ветка `132-uncertainty-improvements`. +Ни одна ещё не начата. Формулировки задач — от мейнтейнера; строки «где смотреть» — +это зацепки для начала раскопок, а не диагноз. + +## 1. Багфикс: mean chi-square вдвое больше любого из значений + +GitHub issue: https://github.com/FertigLab/CoGAPS/issues/159 + +Отчётный средний chi-square выходит примерно в два раза больше, чем любое из тех +значений, по которым он считается. То есть это не среднее. + +Где смотреть: + +- `src/GapsStatistics.cpp:63` и `:88` — `meanChiSq()` для плотной и разреженной + моделей. Обе функции возвращают накопленную сумму `chisq` по всем `i,j` + (`return chisq;`), без деления на что-либо, — стоит начать с того, чтобы + установить, что вообще считается «средним» и по какому множеству. +- `src/GapsStatistics.cpp:134,145` — `addChiSq()`/`chisqHistory()`, история по + итерациям; отдельный от предыдущего путь. +- `R/DistributedCogaps.R:275` — для распределённых прогонов итоговое значение + собирается как `sum(sapply(result, ...))`, то есть сумма по подмножествам. + Если симптом воспроизводится именно на распределённом прогоне, смотреть надо + сюда. +- `tests/testthat/test_chisq.R` — там уже есть пересчёт `getMeanChiSq()` вручную + из `featureLoadings`/`sampleFactors`; тест на согласованность стоит расширить + так, чтобы он ловил и этот случай. + +## 2. Почему BiocParallel не берёт параметр «размер пачки» + +Разобраться, почему до BiocParallel не доходит размер пачки (`tasks` в +`MulticoreParam`/`SnowParam`), и можно ли им управлять. + +Где смотреть: + +- `R/DistributedCogaps.R:55` — если `BPPARAM` не задан, он создаётся как + `MulticoreParam(workers = length(sets))`, без `tasks`. +- `R/DistributedCogaps.R:62,91` — два вызова `bplapply` (черновая и финальная + стадии), оба получают только `BPPARAM`. +- `R/CoGAPS.R:96,131` — `BPPARAM` протаскивается через `allParams`; отдельного + параметра для размера пачки в `CogapsParams` нет. + +Вопрос, на который нужен ответ: пользователь ведь может передать свой `BPPARAM` +с нужным `tasks` — почему этого не хватает и где значение теряется. + +## 3. Единообразие параметров uncertainty между способами хранения данных + +Сделать так, чтобы параметры модели uncertainty были устроены как можно более +одинаково для разных модальностей хранения данных (плотная, разреженная, +чтение из файла). + +Где смотреть: + +- `../uncertainty-model-eng.md` — описание модели целиком; раздел 8 («Known + duplication») уже фиксирует конкретный шаг: константа `factor = 0.1` живёт в + двух местах (`pmax(D, factor, factor)` в плотной модели и `mBeta = 100` плюс + порог `max(d,1)` в `invSSq` в разреженной), согласованность держится только на + комментариях и тесте `[sparsegibbs]`. Предлагаемый там рефакторинг — один общий + `constexpr float UNCERTAINTY_FACTOR = 0.1f`, из которого выводится и порог + `pmax`, и `mBeta`/`invSSq`. +- `src/gibbs_sampler/DenseNormalModel.h:110,114` — построение `S` и + пользовательская `setUncertainty()`. +- `R/HelperFunctions.R:223` — `uncertainty=` вместе с `sparseOptimization=TRUE` + отвергается; сюда же относится вопрос, должно ли так остаться. + +Именно расхождение этих двух путей породило issues 17, 19 и 20 — см. +`../132-LLM-assisted-solved-issues.md`. diff --git a/dev-notes/simd-issue.md b/dev-notes/simd-issue.md new file mode 100644 index 00000000..7122a66e --- /dev/null +++ b/dev-notes/simd-issue.md @@ -0,0 +1,114 @@ +# Bug: SIMD padding zeros cause NaN in alphaParameters, silencing all sampleBirth() calls on Mac Intel + +## Affected platforms + +Fails on **Mac Intel** (Apple Clang with SSE4.1 or AVX enabled by default). +Passes on Apple M1/M2/M3/M4 (ARM, scalar fallback), MinGW32 (SIMD explicitly +disabled), and Intel/Linux (R toolchain typically does not define `__SSE4_1__`, +so the scalar fallback is used there too). + +Failing test: `CoGAPS:::run_catch_unit_tests_by_tag("[densesinglesampler]")` + +## Root cause + +`DenseNormalModel::alphaParameters()` (and the two overloads +`alphaParameters(r1,c1,r2,c2)` and `alphaParametersWithChange()`) runs a SIMD +loop bounded by `mDMatrix.nRow()`: + +```cpp +unsigned size = mDMatrix.nRow(); // e.g. 25 +// ... +for (gaps::simd::Index i(0); i < size; ++i) // increments by SIMD_INC (4 or 8) +{ + pMat.load(mat + i); + pS.load(S + i); + gaps::simd::PackedFloat ratio(pMat / (pS * pS)); + partialS += pMat * ratio; + partialS_mu += ratio * (pD - pAP); +} +``` + +Because `gaps::simd::Index` increments by `SIMD_INC` (4 for SSE4, 8 for AVX), +the last iteration reads up to `SIMD_INC - 1` elements **beyond** the last +real element, i.e. into the SIMD-alignment padding that `Vector` allocates via +`SIMD_PAD(n)`. + +`mSMatrix` is the uncertainty matrix. It is computed in the +`DenseNormalModel` constructor as: + +```cpp +mSMatrix = gaps::pmax(mDMatrix, factor, mLambda); +``` + +`gaps::pmax` (both the `Vector` and `Matrix` overloads) filled only the +`mSize` real elements and left the SIMD padding positions at **0** (the value +set by the `Vector` constructor). + +Therefore, when the SIMD loop reads a padding position: + +``` +pMat[k] = 0 (padding of the other matrix — also zero) +pS[k] = 0 (padding of mSMatrix — zero, not filled by pmax) + +ratio[k] = pMat[k] / (pS[k] * pS[k]) = 0 / 0 = NaN +partialS[k] += pMat[k] * ratio[k] = 0 * NaN = NaN +``` + +NaN propagates through `PackedFloat::scalar()` (which sums all SIMD lanes) +and poisons the returned `AlphaParameters`. + +A poisoned `AlphaParameters` causes `gibbsMass()` to return an empty +`OptionalFloat`, so every `sampleBirth()` call is rejected. As a consequence, +**the P matrix (PSampler) never accumulates any atoms**: `MyMatrix` stays +all-zero, the AP product stays all-zero, and chi-square never decreases. + +### Why Mac Intel specifically + +On Mac Intel, Apple Clang defines `__SSE4_1__` (or `__AVX__`) by default, +activating the SIMD code path (`SIMD_INC = 4` or `8`). On the other +platforms: + +| Platform | SIMD path | `SIMD_INC` | Bug triggered | +|---|---|---|---| +| Apple M1–M4 (ARM) | no SSE/AVX → scalar | 1 | No | +| MinGW32 | explicitly disabled | 1 | No | +| Intel/Linux (R GCC toolchain) | no `__SSE4_1__` → scalar | 1 | No | +| **Mac Intel (Apple Clang)** | SSE4.1 or AVX | **4 or 8** | **Yes** | + +With `SIMD_INC = 1` the loop iterates exactly `size` times and never touches +padding memory. + +### Historical note + +The original code used `mSMatrix.pad(1.f)`, which set **all** allocated +elements — including the SIMD padding — to `1.0f`. This was safe because +padding lanes gave `ratio = 0 / 1 = 0`. When `pad(1.f)` was replaced by +`gaps::pmax(...)` to compute data-driven uncertainty, the SIMD-safety property +was inadvertently lost. + +## Fix + +Added `padSIMD(float val)` methods to `Vector` and `Matrix` that fill only +the padding positions (`mSize .. mData.size() - 1`) with the given value. +Both `gaps::pmax` overloads now call `padSIMD(min_threshold)` before +returning, so that every SIMD padding element is set to `mLambda > 0`. + +With the fix, a padding lane contributes: + +``` +ratio[k] = 0 / mLambda^2 = 0 +partialS[k] += 0 * 0 = 0 +``` + +No NaN, no contribution to the sum — correct behaviour on all platforms. + +### Changed files + +| File | Change | +|---|---| +| `src/data_structures/Vector.h` | declare `void padSIMD(float val)` | +| `src/data_structures/Vector.cpp` | implement `padSIMD`: fills indices `[mSize, mData.size())` | +| `src/data_structures/Matrix.h` | declare `void padSIMD(float val)` | +| `src/data_structures/Matrix.cpp` | implement `padSIMD`: calls `padSIMD` on each column | +| `src/math/VectorMath.cpp` | `gaps::pmax(Vector,…)` calls `res.padSIMD(min_thr)` | +| `src/math/MatrixMath.cpp` | `gaps::pmax(Matrix,…)` calls `rmat.padSIMD(min_threshold)` | diff --git a/dev-notes/static-cast-uint64-reproducer/README.md b/dev-notes/static-cast-uint64-reproducer/README.md new file mode 100644 index 00000000..aa385192 --- /dev/null +++ b/dev-notes/static-cast-uint64-reproducer/README.md @@ -0,0 +1,58 @@ +# `static_cast` of a near-max `double` — standalone reproducer + +This directory holds the reproducer written while investigating issue 8 +(`132-manually-fixed-issues.md` §8). It is **not** part of the package: it has its +own `main()`, is not listed in `configure.ac`, and is never compiled by the build. +It used to sit in `src/cpp_tests/`, where it looked like a unit test and shipped +inside the package tarball; it lives here instead so that `src/` contains only +code that is actually built. + +## What it demonstrates + +`SingleThreadedGibbsSampler` used to keep the atomic domain length as a `double` +and convert it back with `static_cast`. On old Apple Clang (x86-64), +casting a `double` whose value is very close to `UINT64_MAX` does not yield the +neighbouring integer — it overflows to `0`. The atomic domain length was then +silently wrong. + +The program prints `UINT64_MAX` and `UINT64_MAX - 10`, converts both to `double` +and back, and shows the round-trip result. On an affected toolchain the values +come back as `0` instead of the expected magnitude; note that `UINT64_MAX - 10` +is not exactly representable as a `double` in the first place, so the round trip +cannot be lossless even where it does not overflow. + +## How to run it + +It is deliberately self-contained — no R, no package headers: + +``` +c++ static_cast_standalone_test.cpp -std=gnu++17 -O0 +./a.out +``` + +On an unaffected toolchain (checked on Apple arm64) both values round-trip to +`18446744073709551615`, i.e. `UINT64_MAX - 10` rounds *up* to `UINT64_MAX`: + +``` +uint64_t value 1 (max) : 18446744073709551615 +uint64_t value 2 (max-10) : 18446744073709551605 +value 1 as double : 1.84467e+19 +value 2 as double : 1.84467e+19 +value 1 as uint64_t casted back from double : 18446744073709551615 +value 2 as uint64_t casted back from double : 18446744073709551615 +``` + +That is the *benign* outcome — lossy, but the magnitude survives. The bug is the +last two lines coming back as `0`. So running this on arm64 or on Linux/GCC will +not reproduce it; an old Apple Clang on x86-64 is needed. + +## The fix in the package + +`AtomicDomain` gained a `uint64_t DomainLength() const` accessor that returns the +length as an integer directly, and the callers use `mDomain.DomainLength()` +instead of the `double` round trip. The remaining `double` field was renamed to +`mdDomainLength` so that the unsafe value is visibly distinct from the safe +accessor; it is still needed for the birth/death probability arithmetic, which is +genuinely floating point (`SingleThreadedGibbsSampler.h`). + +See `132-manually-fixed-issues.md` §8 for the full write-up and the commits. diff --git a/dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp b/dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp new file mode 100644 index 00000000..e6798280 --- /dev/null +++ b/dev-notes/static-cast-uint64-reproducer/static_cast_standalone_test.cpp @@ -0,0 +1,22 @@ +//compile: +//c++ static_cast_standalone_test.cpp -std=gnu++17 -O0 +#include +#include +#include + +int main(){ + uint64_t uvalue64_m=std::numeric_limits::max(); + uint64_t uvalue64_s=uvalue64_m-10; + //maximal is 18446744073709551615 + //effect holds for value 18446744073709551600 + //std::cout<<"Max uint64_t:\n"<::max()<(uvalue64_m); + double dvalue_s=static_cast(uvalue64_s); + std::cout<<"value 1 as double : "<(dvalue_m)<(dvalue_s)< **Historical note.** Before issue #17 the two models disagreed: the dense floor +> had drifted to `mLambda ≈ 0.006` (issue #2, wrong — `mLambda` is an atom-size +> scale, not an uncertainty), and the sparse model never floored small non-zeros +> at all (a defect inherited from `master`). Issues #17 (alphaParameters) and #19 +> (chiSq) put both models back on `S = max(0.1·D, 0.1)`. + +--- + +## 3. Parameter glossary + +These names appear throughout the two model classes. Only the first two are about +uncertainty; the rest are listed because they are easy to mistake for it. + +### `factor` = 0.1 — the uncertainty coefficient +The relative error **and** the absolute floor: `S = max(factor·D, factor)`. +Defined literally in `DenseNormalModel.h` (`float factor = 0.1f;`); in the sparse +model it is implicit (see `mBeta` below and the `max(d,1)` floor). + +### `mBeta` = 1/factor² = 100 — the sparse precision constant *(sparse only)* +`mBeta` is the **precision** (`1/S²`) of a floored entry, i.e. of any entry with +`S = factor`: + +``` +mBeta = 1 / factor^2 = 1 / 0.1^2 = 100 +``` + +The sparse model factors this constant out of every calculation and multiplies it +back in once at the end. That is *why* the sparse code can avoid ever materialising +an `S` matrix: for the majority of entries (zeros and `D < 1`) the weight `1/S²` +is the *same* number, `mBeta`, so it need not be stored per entry. Only the +non-zero, `D ≥ 1` entries deviate, and they are corrected individually +(section 6). The dense model has no `mBeta`: it divides by the stored `S` directly. + +### `invSSq(d)` = 1 / max(d,1)² — the per-entry weight, factor removed *(sparse only)* +A file-local helper in `SparseNormalModel.cpp`, the single point where the sparse +floor lives. It returns `1/S²` in units where `factor` has been pulled out into +`mBeta`. The full weight of an entry is therefore `mBeta · invSSq(d)`: + +``` +1/S^2 = 1 / (factor · max(d,1))^2 = (1/factor^2) · (1/max(d,1)^2) = mBeta · invSSq(d) +``` + +For `d ≥ 1`: `invSSq = 1/d²`. For `d < 1` (and `d = 0`): `invSSq = 1` (the floor, +so the full weight is just `mBeta`). + +### `mLambda` — **NOT uncertainty** (both models) +`mLambda = alpha · sqrt(nPatterns / meanD)` is the scale of the **sparsity / +atom-size prior** (the exponential prior on atom masses in the atomic domain). It +is used in `gibbsMass(..., mLambda)` to shift the proposal mean by `−mLambda`, and +to scale `mMaxGibbsMass`. It has nothing to do with `S`. Conflating it with the +uncertainty floor was exactly the issue #2 mistake. + +### `mAnnealingTemp` — **NOT uncertainty** (both models) +Equilibration temperature; multiplies the `AlphaParameters` (`s`, `s_mu`) during +annealing. Orthogonal to `S`. + +### `AlphaParameters { s, s_mu }` — the Gaussian conditional of one atom +When the sampler proposes changing a single matrix element by a mass `δ`, the +conditional posterior of `δ` is Gaussian, summarised by two numbers: + +- **`s`** — the **precision** (`1/variance`) of that Gaussian, +- **`s_mu`** — the precision-weighted mean (`s · mean`). + +`gibbsMass()` turns them into a truncated-normal draw with +`mean = s_mu / s` (or `(s_mu − lambda)/s` with the prior) and `sd = 1/sqrt(s)`. +Both are sums over the affected data column, weighted by `1/S²` — this is the one +place uncertainty enters the *sampling* (as opposed to the *reporting* `chiSq`): + +``` +s = Σ_i mat[i]^2 / S[i]^2 +s_mu = Σ_i mat[i] · (D[i] − AP[i]) / S[i]^2 +``` + +where `mat` is the relevant column of the *other* factor matrix (`P` when updating +`A`, and vice-versa). Derivation: substituting `AP → AP + δ·mat` into `chiSq` +gives a quadratic `chiSq(δ) = const − 2δ·s_mu + δ²·s`, whose coefficients are the +two sums above. + +--- + +## 4. The two representations at a glance + +| aspect | `DenseNormalModel` | `SparseNormalModel` | +|------------------------------|--------------------------------------------|------------------------------------------------------| +| stores `S`? | **yes** — full `mSMatrix` (nRow×nCol) | **no** — never materialised | +| how `1/S²` is obtained | divide by the stored `S` | `mBeta · invSSq(d)`, computed on the fly | +| `factor` lives in | `factor = 0.1f` → `pmax(D, factor, factor)`| `mBeta = 100` + `max(d,1)` floor in `invSSq` | +| data storage | dense `D`, dense `AP` (cached) | sparse `D`, `A·P` never cached | +| pre-computed helpers | `mAPMatrix` (cached A·P) | `mZ1`, `mZ2` lookup tables (rebuilt in `sync()`) | +| custom user uncertainty | **supported** (`setUncertainty`) | **rejected** — `checkInputs()` errors out | +| memory footprint | O(nRow·nCol) — fine for bulk data | O(non-zeros) — required for single-cell scale | + +The two are kept consistent only by matching math and by the `[sparsegibbs]` +cross-check test — *not* by shared code. Making them share one `factor` constant +is a possible future refactor (see section 8). + +--- + +## 5. Dense model — explicit `S`, direct division + +`DenseNormalModel` is the straightforward implementation. It builds the uncertainty +matrix once in the constructor and then just divides by it. + +**Construction** (`DenseNormalModel.h`, constructor): +```cpp +float factor = 0.1f; +mSMatrix = gaps::pmax(mDMatrix, factor, factor); // S = max(factor*D, factor) +``` +`pmax(M, mult, floor)` returns `max(mult*M[i,j], floor)`; here both are `factor`, +giving `max(0.1*D, 0.1)`. SIMD tail lanes are padded with `factor` (a positive +value) so later divisions never hit `0/0`. + +**chiSq** (`DenseNormalModel.cpp:55`): +```cpp +chisq += ((D[i,j] − AP[i,j]) / S[i,j])^2; // summed over all i,j +``` +Direct transcription of the model. Uses the cached `mAPMatrix`. + +**alphaParameters** (`DenseNormalModel.cpp:161`, SIMD): +```cpp +ratio = mat / (S * S); // = mat / S^2 +partialS += mat * ratio; // Σ mat^2 / S^2 -> s +partialS_mu += ratio * (D − AP); // Σ mat(D−AP) / S^2 -> s_mu +``` +- The **2-pattern** overload (`r1==r2`, columns `c1,c2`) replaces `mat` with + `mat1 − mat2` — used by exchange / two-atom moves. +- **`alphaParametersWithChange(row,col,ch)`** uses residual `D − (AP + ch·mat)`: + it evaluates the statistics *as if* the element had already been changed by + `ch`, without mutating `mAPMatrix`. + +**Custom uncertainty** (`setUncertainty`): the user may supply their own `S` +matrix, which replaces `mSMatrix` wholesale. Only the SIMD tail is then written, +with `padSIMD(1.f)`, so that the lanes read past the end divide by 1 rather than +by 0. + +The distinction matters more than it looks. Until issue #20 that line was +`pad(1.f)`, and `Vector::pad(val)` fills *every* allocated element, not just the +padding lanes — so a caller's uncertainty matrix was overwritten with 1.0f in its +entirety and `uncertainty=` had no effect at all on a dense run. Issue #17 had +removed the same `pad(1.f)` from the default path above (replacing it with +`pmax`), which fixed that half and left this one standing. **Never use `pad()` on +a matrix whose contents matter.** + +Custom uncertainty is the one capability the sparse model does not have: +`checkInputs()` rejects `uncertainty=` together with `sparseOptimization=TRUE`, +so the sparse path always uses the built-in model. + +--- + +## 6. Sparse model — baseline + correction, `S` never stored + +`SparseNormalModel` exists for data (single-cell) where an `nRow×nCol` dense matrix +does not fit in memory. It stores `D` sparsely and **never** forms `S`, `AP`, or +their dense products. Instead it uses the structure of the uncertainty model: + +> Every zero entry — and there are many — has the **same** uncertainty +> `S = factor`, hence the same weight `1/S² = mBeta`. So compute everything *as if +> all entries were zero* (a cheap dense-algebra baseline), then **correct** only +> the stored non-zero entries. + +### The lookup tables (`mZ1`, `mZ2`), rebuilt in `sync()` +Let `P` denote the *other* matrix (`mOtherMatrix`). Then: +``` +mZ1[k] = Σ_i P[i,k]^2 // squared column norms of P +mZ2[k,l] = Σ_i P[i,k] · P[i,l] // Gram matrix P^T P +``` +These are the "everything is a zero at `S = factor`" baselines: +- `mZ1[col]` is the baseline `s` (`Σ mat²`, weight 1, i.e. before ×mBeta). +- `mZ2` yields the baseline `s_mu` via `−dot(A_row, mZ2.col(col))`, because + `Σ_i mat[i]·AP[i] = Σ_k A[row,k]·mZ2[col,k]` (the zeros contribute `−mat·AP`, + since their `D = 0`). + +They are regenerated whenever the other matrix changes — that is what `sync()` +does. (This is why the sampler must interleave `A.update; P.sync(A); P.update; +A.sync(P)`: the sparse tables would otherwise be stale.) + +### The per-non-zero correction (`invSSq`) +Iterating the sparse non-zero structure, each stored `D` entry is switched from the +baseline weight `1` to its true floored weight `invSSq(D)`: + +**alphaParameters** (`SparseNormalModel.cpp:193`): +```cpp +float invS2 = invSSq(d_val); // = 1/max(d,1)^2 +s += v_val * v_val * (invS2 − 1.f); // fix Σ mat^2 weight: 1 -> invS2 +s_mu += v_val*d_val*invS2 + v_val*(1−invS2) * (A_row · P_row); // fix residual +... +return AlphaParameters(s, s_mu) * mBeta; // restore the factor +``` +The `(invS2 − 1)` / `(1 − invS2)` terms are precisely "remove the baseline +contribution, add the true one". For `d ≥ 1`, `invS2 = 1/d²`; for `d < 1`, +`invS2 = 1`, so the correction is zero — a floored entry already matches the +baseline weight (both are `S = factor`). Multiplying by `mBeta` at the very end +reinstates `factor`. + +**chiSq** (`SparseNormalModel.cpp:53`) has the same shape: +```cpp +// baseline: sum A*P^2 over ALL entries (as if every entry were a zero at S=factor) +for all i,j: chisq += (A·P)[i,j]^2; +// correction: for each stored non-zero D, turn A*P^2 into the floored residual +chisq += d*d*invS2 − 2*d*AP*invS2 + AP*AP*(invS2 − 1); // = (D−AP)^2*invS2 − AP^2 +return chisq * mBeta; +``` +Adding the correction to the baseline `AP²` yields `(D − AP)² · invSSq(D)` for each +non-zero, exactly the floored per-entry chi-square. For `d ≥ 1` this is +algebraically identical to the pre-#19 formula `1 + dot(dot − 2d − dsq·dot)/dsq`, +so integer/count data is bit-for-bit unchanged; only `0 < D < 1` entries move. + +**No-fit / pre-sync branch** (issue #18): before `sync()`, `mOtherMatrix` is +`NULL` and `A·P = 0`. The baseline loop is zero, so each stored `D` contributes +`(D−0)² · invSSq(D) = D²·invSSq(D)`, `×mBeta`. This matches the dense +`Σ D²/S²` and, for `D ≥ 1`, equals `mBeta` per entry (e.g. `100·nRow·nCol` on +fully-observed integer data). Guarding this branch is what fixed the pre-sync +segfault. + +### Why the sparse model cannot just "store an `S` matrix" +Two independent reasons, both fatal: +1. **Memory.** A dense `S` matrix is exactly the footprint the sparse model exists + to avoid (single-cell matrices are mostly zeros). +2. **Algebra.** The `mZ1`/`mZ2` baseline trick works *only* because the zero-entry + weight is a single constant that factors out as `mBeta`. A per-entry `S` on the + zeros would break the factorisation, and the "compute over all, correct the + non-zeros" shortcut collapses. + +So in the sparse model the uncertainty is *computed inline everywhere* (via +`invSSq` + `mBeta`), never stored. + +--- + +## 7. Equivalence and verification + +The two models produce the same numbers (up to float ordering) on the same +`A`, `P`, and `D`: + +- **Unit test** `[sparsegibbs]` (`testSparseGibbsSampler.cpp`): sets identical + `A`/`P` on a sparse and a dense sampler over continuous data that includes + `0 < D < 1`, and asserts `alphaParameters` (1D, 2D, symmetry, with-change) and + `chiSq` agree to `Approx(0.1 %)`. +- **End-to-end** on GIST: `A·P` reconstruction correlation dense-vs-sparse `= 0.999` + (equal to a dense-vs-dense different-seed control), and `meanChiSq` for a sparse + run lands within the dense-run range. + - Caveat: raw `featureLoadings` correlation is meaningless as a metric — NMF is + only identifiable up to a permutation of patterns, so even two dense runs + correlate at ≈ −0.1. Use `A·P` reconstruction or `meanChiSq`. + +That the formula above is the one actually used is checked from R as well, in +`tests/testthat/test_chisq.R`: the reported `getMeanChiSq()` is recomputed by hand +from `featureLoadings`/`sampleFactors`, once against the built-in `max(0.1·D, 0.1)` +(dense and sparse) and once against a matrix passed in `uncertainty=`. The second +case is what catches a dense run that ignores the caller's `S` — it failed before +issue #20 (442 vs 103450) and passes after. + +--- + +## 8. Known duplication (not yet refactored) + +The constant `factor = 0.1` currently lives in **two** places that must stay in +lock-step: + +- dense: `float factor = 0.1f;` (→ `pmax(D, factor, factor)`), and +- sparse: `mBeta = 100` (= `1/factor²`) plus the `max(d,1)` floor inside `invSSq`. + +Agreement is enforced only by comments and the `[sparsegibbs]` test — the same +fragility that produced issues #17/#19. A cheap, numerics-preserving refactor would +introduce one shared `constexpr float UNCERTAINTY_FACTOR = 0.1f` and derive both +`pmax`'s floor and `mBeta`/`invSSq` from it, so a future change to the model +propagates to both branches automatically. Deferred (does not intersect the +current Phase 3/4 work). + +--- + +## 9. Code map + +| what | file:line | +|---|---| +| uncertainty formula (dense build) | `src/gibbs_sampler/DenseNormalModel.h:96,110` | +| dense chiSq | `src/gibbs_sampler/DenseNormalModel.cpp:55` | +| dense alphaParameters (1D / 2D / with-change) | `src/gibbs_sampler/DenseNormalModel.cpp:161,185,216` | +| dense custom uncertainty | `src/gibbs_sampler/DenseNormalModel.h:114` (`setUncertainty`), `padSIMD(1.f)` at `:121` | +| SIMD tail of the default `S` | `src/math/MatrixMath.cpp:89` (`pmax` calls `padSIMD(min_threshold)`) | +| `uncertainty=` rejected for sparse | `R/HelperFunctions.R:223` | +| `invSSq` (single sparse floor) | `src/gibbs_sampler/SparseNormalModel.cpp:25` (doc block from :17) | +| `mBeta` init (= 100) | `src/gibbs_sampler/SparseNormalModel.h:77` | +| sparse chiSq (baseline + correction + no-fit) | `src/gibbs_sampler/SparseNormalModel.cpp:53` | +| sparse alphaParameters (1D / with-change / 2D) | `src/gibbs_sampler/SparseNormalModel.cpp:195,239,285` | +| `mZ1`/`mZ2` lookup tables | `src/gibbs_sampler/SparseNormalModel.cpp:339` (`generateLookupTables`) | +| `s`/`s_mu` → truncated-normal draw | `src/gibbs_sampler/AlphaParameters.cpp:27,38` (`gibbsMass`) | +| cross-check test | `src/cpp_tests/testSparseGibbsSampler.cpp` (`[sparsegibbs]`) | diff --git a/inst/scripts/debugRuns.R b/inst/scripts/debugRuns.R index 9f74f005..e7c11de6 100644 --- a/inst/scripts/debugRuns.R +++ b/inst/scripts/debugRuns.R @@ -16,10 +16,8 @@ CoGAPS(GISTPathGct, nIterations=1000) CoGAPS(GISTPathMtx, nIterations=1000) # all sampler types -CoGAPS(GIST.matrix, sparseOptimization=FALSE, asynchronousUpdates=FALSE, nIterations=1000) -CoGAPS(GIST.matrix, sparseOptimization=TRUE, asynchronousUpdates=FALSE, nIterations=1000) -CoGAPS(GIST.matrix, sparseOptimization=FALSE, asynchronousUpdates=TRUE, nIterations=1000) -CoGAPS(GIST.matrix, sparseOptimization=TRUE, asynchronousUpdates=TRUE, nIterations=1000) +CoGAPS(GIST.matrix, sparseOptimization=FALSE, nIterations=1000) +CoGAPS(GIST.matrix, sparseOptimization=TRUE, nIterations=1000) # all data types and file readers CoGAPS(GISTPathCsv, sparseOptimization=FALSE, nIterations=1000) @@ -27,10 +25,6 @@ CoGAPS(GISTPathCsv, sparseOptimization=TRUE, nIterations=1000) CoGAPS(GISTPathMtx, sparseOptimization=FALSE, nIterations=1000) CoGAPS(GISTPathMtx, sparseOptimization=TRUE, nIterations=1000) -# multiple threads for all data types -CoGAPS(GISTPathCsv, sparseOptimization=FALSE, nThreads=2, nIterations=1000) -CoGAPS(GISTPathCsv, sparseOptimization=TRUE, nThreads=2, nIterations=1000) - # distributed version params <- CogapsParams() params <- setDistributedParams(params, nSets=4) diff --git a/man/CoGAPS.Rd b/man/CoGAPS.Rd index 7a8d8555..e9426fef 100755 --- a/man/CoGAPS.Rd +++ b/man/CoGAPS.Rd @@ -31,7 +31,7 @@ CoGAPS( \item{nPatterns}{rank of the nmf decomposition} -\item{nThreads}{maximum number of threads to run on} +\item{nThreads}{deprecated and ignored; CoGAPS now always runs single-threaded} \item{messages}{T/F for displaying output} @@ -60,7 +60,8 @@ only worker 1 prints output and each worker outputs when it finishes, this is not neccesary when using the default parallel methods (i.e. distributed CoGAPS) but only when the user is manually calling CoGAPS in parallel} -\item{asynchronousUpdates}{enable asynchronous updating which allows for multi-threaded runs} +\item{asynchronousUpdates}{deprecated and ignored; the asynchronous sampler was +removed because it broke MCMC detailed balance} \item{nSnapshots}{how many snapshots to take in each phase, setting this to 0 disables snapshots} diff --git a/man/GWCoGAPS.Rd b/man/GWCoGAPS.Rd index e6a78fb6..03620fbc 100755 --- a/man/GWCoGAPS.Rd +++ b/man/GWCoGAPS.Rd @@ -6,7 +6,8 @@ \usage{ GWCoGAPS( data, - params = new("CogapsParams"), + params = new("CogapsParams", nPatterns = nPatterns), + nPatterns, nThreads = 1, messages = TRUE, outputFrequency = 500, @@ -26,7 +27,9 @@ GWCoGAPS( \item{params}{CogapsParams object} -\item{nThreads}{maximum number of threads to run on} +\item{nPatterns}{rank of the nmf decomposition} + +\item{nThreads}{deprecated and ignored; CoGAPS now always runs single-threaded} \item{messages}{T/F for displaying output} @@ -55,7 +58,8 @@ only worker 1 prints output and each worker outputs when it finishes, this is not neccesary when using the default parallel methods (i.e. distributed CoGAPS) but only when the user is manually calling CoGAPS in parallel} -\item{asynchronousUpdates}{enable asynchronous updating which allows for multi-threaded runs} +\item{asynchronousUpdates}{deprecated and ignored; the asynchronous sampler was +removed because it broke MCMC detailed balance} \item{...}{allows for overwriting parameters in params} } diff --git a/man/compiledWithOpenMPSupport.Rd b/man/compiledWithOpenMPSupport.Rd index f0bb05ac..6abe408c 100644 --- a/man/compiledWithOpenMPSupport.Rd +++ b/man/compiledWithOpenMPSupport.Rd @@ -7,7 +7,8 @@ compiledWithOpenMPSupport() } \value{ -true/false if OpenMP was supported +FALSE (OpenMP support was removed together with the asynchronous +sampler; CoGAPS now always runs single-threaded) } \description{ Check if compiler supported OpenMP diff --git a/man/getPatternGeneSet-methods.Rd b/man/getPatternGeneSet-methods.Rd index 9c94db40..f7bff2d7 100644 --- a/man/getPatternGeneSet-methods.Rd +++ b/man/getPatternGeneSet-methods.Rd @@ -25,9 +25,9 @@ getPatternGeneSet( \item{gene.sets}{a list of gene sets to test. List names should be the names of the gene sets} -\item{method}{enrichment or overrepresentation. Conducts a test for gene set enrichment using {fgsea::gsea} ranking features by pattern amplitude or a test for gene set overrepresentation in pattern markers using {fgsea::fora}, respectively.} +\item{method}{enrichment or overrepresentation. Conducts a test for gene set enrichment using \code{fgsea::gsea} ranking features by pattern amplitude or a test for gene set overrepresentation in pattern markers using \code{fgsea::fora}, respectively.} -\item{...}{additional parameters passed to {patternMarkers} if using overrepresentation method} +\item{...}{additional parameters passed to \code{patternMarkers} if using overrepresentation method} } \value{ list of dataframes containing gene set enrichment or gene set overrepresentation statistics diff --git a/man/sampleWithExplictSets.Rd b/man/sampleWithExplictSets.Rd index 1572fc71..03d9c0ca 100755 --- a/man/sampleWithExplictSets.Rd +++ b/man/sampleWithExplictSets.Rd @@ -9,7 +9,6 @@ sampleWithExplictSets(allParams) \arguments{ \item{allParams}{list of all CoGAPS parameters} -\item{total}{total number of rows (cols) that are being paritioned} } \value{ list of subsets diff --git a/man/scCoGAPS.Rd b/man/scCoGAPS.Rd index 5442a367..ec1e4b75 100755 --- a/man/scCoGAPS.Rd +++ b/man/scCoGAPS.Rd @@ -6,7 +6,8 @@ \usage{ scCoGAPS( data, - params = new("CogapsParams"), + params = new("CogapsParams", nPatterns = nPatterns), + nPatterns, nThreads = 1, messages = TRUE, outputFrequency = 500, @@ -26,7 +27,9 @@ scCoGAPS( \item{params}{CogapsParams object} -\item{nThreads}{maximum number of threads to run on} +\item{nPatterns}{rank of the nmf decomposition} + +\item{nThreads}{deprecated and ignored; CoGAPS now always runs single-threaded} \item{messages}{T/F for displaying output} @@ -55,7 +58,8 @@ only worker 1 prints output and each worker outputs when it finishes, this is not neccesary when using the default parallel methods (i.e. distributed CoGAPS) but only when the user is manually calling CoGAPS in parallel} -\item{asynchronousUpdates}{enable asynchronous updating which allows for multi-threaded runs} +\item{asynchronousUpdates}{deprecated and ignored; the asynchronous sampler was +removed because it broke MCMC detailed balance} \item{...}{allows for overwriting parameters in params} } diff --git a/src/Cogaps.cpp b/src/Cogaps.cpp index 24005687..5e16e7a7 100755 --- a/src/Cogaps.cpp +++ b/src/Cogaps.cpp @@ -82,7 +82,6 @@ GapsParameters getGapsParameters(const DataType &data, const Rcpp::List &allPara params.printThreadUsage = !params.runningDistributed; // get configuration parameters - params.maxThreads = Rcpp::as(allParams["nThreads"]); params.workerID = Rcpp::as(allParams["workerID"]); params.printMessages = Rcpp::as(allParams["messages"]) && (params.workerID == 1); params.outputFrequency = Rcpp::as(allParams["outputFrequency"]); @@ -99,7 +98,6 @@ GapsParameters getGapsParameters(const DataType &data, const Rcpp::List &allPara params.maxGibbsMassA = Rcpp::as(gapsParams.slot("maxGibbsMassA")); params.maxGibbsMassP = Rcpp::as(gapsParams.slot("maxGibbsMassP")); params.useSparseOptimization = Rcpp::as(gapsParams.slot("sparseOptimization")); - params.asynchronousUpdates = Rcpp::as(allParams["asynchronousUpdates"]); // calculate snapshot frequency int nSnapshots = Rcpp::as(allParams["nSnapshots"]); @@ -174,8 +172,6 @@ const DataType &uncertainty) Rcpp::Named("atomsP") = Rcpp::wrap(result.atomHistoryP), Rcpp::Named("pumpStat") = createRMatrix(result.pumpMatrix), Rcpp::Named("meanPatternAssignment") = createRMatrix(result.meanPatternAssignment), - Rcpp::Named("averageQueueLengthA") = result.averageQueueLengthA, - Rcpp::Named("averageQueueLengthP") = result.averageQueueLengthP, Rcpp::Named("totalUpdates") = result.totalUpdates, Rcpp::Named("totalRunningTime") = result.totalRunningTime, Rcpp::Named("equilibrationSnapshotsA") = createListOfRMatrices(result.equilibrationSnapshotsA), @@ -230,15 +226,6 @@ bool checkpointsEnabled_cpp() #endif } -// [[Rcpp::export]] -bool compiledWithOpenMPSupport_cpp() -{ -#ifdef __GAPS_OPENMP__ - return true; -#else - return false; -#endif -} // [[Rcpp::export]] Rcpp::List getFileInfo_cpp(const std::string &path) diff --git a/src/GapsParameters.cpp b/src/GapsParameters.cpp index 59f6dc3d..26f4893d 100755 --- a/src/GapsParameters.cpp +++ b/src/GapsParameters.cpp @@ -15,13 +15,11 @@ void GapsParameters::print() const gaps_printf("nIterations: %d\n", nIterations); gaps_printf("seed: %d\n", seed); gaps_printf("\n"); - gaps_printf("maxThreads: %d\n", maxThreads); gaps_printf("printMessages: %s\n", printMessages ? "TRUE" : "FALSE"); gaps_printf("outputFrequency: %d\n", outputFrequency); gaps_printf("snapshotFrequency: %d\n", snapshotFrequency); gaps_printf("\n"); gaps_printf("useSparseOptimization: %s\n", useSparseOptimization ? "TRUE" : "FALSE"); - gaps_printf("asynchronousUpdates: %s\n", asynchronousUpdates ? "TRUE" : "FALSE"); gaps_printf("takePumpSamples: %s\n", takePumpSamples ? "TRUE" : "FALSE"); gaps_printf("\n"); gaps_printf("runningDistributed: %s\n", runningDistributed ? "TRUE" : "FALSE"); diff --git a/src/GapsParameters.h b/src/GapsParameters.h index b8e05e1b..6ea4bb21 100755 --- a/src/GapsParameters.h +++ b/src/GapsParameters.h @@ -41,7 +41,6 @@ struct GapsParameters unsigned nSamples; unsigned nPatterns; unsigned nIterations; - unsigned maxThreads; unsigned outputFrequency; unsigned checkpointInterval; unsigned snapshotFrequency; @@ -60,7 +59,6 @@ struct GapsParameters bool printThreadUsage; bool useSparseOptimization; bool takePumpSamples; - bool asynchronousUpdates; char whichMatrixFixed; unsigned workerID; bool runningDistributed; @@ -86,7 +84,6 @@ nGenes(0), nSamples(0), nPatterns(3), nIterations(1000), -maxThreads(1), outputFrequency(500), checkpointInterval(250), snapshotFrequency(0), @@ -105,7 +102,6 @@ subsetGenes(t_subsetGenes), printThreadUsage(true), useSparseOptimization(false), takePumpSamples(false), -asynchronousUpdates(true), whichMatrixFixed('N'), workerID(1), runningDistributed(false) diff --git a/src/GapsResult.h b/src/GapsResult.h index dd9a3ee0..6a7e9642 100755 --- a/src/GapsResult.h +++ b/src/GapsResult.h @@ -31,8 +31,6 @@ struct GapsResult uint32_t seed; unsigned totalRunningTime; float meanChiSq; - float averageQueueLengthA; - float averageQueueLengthP; }; #endif // __COGAPS_GAPS_RESULT__ diff --git a/src/GapsRunner.cpp b/src/GapsRunner.cpp index 06c3fe30..801feedb 100755 --- a/src/GapsRunner.cpp +++ b/src/GapsRunner.cpp @@ -5,7 +5,6 @@ #include "math/Random.h" #include "utils/Archive.h" #include "utils/GlobalConfig.h" -#include "gibbs_sampler/AsynchronousGibbsSampler.h" #include "gibbs_sampler/SingleThreadedGibbsSampler.h" #include "gibbs_sampler/DenseNormalModel.h" #include "gibbs_sampler/SparseNormalModel.h" @@ -17,11 +16,6 @@ #pragma GCC diagnostic pop #endif -// library allowing for message passing in distributed mode -#ifdef __GAPS_OPENMP__ -#include -#endif - // boost time helpers #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" @@ -66,12 +60,6 @@ template static GapsResult chooseSampler(const DataType &data, GapsParameters ¶ms, const DataType &uncertainty, GapsRandomState *randState) { - if (params.asynchronousUpdates) - { - GAPS_MESSAGE(params.printMessages, "Sampler Type: Asynchronous\n"); - return runCoGAPSAlgorithm< AsynchronousGibbsSampler >(data, - params, uncertainty, randState); - } GAPS_MESSAGE(params.printMessages, "Sampler Type: Sequential\n"); return runCoGAPSAlgorithm< SingleThreadedGibbsSampler >(data, params, uncertainty, randState); @@ -204,19 +192,19 @@ Sampler &PSampler, unsigned nA, unsigned nP) { if (params.whichMatrixFixed != 'A') { - ASampler.update(nA, params.maxThreads); + ASampler.update(nA); if (params.whichMatrixFixed != 'P') { - PSampler.sync(ASampler, params.maxThreads); + PSampler.sync(ASampler); } } if (params.whichMatrixFixed != 'P') { - PSampler.update(nP, params.maxThreads); + PSampler.update(nP); if (params.whichMatrixFixed != 'A') { - ASampler.sync(PSampler, params.maxThreads); + ASampler.sync(PSampler); } } } @@ -349,19 +337,6 @@ Sampler &PSampler) } } -static void calculateNumberOfThreads(GapsParameters params) -{ - // calculate appropiate number of threads if compiled with openmp - #ifdef __GAPS_OPENMP__ - if (params.printMessages && params.printThreadUsage) - { - unsigned availableThreads = omp_get_max_threads(); - params.maxThreads = gaps::min(availableThreads, params.maxThreads); - gaps_printf("Running on %d out of %d available threads\n", - params.maxThreads, availableThreads); - } - #endif -} template static void processUncertainty(const GapsParameters params, Sampler &ASampler, @@ -438,7 +413,6 @@ const DataType &uncertainty, GapsRandomState *randState) GapsAlgorithmPhase phase(GAPS_EQUILIBRATION_PHASE); unsigned currentIter = 0; processCheckpoint(params, ASampler, PSampler, randState, stats, rng, phase, currentIter); - calculateNumberOfThreads(params); // sync samplers and run any additional initialization needed ASampler.sync(PSampler); @@ -471,8 +445,6 @@ const DataType &uncertainty, GapsRandomState *randState) // get result GapsResult result(stats); result.totalRunningTime = static_cast((bpt_now() - startTime).total_seconds()); - result.averageQueueLengthA = ASampler.getAverageQueueLength(); - result.averageQueueLengthP = PSampler.getAverageQueueLength(); result.totalUpdates = totalUpdates; // do not return meanChisQ if running with a fixed matrix to avoid confusion diff --git a/src/Makevars.win b/src/Makevars.win index f0d0e726..c510709d 100755 --- a/src/Makevars.win +++ b/src/Makevars.win @@ -10,10 +10,7 @@ OBJECTS = Cogaps.o \ RcppExports.o \ test-runner.o \ atomic/Atom.o \ - atomic/ConcurrentAtom.o \ atomic/AtomicDomain.o \ - atomic/ConcurrentAtomicDomain.o \ - atomic/ProposalQueue.o \ data_structures/HashSets.o \ data_structures/HybridMatrix.o \ data_structures/HybridVector.o \ diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000..d81ca0af --- /dev/null +++ b/src/README.md @@ -0,0 +1,53 @@ +# Some notes about the structures CoGAPS uses + +## A(P)Sampler +In the GapsRunner, all the sampling events are done by two samplers, one for a decomposition matrix, +$D=AP$ ASampler and PSampler. The type of the sampler objects depends on the data model it uses (Dense or Sparse). +The data is organised in the sampler as folows. + +Let's say that there are $l$ rows (usually, genes) in $D$ matrix, $m$ (samples) columns in $D$ and the decomposition runs with $k$ patterns. So, $D$ and $AP$ are both of $m \times l$ size. + +Each sampler carries its own copy of $AP$, they are copied by the `sync()` method and they are calculated by the `extraInitialization()`. Each sampler has it purpose matrix, and the uncertanoty matrix. The latter is og the same size as the AP. Each sampler has a `const sampler &` reference to its *vis-a-vis*. + +A sampler can be transposed or not. The run involves a transposed and a nontransposes sampler. If the input D matrix is not transposed by the call, the left (A) sampler is transposed, the right (P) is not. + +A transposed (left, A, if the D is non-transposed) carries a transposed AP matrix of $l \times m$ and the puspose (A) matrix of $m \times k$ size. + +A non-transposed (right, P, if the D is non-transposed) carries a nontransposed AP matrix of $m \times l$ and the puspose (P) matrix of $l \times k$ size. + +So, for any correct sampler, nrows(MyMatrix)==ncols(APMatrix); ncols(MyMatrix) is the pattern munber, and APMatrix has the same dimesions as transposed other\_sampler.APMatrix. After `sync()` APMatrix==tr(other\_sampler.APMatrix). + +## Debug, etc congigure options +All the options for config are given in ../configure.ac + +After changing, regenerate `configure`. Plain `autoconf` is **not** enough: +`configure.ac` uses `AX_COMPILER_VENDOR` and `AX_COMPILER_VERSION` from +autoconf-archive, and those are pulled in by `aclocal`, not by `autoconf`. Run +both, from the package root: + +``` +aclocal -I /opt/homebrew/share/aclocal # path to the autoconf-archive macros +autoconf +``` + +If you skip `aclocal`, the two `AX_*` macros are left unexpanded and end up in +`configure` as literal shell commands. It still "works" — you just get + +``` +./configure: line 2940: AX_COMPILER_VENDOR: command not found +building on compiler version +``` + +and `$ax_cv_cxx_compiler_vendor` stays empty, which silently disables the +vendor-dependent branch: `--enable-warnings` then adds no flags at all. The +`configure` on `master` has exactly this problem; the one on this branch was +regenerated correctly and does not. + +`aclocal.m4` is a build artefact of that procedure and is not committed. + +To pass and option to configure when running devtools::load_all or similar, set the environment varianle, for example, to run ./configure --enable-debug, run the following in R: + +Sys.setenv(enable_debug="yes") +devtools::load_all(recompile = TRUE) + +We suppose we are in the root of the package. \ No newline at end of file diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 354fb9f1..3b0ff2bb 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -56,16 +56,6 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } -// compiledWithOpenMPSupport_cpp -bool compiledWithOpenMPSupport_cpp(); -RcppExport SEXP _CoGAPS_compiledWithOpenMPSupport_cpp() { -BEGIN_RCPP - Rcpp::RObject rcpp_result_gen; - Rcpp::RNGScope rcpp_rngScope_gen; - rcpp_result_gen = Rcpp::wrap(compiledWithOpenMPSupport_cpp()); - return rcpp_result_gen; -END_RCPP -} // getFileInfo_cpp Rcpp::List getFileInfo_cpp(const std::string& path); RcppExport SEXP _CoGAPS_getFileInfo_cpp(SEXP pathSEXP) { @@ -78,25 +68,37 @@ BEGIN_RCPP END_RCPP } // run_catch_unit_tests -int run_catch_unit_tests(Rcpp::String reporter); -RcppExport SEXP _CoGAPS_run_catch_unit_tests(SEXP reporterSEXP) { +int run_catch_unit_tests(Rcpp::String reporter, Rcpp::String output); +RcppExport SEXP _CoGAPS_run_catch_unit_tests(SEXP reporterSEXP, SEXP outputSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< Rcpp::String >::type reporter(reporterSEXP); - rcpp_result_gen = Rcpp::wrap(run_catch_unit_tests(reporter)); + Rcpp::traits::input_parameter< Rcpp::String >::type output(outputSEXP); + rcpp_result_gen = Rcpp::wrap(run_catch_unit_tests(reporter, output)); return rcpp_result_gen; END_RCPP } // run_catch_unit_tests_by_tag -int run_catch_unit_tests_by_tag(Rcpp::String tag, Rcpp::String reporter); -RcppExport SEXP _CoGAPS_run_catch_unit_tests_by_tag(SEXP tagSEXP, SEXP reporterSEXP) { +int run_catch_unit_tests_by_tag(Rcpp::String tag, Rcpp::String reporter, Rcpp::String output); +RcppExport SEXP _CoGAPS_run_catch_unit_tests_by_tag(SEXP tagSEXP, SEXP reporterSEXP, SEXP outputSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< Rcpp::String >::type tag(tagSEXP); Rcpp::traits::input_parameter< Rcpp::String >::type reporter(reporterSEXP); - rcpp_result_gen = Rcpp::wrap(run_catch_unit_tests_by_tag(tag, reporter)); + Rcpp::traits::input_parameter< Rcpp::String >::type output(outputSEXP); + rcpp_result_gen = Rcpp::wrap(run_catch_unit_tests_by_tag(tag, reporter, output)); + return rcpp_result_gen; +END_RCPP +} +// catch_test_case_names +Rcpp::CharacterVector catch_test_case_names(); +RcppExport SEXP _CoGAPS_catch_test_case_names() { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + rcpp_result_gen = Rcpp::wrap(catch_test_case_names()); return rcpp_result_gen; END_RCPP } @@ -106,10 +108,10 @@ static const R_CallMethodDef CallEntries[] = { {"_CoGAPS_cogaps_cpp", (DL_FUNC) &_CoGAPS_cogaps_cpp, 3}, {"_CoGAPS_getBuildReport_cpp", (DL_FUNC) &_CoGAPS_getBuildReport_cpp, 0}, {"_CoGAPS_checkpointsEnabled_cpp", (DL_FUNC) &_CoGAPS_checkpointsEnabled_cpp, 0}, - {"_CoGAPS_compiledWithOpenMPSupport_cpp", (DL_FUNC) &_CoGAPS_compiledWithOpenMPSupport_cpp, 0}, {"_CoGAPS_getFileInfo_cpp", (DL_FUNC) &_CoGAPS_getFileInfo_cpp, 1}, - {"_CoGAPS_run_catch_unit_tests", (DL_FUNC) &_CoGAPS_run_catch_unit_tests, 1}, - {"_CoGAPS_run_catch_unit_tests_by_tag", (DL_FUNC) &_CoGAPS_run_catch_unit_tests_by_tag, 2}, + {"_CoGAPS_run_catch_unit_tests", (DL_FUNC) &_CoGAPS_run_catch_unit_tests, 2}, + {"_CoGAPS_run_catch_unit_tests_by_tag", (DL_FUNC) &_CoGAPS_run_catch_unit_tests_by_tag, 3}, + {"_CoGAPS_catch_test_case_names", (DL_FUNC) &_CoGAPS_catch_test_case_names, 0}, {NULL, NULL, 0} }; diff --git a/src/atomic/Atom.cpp b/src/atomic/Atom.cpp index b7101875..5edad376 100644 --- a/src/atomic/Atom.cpp +++ b/src/atomic/Atom.cpp @@ -13,7 +13,7 @@ AtomNeighborhood::AtomNeighborhood(Atom *l, Atom *c, Atom *r) bool AtomNeighborhood::hasLeft() const { - return left != NULL; + return left != NULL; } bool AtomNeighborhood::hasRight() const @@ -21,8 +21,7 @@ bool AtomNeighborhood::hasRight() const return right != NULL; } -Atom::Atom(uint64_t p, float m) - : mIterator(), mPos(p), mLeftIndex(-1), mRightIndex(-1), mIndex(-1), mMass(m) +Atom::Atom(uint64_t p, float m): mIterator(), mPos(p), mHasRight(false), mHasLeft(false), mMass(m) {} uint64_t Atom::pos() const @@ -45,17 +44,29 @@ void Atom::updatePos(uint64_t newPos) mPos = newPos; } -void Atom::setLeftIndex(int index) +void Atom::setLeftIndex(size_t index) { mLeftIndex = index; + mHasLeft = true; } -void Atom::setRightIndex(int index) +void Atom::setRightIndex(size_t index) { mRightIndex = index; + mHasRight = true; } -void Atom::setIndex(int index) +void Atom::unsetLeftIndex(){ + mLeftIndex = (size_t)0; + mHasLeft = false; +} + +void Atom::unsetRightIndex(){ + mRightIndex = (size_t)0; + mHasRight = false; +} + +void Atom::setIndex(size_t index) { mIndex = index; } @@ -67,25 +78,27 @@ void Atom::setIterator(AtomMapType::iterator it) bool Atom::hasLeft() const { - return mLeftIndex >= 0; + return mHasLeft; } bool Atom::hasRight() const { - return mRightIndex >= 0; + return mHasRight; } -int Atom::leftIndex() const +size_t Atom::leftIndex() const { + GAPS_ASSERT(mHasLeft); return mLeftIndex; } -int Atom::rightIndex() const +size_t Atom::rightIndex() const { + GAPS_ASSERT(mHasRight); return mRightIndex; } -int Atom::index() const +size_t Atom::index() const { return mIndex; } diff --git a/src/atomic/Atom.h b/src/atomic/Atom.h index bf197971..956cbffb 100644 --- a/src/atomic/Atom.h +++ b/src/atomic/Atom.h @@ -1,13 +1,16 @@ #ifndef __COGAPS_ATOM_H__ #define __COGAPS_ATOM_H__ +#include +#include + struct Atom; class Archive; class AtomicDomain; // this is the map used internally by the atomic domain -#include "../data_structures/MutableMap.h" -typedef MutableMap AtomMapType; +// #include "../data_structures/MutableMap.h" -- it is unsafe +typedef std::map AtomMapType; struct AtomNeighborhood { @@ -30,28 +33,32 @@ struct Atom void updateMass(float newMass); friend Archive& operator<<(Archive& ar, const Atom &a); friend Archive& operator>>(Archive& ar, Atom &a); -private: // only the atomic domain can change the position of an atom, since it is // responsible for keeping them ordered friend class AtomicDomain; - void updatePos(uint64_t newPos); - void setLeftIndex(int index); - void setRightIndex(int index); - void setIndex(int index); - void setIterator(AtomMapType::iterator it); bool hasLeft() const; bool hasRight() const; - int leftIndex() const; - int rightIndex() const; - int index() const; + size_t leftIndex() const; + size_t rightIndex() const; + size_t index() const; AtomMapType::iterator iterator() const; +private: + void updatePos(uint64_t newPos); + void setLeftIndex(size_t index); + void setRightIndex(size_t index); + void unsetLeftIndex(); + void unsetRightIndex(); + void setIndex(size_t index); + void setIterator(AtomMapType::iterator it); + AtomMapType::iterator mIterator; // iterator to position in map uint64_t mPos; // position of the atom - int mLeftIndex; // index of left neighbor - int mRightIndex; // index of right neighbor - int mIndex; // storing the index allows vector lookup once found in map + bool mHasRight,mHasLeft; //are the following two defined + std::size_t mLeftIndex; // index of left neighbor in the atomic storage + std::size_t mRightIndex; // index of right neighbor in the atomic storage + std::size_t mIndex; // storing the index allows vector lookup once found in map float mMass; // mass of the atom }; -#endif // __COGAPS_ATOM_H__ \ No newline at end of file +#endif // __COGAPS_ATOM_H__ diff --git a/src/atomic/AtomicDomain.cpp b/src/atomic/AtomicDomain.cpp index dd62cd64..365d3ddc 100755 --- a/src/atomic/AtomicDomain.cpp +++ b/src/atomic/AtomicDomain.cpp @@ -11,6 +11,9 @@ AtomicDomain::AtomicDomain(uint64_t nBins) { + //nBins is the number of matrix elements to be fitted (nrows(A)*ncols(A)+nrows(P)*ncols(P)) in our run; + //each element has a band in atomic space; the max possible atomic coord is the max uit_64 and + //we want the bands to be equal, so see below uint64_t binLength = std::numeric_limits::max() / nBins; mDomainLength = binLength * nBins; } @@ -19,19 +22,30 @@ Atom* AtomicDomain::front() { GAPS_ASSERT(size() > 0); return &(mAtoms[mAtomMap.begin()->second]); + //we take the first element in atom map, take the value (we use key->value notation in comments), + //and index the atom storage by the value and return the reference to the atom } -Atom* AtomicDomain::randomAtom(GapsRng *rng) -{ - GAPS_ASSERT(size() > 0); - unsigned index = rng->uniform32(0, mAtoms.size() - 1); +Atom* AtomicDomain::storedAtom(size_t ind){ + GAPS_ASSERT_MSG(size() > ind, "AtomicDomain tries to get atom higher than ut has"); + return &(mAtoms[ind]); +} + +std::map::const_iterator AtomicDomain::front_it(){ + GAPS_ASSERT_MSG(size() > 0, "empty AtomicDomain tries to get random atom"); + return mAtomMap.begin(); +} + +Atom* AtomicDomain::randomAtom(GapsRng *rng){ + GAPS_ASSERT_MSG(size() > 0, "empty AtomicDomain tries to get random atom"); + size_t index = rng->uniform64(0, mAtoms.size() - 1); return &(mAtoms[index]); } AtomNeighborhood AtomicDomain::randomAtomWithNeighbors(GapsRng *rng) { GAPS_ASSERT(size() > 0); - unsigned index = rng->uniform32(0, mAtoms.size() - 1); + size_t index = rng->uniform64(0, mAtoms.size() - 1); Atom *center = &(mAtoms[index]); // [AI-generated] Build the neighborhood used by move/exchange proposals; edge atoms have // a missing neighbor on one side of the ordered atomic domain. @@ -55,12 +69,22 @@ uint64_t AtomicDomain::size() const return mAtoms.size(); } + Atom* AtomicDomain::insert(uint64_t pos, float mass) { - unsigned index = mAtoms.size(); + size_t index = mAtoms.size(); + //it will be the index of the atom if inserted + auto insertresult=mAtomMap.insert(std::pair(pos, index)); + if (!insertresult.second) { + //already exists - so the second ib the returned pair is false + //and the first is the map iterator to the existing pair + index=insertresult.first->second; + return &(mAtoms[index]); + } + //if we are here, we did de novo insertion mAtoms.push_back(Atom(pos, mass)); mAtoms[index].setIndex(index); - mAtoms[index].setIterator(mAtomMap.insert(std::pair(pos, index)).first); + mAtoms[index].setIterator(insertresult.first); // connect with right and left neighbors AtomMapType::iterator itRight(mAtoms[index].iterator()); @@ -81,34 +105,57 @@ Atom* AtomicDomain::insert(uint64_t pos, float mass) void AtomicDomain::erase(Atom *atom) { + GAPS_ASSERT_MSG(size() > 0, "empty AtomicDomain tries to erase an atom"); mAtomMap.erase(atom->iterator()); - if (atom->hasLeft()) + //remove from map + if (atom->hasLeft() && atom->hasRight()) { mAtoms[atom->leftIndex()].setRightIndex(atom->rightIndex()); - } - if (atom->hasRight()) - { mAtoms[atom->rightIndex()].setLeftIndex(atom->leftIndex()); + } else { + if (atom->hasRight()) + //is we are here, hasLeft is false + { + mAtoms[atom->rightIndex()].unsetLeftIndex(); + } + if (atom->hasLeft()) + //is we are here, hasRight is false + { + mAtoms[atom->leftIndex()].unsetRightIndex(); + } } - // update the neighbors and the map - unsigned index = atom->index(); - if (index < mAtoms.size() - 1) // we are moving the last atom + size_t index = atom->index(); + if (index < mAtoms.size() - 1) + // we are erasing not the last atom + // we copy the last to [index] and then + // erase the last, the same axtion, + // saves time + // we just copy all the data members, + // because we can + // AtomicDomain is a friend of Atom { - int leftIndex = mAtoms.back().leftIndex(); - int rightIndex = mAtoms.back().rightIndex(); - mAtoms[index] = mAtoms.back(); - mAtoms[index].setIndex(index); - mAtoms[index].iterator()->second = index; - if (leftIndex >= 0) - { - mAtoms[leftIndex].setRightIndex(index); + const Atom &src = mAtoms.back(); + Atom &dst = mAtoms[index]; + + dst.mIterator=src.mIterator; + dst.mPos=src.mPos; // position of the atom + dst.mHasRight=src.mHasRight; + dst.mHasLeft=src.mHasLeft; + //are the following two defined + dst.mLeftIndex=src.mLeftIndex; // index of left neighbor in the atomic storage + dst.mRightIndex=src.mRightIndex; // index of right neighbor in the atomic storage + dst.mMass=src.mMass; + GAPS_ASSERT(dst.mIndex==index); + dst.mIterator->second=index; //we tell the map we moved the atom inside the array + if (dst.hasLeft()){ + mAtoms[dst.leftIndex()].setRightIndex(index); } - if (rightIndex >= 0) - { - mAtoms[rightIndex].setLeftIndex(index); + if (dst.hasRight()){ + mAtoms[dst.rightIndex()].setLeftIndex(index); } } + //we remove the last atom now -- whatever mAtoms.pop_back(); } @@ -118,8 +165,12 @@ void AtomicDomain::move(Atom *atom, uint64_t newPos) // in for missing neighbors at the edges. GAPS_ASSERT(newPos > (atom->hasLeft() ? mAtoms[atom->leftIndex()].pos() : 0)); GAPS_ASSERT(newPos < (atom->hasRight() ? mAtoms[atom->rightIndex()].pos() : mDomainLength)); + //we do not jump over neighbour + + size_t storageIdx = atom->iterator()->second; + mAtomMap.erase(atom->pos()); atom->updatePos(newPos); - mAtomMap.updateKey(atom->iterator(), newPos); + atom->setIterator(mAtomMap.insert(std::pair(newPos, storageIdx)).first); } Archive& operator<<(Archive &ar, const AtomicDomain &domain) diff --git a/src/atomic/AtomicDomain.h b/src/atomic/AtomicDomain.h index 8ffcb898..c41a73ba 100755 --- a/src/atomic/AtomicDomain.h +++ b/src/atomic/AtomicDomain.h @@ -17,10 +17,18 @@ class AtomicDomain public: explicit AtomicDomain(uint64_t nBins); Atom* front(); + std::map::const_iterator front_it(); + Atom* storedAtom(size_t ind); Atom* randomAtom(GapsRng *rng); AtomNeighborhood randomAtomWithNeighbors(GapsRng *rng); uint64_t randomFreePosition(GapsRng *rng) const; uint64_t size() const; + uint64_t DomainLength() const; + //we move the access function to public to be able to test them + Atom* insert(uint64_t pos, float mass); + void erase(Atom *atom); + void move(Atom *atom, uint64_t newPos); + friend Archive& operator<<(Archive &ar, const AtomicDomain &domain); friend Archive& operator>>(Archive &ar, AtomicDomain &domain); private: @@ -32,13 +40,26 @@ class AtomicDomain template friend class SingleThreadedGibbsSampler; - Atom* insert(uint64_t pos, float mass); - void erase(Atom *atom); - void move(Atom *atom, uint64_t newPos); + std::vector mAtoms; + //It is the vector of atoms, we store them here, it is storage + //CoPilot: + //The index type for the std::vector named mAtoms is std::vector::size_type, + //which is typically defined as std::size_t. + //This type is used for indexing and specifying the size of the vector. + + AtomMapType mAtomMap; + //it is a map from atomic space coordinate (key) to index in the storage + //sorted, used when inserting atoms to find neighbors + + //Two structures are easy to understand in DB metaphor: mAtoms is the table of atoms; mAtomMap is the index by atomic coord - AtomMapType mAtomMap; // sorted, used when inserting atoms to find neighbors - std::vector mAtoms; // unsorted, used for reads uint64_t mDomainLength; // size of atomic domain to ensure all bins are equal length + }; +inline uint64_t AtomicDomain::DomainLength() const +{ + return mDomainLength; +} + #endif // __COGAPS_ATOMIC_DOMAIN_H__ diff --git a/src/atomic/ConcurrentAtom.cpp b/src/atomic/ConcurrentAtom.cpp deleted file mode 100644 index 4e4dae0c..00000000 --- a/src/atomic/ConcurrentAtom.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "ConcurrentAtom.h" -#include "../utils/Archive.h" - -#include - -ConcurrentAtomNeighborhood::ConcurrentAtomNeighborhood() - : center(NULL), left(NULL), right(NULL) -{} - -ConcurrentAtomNeighborhood::ConcurrentAtomNeighborhood(ConcurrentAtom *l, ConcurrentAtom *c, ConcurrentAtom *r) - : center(c), left(l), right(r) -{} - -bool ConcurrentAtomNeighborhood::hasLeft() const -{ - return left != NULL; -} - -bool ConcurrentAtomNeighborhood::hasRight() const -{ - return right != NULL; -} - -ConcurrentAtom::ConcurrentAtom(uint64_t p, float m) - : mPos(p), mLeft(NULL), mRight(NULL), mIterator(), mIndex(0), mMass(m) -{} - -uint64_t ConcurrentAtom::pos() const -{ - return mPos; -} - -float ConcurrentAtom::mass() const -{ - return mMass; -} - -bool ConcurrentAtom::hasLeft() const -{ - return mLeft != NULL; -} - -bool ConcurrentAtom::hasRight() const -{ - return mRight != NULL; -} - -ConcurrentAtom* ConcurrentAtom::left() const -{ - return mLeft; -} - -ConcurrentAtom* ConcurrentAtom::right() const -{ - return mRight; -} - -void ConcurrentAtom::updateMass(float newMass) -{ - mMass = newMass; -} - -void ConcurrentAtom::updatePos(uint64_t newPos) -{ - mPos = newPos; -} - -void ConcurrentAtom::setLeft(ConcurrentAtom* atom) -{ - mLeft = atom; -} - -void ConcurrentAtom::setRight(ConcurrentAtom* atom) -{ - mRight = atom; -} - -void ConcurrentAtom::setIndex(unsigned index) -{ - mIndex = index; -} - -void ConcurrentAtom::setIterator(ConcurrentAtomMapType::iterator it) -{ - mIterator = it; -} - -unsigned ConcurrentAtom::index() const -{ - return mIndex; -} - -ConcurrentAtomMapType::iterator ConcurrentAtom::iterator() const -{ - return mIterator; -} - -Archive& operator<<(Archive &ar, const ConcurrentAtom &a) -{ - ar << a.mPos << a.mMass; - return ar; -} - -Archive& operator>>(Archive &ar, ConcurrentAtom &a) -{ - ar >> a.mPos >> a.mMass; - return ar; -} diff --git a/src/atomic/ConcurrentAtom.h b/src/atomic/ConcurrentAtom.h deleted file mode 100644 index 37b810a2..00000000 --- a/src/atomic/ConcurrentAtom.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef __COGAPS_CONCURRENT_ATOM_H__ -#define __COGAPS_CONCURRENT_ATOM_H__ - -struct ConcurrentAtom; -class ConcurrentAtomicDomain; -class Archive; - -// this is the map used internally by the atomic domain -#include "../data_structures/MutableMap.h" -typedef MutableMap ConcurrentAtomMapType; - -struct ConcurrentAtomNeighborhood -{ - ConcurrentAtomNeighborhood(); - ConcurrentAtomNeighborhood(ConcurrentAtom *l, ConcurrentAtom *c, ConcurrentAtom *r); - bool hasLeft() const; - bool hasRight() const; - ConcurrentAtom *center; - ConcurrentAtom *left; - ConcurrentAtom *right; -}; - -struct ConcurrentAtom -{ -public: - ConcurrentAtom(uint64_t p, float m); - uint64_t pos() const; - float mass() const; - void updateMass(float newMass); - friend Archive& operator<<(Archive& ar, const ConcurrentAtom &a); - friend Archive& operator>>(Archive& ar, ConcurrentAtom &a); -//private: // TODO - // only the atomic domain can change the position of an atom, since it is - // responsible for keeping them ordered - friend class ConcurrentAtomicDomain; - void updatePos(uint64_t newPos); - void setLeft(ConcurrentAtom *atom); - void setRight(ConcurrentAtom *atom); - void setIndex(unsigned index); - void setIterator(ConcurrentAtomMapType::iterator it); - bool hasLeft() const; - bool hasRight() const; - ConcurrentAtom* left() const; - ConcurrentAtom* right() const; - unsigned index() const; - ConcurrentAtomMapType::iterator iterator() const; - - uint64_t mPos; - ConcurrentAtom *mLeft; - ConcurrentAtom *mRight; - ConcurrentAtomMapType::iterator mIterator; // iterator to position in map - unsigned mIndex; // storing the index allows vector lookup once found in map - float mMass; -}; - -#endif // __COGAPS_CONCURRENT_ATOM_H__ \ No newline at end of file diff --git a/src/atomic/ConcurrentAtomicDomain.cpp b/src/atomic/ConcurrentAtomicDomain.cpp deleted file mode 100644 index 18aea6f3..00000000 --- a/src/atomic/ConcurrentAtomicDomain.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include "ConcurrentAtomicDomain.h" -#include "../math/Random.h" -#include "../utils/Archive.h" -#include "../utils/GapsAssert.h" - -#include -#include - -static bool compareAtoms(ConcurrentAtom *a1, ConcurrentAtom *a2) -{ - return a1->pos() < a2->pos(); -} - -ConcurrentAtomicDomain::ConcurrentAtomicDomain(uint64_t nBins) -{ - uint64_t binLength = std::numeric_limits::max() / nBins; - mDomainLength = binLength * nBins; -} - -ConcurrentAtom* ConcurrentAtomicDomain::front() -{ - GAPS_ASSERT(size() > 0); - return (*mAtomMap.begin()).second; -} - -const ConcurrentAtom* ConcurrentAtomicDomain::front() const -{ - GAPS_ASSERT(size() > 0); - return (*mAtomMap.begin()).second; -} - -ConcurrentAtom* ConcurrentAtomicDomain::randomAtom(GapsRng *rng) -{ - GAPS_ASSERT(size() > 0); - unsigned index = rng->uniform32(0, mAtoms.size() - 1); - return mAtoms[index]; -} - -ConcurrentAtomNeighborhood ConcurrentAtomicDomain::randomAtomWithNeighbors(GapsRng *rng) -{ - GAPS_ASSERT(size() > 0); - unsigned index = rng->uniform32(0, mAtoms.size() - 1); - return ConcurrentAtomNeighborhood(mAtoms[index]->left(), mAtoms[index], mAtoms[index]->right()); -} - -uint64_t ConcurrentAtomicDomain::randomFreePosition(GapsRng *rng) const -{ - uint64_t pos = rng->uniform64(1, mDomainLength); - while (mAtomMap.count(pos)) - { - pos = rng->uniform64(1, mDomainLength); - } - return pos; -} - -uint64_t ConcurrentAtomicDomain::size() const -{ - return mAtoms.size(); -} - -// safe to call concurrently from OpenMP threads -void ConcurrentAtomicDomain::cacheErase(ConcurrentAtom *atom) -{ - #pragma omp critical(AtomicInsertOrErase) - { - mEraseCache.push_back(atom); - } -} - -// not thread safe -void ConcurrentAtomicDomain::flushEraseCache() -{ - std::sort(mEraseCache.begin(), mEraseCache.end(), compareAtoms); - for (unsigned i = 0; i < mEraseCache.size(); ++i) - { - erase(mEraseCache[i]); - } - mEraseCache.clear(); -} - -// not thread safe -ConcurrentAtom* ConcurrentAtomicDomain::insert(uint64_t pos, float mass) -{ - // insert atom into vector and map, record the iterator and index in each - ConcurrentAtom *atom = new ConcurrentAtom(pos, mass); - atom->setIterator(mAtomMap.insert(std::pair(pos, atom)).first); - atom->setIndex(mAtoms.size()); - mAtoms.push_back(atom); - - // connect with right and left neighbors - ConcurrentAtomMapType::iterator itRight(atom->iterator()); - if (++itRight != mAtomMap.end()) - { - atom->setRight((*itRight).second); - (*itRight).second->setLeft(atom); - } - ConcurrentAtomMapType::iterator itLeft(atom->iterator()); - if (itLeft != mAtomMap.begin()) - { - --itLeft; - atom->setLeft((*itLeft).second); - (*itLeft).second->setRight(atom); - } - return atom; -} - -// not thread safe -void ConcurrentAtomicDomain::erase(ConcurrentAtom *atom) -{ - mAtomMap.erase(atom->iterator()); - mAtoms[atom->index()] = mAtoms.back(); - mAtoms[atom->index()]->setIndex(atom->index()); - mAtoms.pop_back(); - if (atom->hasLeft()) - { - atom->left()->setRight(atom->right()); - } - if (atom->hasRight()) - { - atom->right()->setLeft(atom->left()); - } - delete atom; -} - -// safe to call concurrently from OpenMP threads -void ConcurrentAtomicDomain::move(ConcurrentAtom *atom, uint64_t newPos) -{ - // [AI-generated] Validate the move stays between neighboring atoms; domain endpoints stand - // in for missing neighbors at the edges. - GAPS_ASSERT(newPos > (atom->hasLeft() ? atom->left()->pos() : 0)); - GAPS_ASSERT(newPos < (atom->hasRight() ? atom->right()->pos() : mDomainLength)); - atom->updatePos(newPos); - mAtomMap.updateKey(atom->iterator(), newPos); -} - -Archive& operator<<(Archive &ar, const ConcurrentAtomicDomain &domain) -{ - ar << domain.mDomainLength << domain.mAtoms.size(); - for (unsigned i = 0; i < domain.mAtoms.size(); ++i) - { - ar << *(domain.mAtoms[i]); - } - return ar; -} - -Archive& operator>>(Archive &ar, ConcurrentAtomicDomain &domain) -{ - ConcurrentAtom temp(0, 0.f); - uint64_t size = 0; - ar >> domain.mDomainLength >> size; - for (unsigned i = 0; i < size; ++i) - { - ar >> temp; - domain.insert(temp.pos(), temp.mass()); - } - return ar; -} - -#ifdef GAPS_DEBUG -bool ConcurrentAtomicDomain::isSorted() const -{ - if (size() == 0) - return true; - const ConcurrentAtom *atom = front(); - bool valid = !atom->hasLeft(); - while (atom->hasRight()) - { - valid = valid && (atom->right()->pos() > atom->pos()); - atom = atom->right(); - valid = valid && (atom->left()->pos() < atom->pos()); - } - return valid; -} -#endif \ No newline at end of file diff --git a/src/atomic/ConcurrentAtomicDomain.h b/src/atomic/ConcurrentAtomicDomain.h deleted file mode 100644 index 9876b704..00000000 --- a/src/atomic/ConcurrentAtomicDomain.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef __COGAPS_CONCURRENT_ATOMIC_DOMAIN_H__ -#define __COGAPS_CONCURRENT_ATOMIC_DOMAIN_H__ - -#include "ConcurrentAtom.h" - -#include - -template -class SingleThreadedGibbsSampler; - -struct ConcurrentAtom; -class Archive; -class GapsRng; -class ProposalQueue; - -class ConcurrentAtomicDomain -{ -public: - explicit ConcurrentAtomicDomain(uint64_t nBins); - ConcurrentAtom* front(); - const ConcurrentAtom* front() const; - ConcurrentAtom* randomAtom(GapsRng *rng); - ConcurrentAtomNeighborhood randomAtomWithNeighbors(GapsRng *rng); - uint64_t randomFreePosition(GapsRng *rng) const; - uint64_t size() const; - void cacheErase(ConcurrentAtom *atom); // OpenMP thread safe - void move(ConcurrentAtom *atom, uint64_t newPos); // OpenMP thread safe - void flushEraseCache(); - friend Archive& operator<<(Archive &ar, const ConcurrentAtomicDomain &domain); - friend Archive& operator>>(Archive &ar, ConcurrentAtomicDomain &domain); -#ifdef GAPS_DEBUG - bool isSorted() const; -#endif -private: - - // only these classes can call the non-thread safe insert and erase functions - template - friend class SingleThreadedGibbsSampler; - friend class ProposalQueue; - - // these functions are not thread safe - ConcurrentAtom* insert(uint64_t pos, float mass); - void erase(ConcurrentAtom *atom); - - ConcurrentAtomMapType mAtomMap; // sorted, used when inserting atoms to find neighbors - std::vector mAtoms; // unsorted, used for random selection of atoms - std::vector mEraseCache; - uint64_t mDomainLength; // size of atomic domain to ensure all bins are equal length -}; - -#endif // __COGAPS_CONCURRENT_ATOMIC_DOMAIN_H__ diff --git a/src/atomic/ProposalQueue.cpp b/src/atomic/ProposalQueue.cpp deleted file mode 100755 index adf73a5d..00000000 --- a/src/atomic/ProposalQueue.cpp +++ /dev/null @@ -1,306 +0,0 @@ -#include "ProposalQueue.h" -#include "../atomic/ConcurrentAtomicDomain.h" -#include "../math/Math.h" -#include "../math/Random.h" -#include "../utils/Archive.h" -#include "../utils/GapsAssert.h" - -#include - -//////////////////////////////// AtomicProposal //////////////////////////////// - -AtomicProposal::AtomicProposal(char t, GapsRandomState *randState) - : rng(randState), pos(0), atom1(NULL), atom2(NULL), r1(0), c1(0), r2(0), - c2(0), type(t) -{} - -//////////////////////////////// ProposalQueue ///////////////////////////////// - -ProposalQueue::ProposalQueue(uint64_t nElements, uint64_t nPatterns, -GapsRandomState *randState) - : -mUsedMatrixIndices(nElements / nPatterns), -mRandState(randState), -mRng(randState), -mMinAtoms(0), -mMaxAtoms(0), -mBinLength(std::numeric_limits::max() / nElements), -mNumCols(nPatterns), -mAlpha(0.0), -mDomainLength(static_cast(mBinLength * nElements)), -mNumBins(static_cast(nElements)), -mU1(0.f), -mU2(0.f), -mNumProcessed(0), -mUseCachedRng(false) -{} - -void ProposalQueue::setAlpha(float alpha) -{ - mAlpha = static_cast(alpha); -} - -void ProposalQueue::setLambda(float lambda) -{ - mLambda = lambda; -} - -unsigned ProposalQueue::nProcessed() const -{ - return mNumProcessed; -} - -void ProposalQueue::populate(ConcurrentAtomicDomain &domain, unsigned limit) -{ - GAPS_ASSERT(mQueue.empty()); - GAPS_ASSERT(mUsedAtoms.isEmpty()); - GAPS_ASSERT(mUsedMatrixIndices.isEmpty()); - GAPS_ASSERT(mProposedMoves.isEmpty()); - GAPS_ASSERT(mMinAtoms == mMaxAtoms); - GAPS_ASSERT_MSG(mMaxAtoms == domain.size(), mMaxAtoms << " != " << domain.size()); - - bool success = true; - mNumProcessed = 0; - while (mNumProcessed < limit && success) - { - if (!makeProposal(domain)) - { - success = false; - mUseCachedRng = true; - } - else - { - ++mNumProcessed; - } - } -} - -void ProposalQueue::clear() -{ - GAPS_ASSERT(mMinAtoms == mMaxAtoms); - mQueue.clear(); - mUsedMatrixIndices.clear(); - mUsedAtoms.clear(); - mProposedMoves.clear(); -} - -unsigned ProposalQueue::size() const -{ - return mQueue.size(); -} - -AtomicProposal& ProposalQueue::operator[](int n) -{ - GAPS_ASSERT(mQueue.size() > 0); - GAPS_ASSERT(static_cast(n) < mQueue.size()); - return mQueue[n]; -} - -void ProposalQueue::acceptDeath() -{ - #pragma omp atomic - --mMaxAtoms; -} - -void ProposalQueue::rejectDeath() -{ - #pragma omp atomic - ++mMinAtoms; -} - -void ProposalQueue::acceptBirth() -{ - #pragma omp atomic - ++mMinAtoms; -} - -void ProposalQueue::rejectBirth() -{ - #pragma omp atomic - --mMaxAtoms; -} - -float ProposalQueue::deathProb(double nAtoms) const -{ - double numer = nAtoms * mDomainLength; - return numer / (numer + mAlpha * mNumBins * (mDomainLength - nAtoms)); -} - -bool ProposalQueue::makeProposal(ConcurrentAtomicDomain &domain) -{ - // [AI-generated] Reuse cached uniforms when a failed proposal is retried; otherwise draw - // fresh uniforms for this proposal decision. - mU1 = mUseCachedRng ? mU1 : mRng.uniform(); - mU2 = mUseCachedRng ? mU2: mRng.uniform(); - mUseCachedRng = false; - - if (mMinAtoms < 2 && mMaxAtoms >= 2) - { - return false; // special indeterminate case - } - - if (mMaxAtoms < 2) - { - return birth(domain); // always birth when 0 or 1 atoms exist - } - - float lowerBound = deathProb(static_cast(mMinAtoms)); - float upperBound = deathProb(static_cast(mMaxAtoms)); - if (mU1 < 0.5f) - { - if (mU2 < lowerBound) - { - return death(domain); - } - if (mU2 >= upperBound) - { - return birth(domain); - } - return false; // can't determine B/D since range is too wide - } - // [AI-generated] Split the upper half of the proposal interval between move and exchange. - return (mU1 < 0.75f) ? move(domain) : exchange(domain); -} - -bool ProposalQueue::birth(ConcurrentAtomicDomain &domain) -{ - AtomicProposal prop('B', mRandState); - uint64_t pos = domain.randomFreePosition(&(prop.rng)); - - if (mProposedMoves.overlap(pos)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // this birth would break assumption moves doesn't re-order domain - } - - prop.r1 = (pos / mBinLength) / mNumCols; - prop.c1 = (pos / mBinLength) % mNumCols; - if (mUsedMatrixIndices.contains(prop.r1)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // matrix conflict - can't compute gibbs mass - } - prop.atom1 = domain.insert(pos, 0.f); - - mUsedMatrixIndices.insert(prop.r1); - mUsedAtoms.insert(prop.atom1->pos()); - mQueue.push_back(prop); - ++mMaxAtoms; - return true; -} - -bool ProposalQueue::death(ConcurrentAtomicDomain &domain) -{ - AtomicProposal prop('D', mRandState); - prop.atom1 = domain.randomAtom(&(prop.rng)); - prop.r1 = (prop.atom1->pos() / mBinLength) / mNumCols; - prop.c1 = (prop.atom1->pos() / mBinLength) % mNumCols; - - if (mUsedMatrixIndices.contains(prop.r1)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // matrix conflict - can't compute gibbs mass or deltaLL - } - - mUsedMatrixIndices.insert(prop.r1); - mUsedAtoms.insert(prop.atom1->pos()); - mQueue.push_back(prop); - --mMinAtoms; - return true; -} - -bool ProposalQueue::move(ConcurrentAtomicDomain &domain) -{ - AtomicProposal prop('M', mRandState); - ConcurrentAtomNeighborhood hood = domain.randomAtomWithNeighbors(&(prop.rng)); - prop.atom1 = hood.center; - - // [AI-generated] Bound the move by neighboring atom positions; use domain endpoints for - // edge atoms. - uint64_t lbound = hood.hasLeft() ? hood.left->pos() : 0; - uint64_t rbound = hood.hasRight() ? hood.right->pos() : static_cast(mDomainLength); - - if (mUsedAtoms.contains(lbound) || mUsedAtoms.contains(rbound)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // atomic conflict - don't know neighbors - } - - prop.pos = prop.rng.uniform64(lbound + 1, rbound - 1); - prop.r1 = (prop.atom1->pos() / mBinLength) / mNumCols; - prop.c1 = (prop.atom1->pos() / mBinLength) % mNumCols; - prop.r2 = (prop.pos / mBinLength) / mNumCols; - prop.c2 = (prop.pos / mBinLength) % mNumCols; - - if (mUsedMatrixIndices.contains(prop.r1) || mUsedMatrixIndices.contains(prop.r2)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // matrix conflict - can't compute deltaLL - } - - if (prop.r1 == prop.r2 && prop.c1 == prop.c2) - { - domain.move(prop.atom1, prop.pos); - return true; // automatically accept moves in same bin - } - - mQueue.push_back(prop); - mUsedMatrixIndices.insert(prop.r1); - mUsedMatrixIndices.insert(prop.r2); - mUsedAtoms.insert(prop.atom1->pos()); - mProposedMoves.insert(prop.atom1->pos(), prop.pos); - return true; -} - -bool ProposalQueue::exchange(ConcurrentAtomicDomain &domain) -{ - AtomicProposal prop('E', mRandState); - ConcurrentAtomNeighborhood hood = domain.randomAtomWithNeighbors(&(prop.rng)); - prop.atom1 = hood.center; - // [AI-generated] Exchange with the right neighbor, wrapping the last atom to the first. - prop.atom2 = hood.hasRight() ? hood.right : domain.front(); - prop.r1 = (prop.atom1->pos() / mBinLength) / mNumCols; - prop.c1 = (prop.atom1->pos() / mBinLength) % mNumCols; - prop.r2 = (prop.atom2->pos() / mBinLength) / mNumCols; - prop.c2 = (prop.atom2->pos() / mBinLength) % mNumCols; - - if (mUsedMatrixIndices.contains(prop.r1) || mUsedMatrixIndices.contains(prop.r2)) - { - mRandState->rollBackOnce(); // ensure same proposal next time - return false; // matrix conflict - can't compute deltaLL or gibbs mass - } - - if (prop.r1 == prop.r2 && prop.c1 == prop.c2) - { - float newMass = prop.rng.truncGammaUpper(prop.atom1->mass() + prop.atom2->mass(), 1.f / mLambda); - // [AI-generated] Keep the larger atom as the reference for the mass-transfer direction. - float delta = (prop.atom1->mass() > prop.atom2->mass()) ? newMass - prop.atom1->mass() : prop.atom2->mass() - newMass; - if (prop.atom1->mass() + delta > gaps::epsilon && prop.atom2->mass() - delta > gaps::epsilon) - { - prop.atom1->updateMass(prop.atom1->mass() + delta); - prop.atom2->updateMass(prop.atom2->mass() - delta); - } - return true; // automatically accept exchanges in same bin - } - - mQueue.push_back(prop); - mUsedMatrixIndices.insert(prop.r1); - mUsedMatrixIndices.insert(prop.r2); - return true; -} - -Archive& operator<<(Archive &ar, const ProposalQueue &q) -{ - ar << q.mRng << q.mMinAtoms << q.mMaxAtoms << q.mBinLength << q.mNumCols - << q.mAlpha << q.mDomainLength << q.mNumBins << q.mLambda - << q.mUseCachedRng << q.mU1 << q.mU2; - return ar; -} - -Archive& operator>>(Archive &ar, ProposalQueue &q) -{ - ar >> q.mRng >> q.mMinAtoms >> q.mMaxAtoms >> q.mBinLength >> q.mNumCols - >> q.mAlpha >> q.mDomainLength >> q.mNumBins >> q.mLambda - >> q.mUseCachedRng >> q.mU1 >> q.mU2; - return ar; -} \ No newline at end of file diff --git a/src/atomic/ProposalQueue.h b/src/atomic/ProposalQueue.h deleted file mode 100755 index d542ce44..00000000 --- a/src/atomic/ProposalQueue.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef __COGAPS_PROPOSAL_QUEUE_H__ -#define __COGAPS_PROPOSAL_QUEUE_H__ - -#include "../math/Random.h" -#include "../data_structures/HashSets.h" - -#include -#include -#include - -struct ConcurrentAtom; -class Archive; -class ConcurrentAtomicDomain; - -struct AtomicProposal -{ - AtomicProposal(char t, GapsRandomState *randState); - - mutable GapsRng rng; // used for consistency no matter number of threads - uint64_t pos; // used for move - ConcurrentAtom *atom1; // used for birth/death/move/exchange - ConcurrentAtom *atom2; // used for exchange - uint32_t r1; // row of atom1 - uint32_t c1; // col of atom1 - uint32_t r2; // row of pos (move) or atom2 (exchange) - uint32_t c2; // col of pos (move) or atom2 (exchange) - char type; // birth (B), death (D), move (M), exchange (E) -}; - -class ProposalQueue -{ -public: - ProposalQueue(uint64_t nElements, uint64_t nPatterns, GapsRandomState *randState); - void setAlpha(float alpha); - void setLambda(float lambda); - void populate(ConcurrentAtomicDomain &domain, unsigned limit); - void clear(); - unsigned size() const; - AtomicProposal& operator[](int n); - void acceptDeath(); - void rejectDeath(); - void acceptBirth(); - void rejectBirth(); - unsigned nProcessed() const; - friend Archive& operator<<(Archive &ar, const ProposalQueue &queue); - friend Archive& operator>>(Archive &ar, ProposalQueue &queue); -private: - float deathProb(double nAtoms) const; - bool makeProposal(ConcurrentAtomicDomain &domain); - bool birth(ConcurrentAtomicDomain &domain); - bool death(ConcurrentAtomicDomain &domain); - bool move(ConcurrentAtomicDomain &domain); - bool exchange(ConcurrentAtomicDomain &domain); - - std::vector mQueue; // not really a queue for now - FixedHashSetU32 mUsedMatrixIndices; - SmallHashSetU64 mUsedAtoms; - SmallPairedHashSetU64 mProposedMoves; - GapsRandomState *mRandState; - mutable GapsRng mRng; - uint64_t mMinAtoms; - uint64_t mMaxAtoms; - uint64_t mBinLength; // length of single bin - uint64_t mNumCols; - double mAlpha; - double mDomainLength; // length of entire atomic domain - double mNumBins; // number of matrix elements - float mLambda; - float mU1; - float mU2; - unsigned mNumProcessed; - bool mUseCachedRng; -}; - -#endif // __COGAPS_PROPOSAL_QUEUE_H__ \ No newline at end of file diff --git a/src/cpp_tests/README.md b/src/cpp_tests/README.md index 001967a1..6ac75184 100644 --- a/src/cpp_tests/README.md +++ b/src/cpp_tests/README.md @@ -19,13 +19,21 @@ by testthat ## test-runner.cpp -a custom (as compared to `testthat::use_catch()`) runner to run cpp tests, exposes `run_catch_unit_tests()` in R, usage: +a custom (as compared to `testthat::use_catch()`) runner to run cpp tests. It +exposes three functions in R. They are internal, so reach them with `:::`. + +### Running them by hand -- the plain, readable way + +This is what you want while working on the C++ code. Output goes to the console +in the usual Catch format; the return value is the number of failed assertions, +0 meaning everything passed. + ``` #just run all cpp tests -CoGAPS::run_catch_unit_tests() +CoGAPS:::run_catch_unit_tests() #use cpp xml reporter to see debug info -CoGAPS:::run_catch_unit_tests(reporter=“xml") +CoGAPS:::run_catch_unit_tests(reporter="xml") #call a single cpp test by name CoGAPS:::run_catch_unit_tests_by_tag("Test Vector.h") @@ -33,11 +41,66 @@ CoGAPS:::run_catch_unit_tests_by_tag("Test Vector.h") #call cpp test(s) by tag "vector" CoGAPS:::run_catch_unit_tests_by_tag("[vector]") +#call cpp test(s) by tag "vector" or "green" +CoGAPS:::run_catch_unit_tests_by_tag("[vector],[green]") + +#call cpp test(s) by tag "vector" and "green" +CoGAPS:::run_catch_unit_tests_by_tag("[vector][green]") +CoGAPS:::run_catch_unit_tests_by_tag("[green][vector]") + +``` + +Some of the cpp tests read the paths of the packaged data files from the global +environment, so set those first or the file-parser cases will fail: + +``` +gistCsvPath <<- system.file("extdata/GIST.csv", package="CoGAPS") +gistTsvPath <<- system.file("extdata/GIST.tsv", package="CoGAPS") +gistMtxPath <<- system.file("extdata/GIST.mtx", package="CoGAPS") +gistGctPath <<- system.file("extdata/GIST.gct", package="CoGAPS") +``` + +### How testthat runs them + +`tests/testthat/test_cpp.R` runs the same suite as part of `devtools::test()` and +`R CMD check`, so a broken C++ test breaks the R build. It does not use the +console form: Catch writes its report from C++, straight to stdout, where +testthat cannot capture it, and the whole suite would collapse into one +pass/fail. Instead it asks for the xml reporter and a file: + +``` +reportFile <- tempfile(fileext=".xml") +CoGAPS:::run_catch_unit_tests(reporter="xml", output=reportFile) +``` + +`output=""` (the default) keeps writing to stdout, which is why the plain call +above still behaves the way it always has. The test then parses the file with +`xml2` and turns every `` into its own testthat expectation, so a +failure is reported by name and source location rather than as "expected 0, got +N". + +`catch_test_case_names()` returns the names of every TEST_CASE compiled in. The +test uses it to tell "all C++ tests passed" apart from "no C++ tests were built" +-- with `--disable-cpp-tests`, or on Windows where `Makevars.win` lists no +`cpp_tests` objects, the suite is empty and would otherwise pass vacuously. In +that case the test skips instead, with an explanation. + +``` +length(CoGAPS:::catch_test_case_names()) # 58 as of this writing +``` + +The tags need to be defined in `/src/cpp_tests/[test-name].cpp` in TEST_CASE: + +``` +TEST_CASE("Mt test case -- what is it","[tag1][tag2]") +{ +} ``` -The tags need to be defined in `/src/cpp_tests/[test-name].cpp` ## adding cpp tests to compilation / making changes + To ask for the cpp tests to be compiled, each `test.cpp` needs to be added to the `configure.ac`. Example from `configure.ac`: + ``` # add c++ tests to source list if test "x$cpp_tests_disable" != "xyes" ; then @@ -46,15 +109,17 @@ if test "x$cpp_tests_disable" != "xyes" ; then GAPS_SOURCE_FILES+=" cpp_tests/testVector.o" fi ``` + In the above example, the `testVector.o` is being added and will be compiled (if test compilation is not disabled). -After changes are done, update `configure.ac` by running +After changes are done, update configure from `configure.ac` by running autoconf in terminal: -in terminal: ``` autoconf ``` + in R session: + ``` Rcpp::compileAttributes() ``` @@ -62,6 +127,7 @@ Rcpp::compileAttributes() ## disabling cpp test compilation It may be needed to disable the compilation of tests. There is a specific compilation parameter that controls it. Again, this parameter is set in `configure.ac` (see above). Usage: + ``` #install with tests disabled options(configure.args = list(CoGAPS = "--disable-cpp-tests")) diff --git a/src/cpp_tests/testAtomicDomain.cpp b/src/cpp_tests/testAtomicDomain.cpp new file mode 100644 index 00000000..51edbbfc --- /dev/null +++ b/src/cpp_tests/testAtomicDomain.cpp @@ -0,0 +1,284 @@ +#include +#include +#include "../testthat-tweak.h" +#include "../atomic/AtomicDomain.h" +#include "../math/Random.h" +#include "../utils/GapsPrint.h" + +GapsRandomState randState(123); +GapsRng AtomicRng(&randState); + + + +TEST_CASE("AtomicDomain populate","[atomicdomain][populate]") +{ + + SECTION("Construction") + { + AtomicDomain domain(2); + REQUIRE(domain.size() == 0); + } + SECTION("Populate") + { + AtomicDomain domain(2); + domain.insert((uint64_t)100000,0.01); + //atomic coord and mass + REQUIRE(domain.size() == 1); + domain.insert((uint64_t)200000,0.02); + //atomic coord and mass + REQUIRE(domain.size() == 2); + domain.insert((uint64_t)400000,0.01); + //atomic coord and mass + REQUIRE(domain.size() == 3); + domain.insert((uint64_t)400000,0.05); + //atomic coord and mass + REQUIRE(domain.size() == 3); + } +} + +TEST_CASE("AtomicDomain move","[atomicdomain][move]") +{ + AtomicDomain domain(100); + domain.insert((uint64_t)200000,0.01); + //atomic coord and mass + domain.insert((uint64_t)100000,0.02); + //atomic coord and mass + domain.insert((uint64_t)400000,0.03); + //atomic coord and mass + domain.insert((uint64_t)300000,0.04); + //domain.move(a2p,30000); + REQUIRE(domain.size() == 4); + + SECTION("Test move") + { + domain.move(domain.storedAtom(3),310000); + REQUIRE(domain.storedAtom(3)->pos() == 310000); + auto iter=domain.front_it(); + REQUIRE(iter->first==100000); + REQUIRE(iter->second==1); + iter++; + REQUIRE(iter->first==200000); + REQUIRE(iter->second==0); + iter++; + REQUIRE(iter->first==310000); + REQUIRE(iter->second==3); + iter++; + REQUIRE(iter->first==400000); + REQUIRE(iter->second==2); + } + +} + +TEST_CASE("AtomicDomain structure","[atomicdomain][structure]") +{ + AtomicDomain domain(100); + domain.insert((uint64_t)200000,0.01); + //atomic coord and mass + domain.insert((uint64_t)100000,0.02); + //atomic coord and mass + domain.insert((uint64_t)400000,0.03); + //atomic coord and mass + domain.insert((uint64_t)300000,0.04); + //domain.move(a2p,30000); + REQUIRE(domain.size() == 4); + + SECTION("Test structure") + { + REQUIRE(domain.front()->pos() == 100000); + REQUIRE(domain.front()->index() == 1); + REQUIRE(domain.front()->hasLeft() == false); + REQUIRE(domain.front()->hasRight() == true); + REQUIRE(domain.front()->rightIndex() == 0); + + REQUIRE(domain.storedAtom(0)->pos() == 200000); + REQUIRE(domain.storedAtom(0)->index() == 0); + REQUIRE(domain.storedAtom(0)->hasLeft() == true); + REQUIRE(domain.storedAtom(0)->hasRight() == true); + REQUIRE(domain.storedAtom(0)->leftIndex() == 1); + REQUIRE(domain.storedAtom(0)->rightIndex() == 3); + + REQUIRE(domain.storedAtom(1)->pos() == 100000); + REQUIRE(domain.storedAtom(1)->index() == 1); + REQUIRE(domain.storedAtom(1)->hasLeft() == false); + REQUIRE(domain.storedAtom(1)->hasRight() == true); + //REQUIRE(domain.storedAtom(1)->leftIndex() == 100); + REQUIRE(domain.storedAtom(1)->rightIndex() == 0); + + REQUIRE(domain.storedAtom(2)->pos() == 400000); + REQUIRE(domain.storedAtom(2)->index() == 2); + REQUIRE(domain.storedAtom(2)->hasLeft() == true); + REQUIRE(domain.storedAtom(2)->hasRight() == false); + REQUIRE(domain.storedAtom(2)->leftIndex() == 3); + //REQUIRE(domain.storedAtom(2)->rightIndex() == 100); + + REQUIRE(domain.storedAtom(3)->pos() == 300000); + REQUIRE(domain.storedAtom(3)->index() == 3); + REQUIRE(domain.storedAtom(3)->hasLeft() == true); + REQUIRE(domain.storedAtom(3)->hasRight() == true); + REQUIRE(domain.storedAtom(3)->leftIndex() == 0); + REQUIRE(domain.storedAtom(3)->rightIndex() == 2); + } + SECTION("Test map structure") + { + auto iter=domain.front_it(); + REQUIRE(iter->first==100000); + REQUIRE(iter->second==1); + iter++; + REQUIRE(iter->first==200000); + REQUIRE(iter->second==0); + iter++; + REQUIRE(iter->first==300000); + REQUIRE(iter->second==3); + iter++; + REQUIRE(iter->first==400000); + REQUIRE(iter->second==2); + } + SECTION("Test move") + { + domain.move(domain.storedAtom(3),310000); + REQUIRE(domain.storedAtom(3)->pos() == 310000); + auto iter=domain.front_it(); + REQUIRE(iter->first==100000); + REQUIRE(iter->second==1); + iter++; + REQUIRE(iter->first==200000); + REQUIRE(iter->second==0); + iter++; + REQUIRE(iter->first==310000); + REQUIRE(iter->second==3); + iter++; + REQUIRE(iter->first==400000); + REQUIRE(iter->second==2); + } + +} + + +TEST_CASE("AtomicDomain depopulate","[atomicdomain][depopulate]") { + AtomicDomain domain(100); + domain.insert((uint64_t)200000,0.01); + //atomic coord and mass + domain.insert((uint64_t)100000,0.02); + //atomic coord and mass + domain.insert((uint64_t)400000,0.03); + //atomic coord and mass + domain.insert((uint64_t)300000,0.04); + //domain.move(a2p,30000); + REQUIRE(domain.size() == 4); + SECTION("DePopulate") + { + domain.erase(domain.storedAtom(0)); + + REQUIRE(domain.front()->pos() == 100000); + REQUIRE(domain.front()->index() == 1); + REQUIRE(domain.front()->hasLeft() == false); + REQUIRE(domain.front()->hasRight() == true); + REQUIRE(domain.front()->rightIndex() == 0); + + REQUIRE(domain.storedAtom(0)->pos() == 300000); + REQUIRE(domain.storedAtom(0)->index() == 0); + REQUIRE(domain.storedAtom(0)->hasLeft() == true); + REQUIRE(domain.storedAtom(0)->hasRight() == true); + REQUIRE(domain.storedAtom(0)->leftIndex() == 1); + REQUIRE(domain.storedAtom(0)->rightIndex() == 2); + + REQUIRE(domain.storedAtom(1)->pos() == 100000); + REQUIRE(domain.storedAtom(1)->index() == 1); + REQUIRE(domain.storedAtom(1)->hasLeft() == false); + REQUIRE(domain.storedAtom(1)->hasRight() == true); + //REQUIRE(domain.storedAtom(1)->leftIndex() == 100); + REQUIRE(domain.storedAtom(1)->rightIndex() == 0); + + REQUIRE(domain.storedAtom(2)->pos() == 400000); + REQUIRE(domain.storedAtom(2)->index() == 2); + REQUIRE(domain.storedAtom(2)->hasLeft() == true); + REQUIRE(domain.storedAtom(2)->hasRight() == false); + REQUIRE(domain.storedAtom(2)->leftIndex() == 0); + //REQUIRE(domain.storedAtom(2)->rightIndex() == 100); + } + SECTION("Test map structure") + { + //in new section, we get a new copy of domain + //to be sure, + REQUIRE(domain.size()==4); + //erase + domain.erase(domain.storedAtom(0)); + auto iter=domain.front_it(); + REQUIRE(iter->first==100000); + REQUIRE(iter->second==1); + iter++; + REQUIRE(iter->first==300000); + REQUIRE(iter->second==0); + iter++; + REQUIRE(iter->first==400000); + REQUIRE(iter->second==2); + } +} + +TEST_CASE("AtomicDomain move then erase", "[atomicdomain][movethenerase]") +{ + // Regression test: after move(), the atom's internal map iterator must be + // updated so that a subsequent erase() on the same atom does not use a + // stale iterator (stale iterator caused UB / map corruption). + AtomicDomain domain(100); + domain.insert((uint64_t)200000, 0.01); // storedAtom(0) + domain.insert((uint64_t)100000, 0.02); // storedAtom(1) + domain.insert((uint64_t)400000, 0.03); // storedAtom(2) + domain.insert((uint64_t)300000, 0.04); // storedAtom(3) + REQUIRE(domain.size() == 4); + + Atom *atom = domain.storedAtom(3); // atom at 300000 + REQUIRE(atom->pos() == 300000); + domain.move(atom, 310000); + REQUIRE(atom->pos() == 310000); + + // Erase the moved atom — must use the updated iterator, not the stale one + domain.erase(atom); + REQUIRE(domain.size() == 3); + + // 310000 must be absent; map must contain exactly 100000, 200000, 400000 + auto iter = domain.front_it(); + REQUIRE(iter->first == 100000); + iter++; + REQUIRE(iter->first == 200000); + iter++; + REQUIRE(iter->first == 400000); +} + +TEST_CASE("AtomicDomain randompick","[atomicdomain][randompick]") { + SECTION("RandomAtomChoose") + { + AtomicDomain domainp(2); + domainp.insert((uint64_t)200000,0.02); + //atomic coord and mass + domainp.insert((uint64_t)400000,0.01); + size_t sz=domainp.size(); + Atom * del = domainp.randomAtom(&AtomicRng); + std::cout<<"Picked position "<pos()< -#include "../testthat-tweak.h" -#include "../atomic/ConcurrentAtomicDomain.h" -#include "../utils/GapsPrint.h" - -TEST_CASE("AtomicDomain") -{ - GapsRandomState randState(123); - GapsRng rng(&randState); - - SECTION("Construction") - { - AtomicDomain domain(10); - REQUIRE(domain.size() == 0); - } -} diff --git a/src/cpp_tests/testDenseGibbsSampler.cpp b/src/cpp_tests/testDenseGibbsSampler.cpp index 41b05c94..aa4019d1 100755 --- a/src/cpp_tests/testDenseGibbsSampler.cpp +++ b/src/cpp_tests/testDenseGibbsSampler.cpp @@ -1,12 +1,86 @@ #include #include "../testthat-tweak.h" -#include "../gibbs_sampler/AsynchronousGibbsSampler.h" -#include "../gibbs_sampler/DenseStoragePolicy.h" +#include "../gibbs_sampler/SingleThreadedGibbsSampler.h" +#include "../gibbs_sampler/DenseNormalModel.h" +#include "../data_structures/Matrix.h" -TEST_CASE("Test DenseGibbsSampler") +#include + +// convert R to C++ data type +// it is copied from CoGAPS.cpp not to change the main headers +static Matrix convertRMatrix(const Rcpp::NumericMatrix &rmat) +{ + Matrix mat(rmat.nrow(), rmat.ncol()); + for (unsigned i = 0; i < mat.nRow(); ++i) + { + for (unsigned j = 0; j < mat.nCol(); ++j) + { + mat(i,j) = rmat(i,j); + } + } + return mat; +} + + +//copied from Matrix.cpp, changed: std::ostream instead of Archive +std::ostream& operator<<(std::ostream &ar, const Matrix &mat) +{ + ar << mat.nRow() <<" x "<< mat.nCol() << std::endl; + for (unsigned i = 0; i < mat.nRow(); ++i) { + for (unsigned j = 0; j < mat.nCol(); ++j) + { + ar << mat(i,j) << " "; + } + ar< ASampler(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + Matrix AMM(ASampler.MyMatrix()); + AMM.pad(5); + ASampler.setMatrix(AMM); + SingleThreadedGibbsSampler PSampler(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + Matrix PMM(PSampler.MyMatrix()); + PMM.pad(2); + PSampler.setMatrix(PMM); + ASampler.sync(PSampler); + PSampler.sync(ASampler); + ASampler.extraInitialization(); + //actually, it is AP = A times P + PSampler.extraInitialization(); + //actually, it is AP = A times P + const Matrix & AAP=ASampler.APMatrix(); + const Matrix & PAP=PSampler.APMatrix(); + //just a ref + //print the matrices std::cout< ASampler(data, true, false, params.alphaA, + SingleThreadedGibbsSampler ASampler(data, true, false, params.alphaA, params.maxGibbsMassA, params, &randState); - GibbsSampler PSampler(data, false, false, params.alphaP, + SingleThreadedGibbsSampler PSampler(data, false, false, params.alphaP, params.maxGibbsMassP, params, &randState); - + + REQUIRE(ASampler.chiSq() == 100.f * data.nRow() * data.nCol()); REQUIRE(PSampler.chiSq() == 100.f * data.nRow() * data.nCol()); + double AChiInit=ASampler.chiSq(); + double PChiInit=PSampler.chiSq(); + ASampler.sync(PSampler); PSampler.sync(ASampler); ASampler.extraInitialization(); PSampler.extraInitialization(); - REQUIRE(ASampler.chiSq() == 100.f * data.nRow() * data.nCol()); - REQUIRE(PSampler.chiSq() == 100.f * data.nRow() * data.nCol()); + REQUIRE(ASampler.chiSq() == AChiInit); + REQUIRE(PSampler.chiSq() == PChiInit); #ifdef GAPS_DEBUG REQUIRE(ASampler.internallyConsistent()); REQUIRE(PSampler.internallyConsistent()); #endif + float A_APSumInit=gaps::sum(ASampler.APMatrix()); + float P_APSumInit=gaps::sum(PSampler.APMatrix()); + float A_SumInit=gaps::sum(ASampler.MyMatrix()); + float P_SumInit=gaps::sum(PSampler.MyMatrix()); + + ASampler.update(100); + PSampler.update(100); + + ASampler.sync(PSampler); + PSampler.sync(ASampler); + + ASampler.extraInitialization(); + PSampler.extraInitialization(); + + REQUIRE(ASampler.chiSq() < AChiInit); + REQUIRE(PSampler.chiSq() < PChiInit); + + REQUIRE(gaps::sum(ASampler.APMatrix()) != A_APSumInit); + REQUIRE(gaps::sum(PSampler.APMatrix()) != P_APSumInit); + + REQUIRE(gaps::sum(ASampler.MyMatrix()) != A_SumInit); + REQUIRE(gaps::sum(PSampler.MyMatrix()) != P_SumInit); + } +} + +TEST_CASE("Test DenseGibbsSampler on gist matrix","[densesinglesampler][gistmat]") +{ + + SECTION("Construct from gist matrix") + { + Rcpp::Environment env = Rcpp::Environment::global_env(); + Rcpp::Function load("data"); + //R function is data() + load("GIST"); + Matrix data=convertRMatrix(env["GIST.matrix"]); + + GapsRandomState randState(42); + GapsParameters params(data); + SingleThreadedGibbsSampler ASampler(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler PSampler(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + + double AChiInit=ASampler.chiSq(); + double PChiInit=PSampler.chiSq(); + + ASampler.sync(PSampler); + PSampler.sync(ASampler); + ASampler.extraInitialization(); + PSampler.extraInitialization(); + + REQUIRE(ASampler.chiSq() == AChiInit); + REQUIRE(PSampler.chiSq() == PChiInit); + + #ifdef GAPS_DEBUG + REQUIRE(ASampler.internallyConsistent()); + REQUIRE(PSampler.internallyConsistent()); + #endif + float A_APSumInit=gaps::sum(ASampler.APMatrix()); + float P_APSumInit=gaps::sum(PSampler.APMatrix()); + + float A_SumInit=gaps::sum(ASampler.MyMatrix()); + float P_SumInit=gaps::sum(PSampler.MyMatrix()); + + ASampler.update(100); + PSampler.update(100); + + ASampler.sync(PSampler); + PSampler.sync(ASampler); + + ASampler.extraInitialization(); + PSampler.extraInitialization(); + + REQUIRE(ASampler.chiSq() < AChiInit); + REQUIRE(PSampler.chiSq() < PChiInit); + + REQUIRE(gaps::sum(ASampler.APMatrix()) != A_APSumInit); + REQUIRE(gaps::sum(PSampler.APMatrix()) != P_APSumInit); + + REQUIRE(gaps::sum(ASampler.MyMatrix()) != A_SumInit); + REQUIRE(gaps::sum(PSampler.MyMatrix()) != P_SumInit); + + } } diff --git a/src/cpp_tests/testFileParsers.cpp b/src/cpp_tests/testFileParsers.cpp index ecb32e8b..a396ee23 100755 --- a/src/cpp_tests/testFileParsers.cpp +++ b/src/cpp_tests/testFileParsers.cpp @@ -1,25 +1,21 @@ #include +#include #include "../testthat-tweak.h" -#include "../file_parser/CsvParser.h" -#include "../file_parser/TsvParser.h" +#include "../file_parser/CharacterDelimitedParser.h" #include "../file_parser/MtxParser.h" -#include "../file_parser/GctParser.h" - #include "../data_structures/Matrix.h" -#include - -TEST_CASE("Test Parsers") +TEST_CASE("Test Parsers", "[fileparsers]") { - Rcpp::Environment env = Rcpp::Environment::global_env(); - std::string csvPath = Rcpp::as(env["gistCsvPath"]); - std::string tsvPath = Rcpp::as(env["gistTsvPath"]); - std::string mtxPath = Rcpp::as(env["gistMtxPath"]); - std::string gctPath = Rcpp::as(env["gistGctPath"]); + Rcpp::Function systemfile("system.file"); + std::string csvPath = Rcpp::as(systemfile("extdata", "GIST.csv", Rcpp::Named("package") = "CoGAPS")); + std::string tsvPath = Rcpp::as(systemfile("extdata", "GIST.tsv", Rcpp::Named("package") = "CoGAPS")); + std::string mtxPath = Rcpp::as(systemfile("extdata", "GIST.mtx", Rcpp::Named("package") = "CoGAPS")); + std::string gctPath = Rcpp::as(systemfile("extdata", "GIST.gct", Rcpp::Named("package") = "CoGAPS")); SECTION("Test CsvParser") { - CsvParser p(csvPath); + CharacterDelimitedParser p(csvPath, ','); REQUIRE(p.nRow() == 1363); REQUIRE(p.nCol() == 9); @@ -45,7 +41,7 @@ TEST_CASE("Test Parsers") SECTION("Test TsvParser") { - TsvParser p(tsvPath); + CharacterDelimitedParser p(tsvPath, '\t'); REQUIRE(p.nRow() == 1363); REQUIRE(p.nCol() == 9); @@ -89,7 +85,7 @@ TEST_CASE("Test Parsers") SECTION("Test GctParser") { - GctParser p(gctPath); + CharacterDelimitedParser p(gctPath, '\t', true); REQUIRE(p.nRow() == 1363); REQUIRE(p.nCol() == 9); diff --git a/src/cpp_tests/testHashSets.cpp b/src/cpp_tests/testHashSets.cpp index 8e3e054d..325e8565 100755 --- a/src/cpp_tests/testHashSets.cpp +++ b/src/cpp_tests/testHashSets.cpp @@ -3,8 +3,10 @@ #include "../data_structures/HashSets.h" #include "../math/Random.h" -TEST_CASE("Test HashSets.h - FixedHashSetU32","[hashset][U32]") +TEST_CASE("Test HashSets.h - FixedHashSetU32", "[hashsets][fixedhashsetu32]") { + SECTION("Basic insert/contains/clear") + { GapsRandomState randState(123); FixedHashSetU32 hSet(1000); @@ -25,10 +27,13 @@ TEST_CASE("Test HashSets.h - FixedHashSetU32","[hashset][U32]") REQUIRE(!hSet.contains(u)); REQUIRE(hSet.isEmpty()); } + } // closes SECTION } -TEST_CASE("Test HashSets.h - SmallHashSetU64","[hashset][U64]") +TEST_CASE("Test HashSets.h - SmallHashSetU64", "[hashsets][smallhashsetu64]") { + SECTION("Basic insert/contains/clear") + { GapsRandomState randState(123); SmallHashSetU64 hSet; @@ -49,10 +54,13 @@ TEST_CASE("Test HashSets.h - SmallHashSetU64","[hashset][U64]") REQUIRE(!hSet.contains(u)); REQUIRE(hSet.isEmpty()); } + } // closes SECTION } -TEST_CASE("Test HashSets.h - SmallPairedHashSetU64","[hashset][pairedU64]") +TEST_CASE("Test HashSets.h - SmallPairedHashSetU64", "[hashsets][smallpairedhashsetu64]") { + SECTION("Basic insert/contains/clear") + { SmallPairedHashSetU64 hSet; REQUIRE(hSet.isEmpty()); @@ -67,15 +75,12 @@ TEST_CASE("Test HashSets.h - SmallPairedHashSetU64","[hashset][pairedU64]") hSet.insert(u1, u2); REQUIRE(hSet.contains(u1)); REQUIRE(hSet.contains(u2)); - // [AI-generated] Compute the unsigned distance between two positions without - // assuming which position is larger. uint64_t d = u1 > u2 ? u1 - u2 : u2 - u1; - // [AI-generated] Check an interior point between the two positions, choosing the - // lower endpoint as the start of the interval. REQUIRE(hSet.overlap(u1 > u2 ? u2 + d/2 : u1 + d/2)); } hSet.clear(); - REQUIRE(hSet.isEmpty()); } + } // closes SECTION } + diff --git a/src/cpp_tests/testHybridMatrix.cpp b/src/cpp_tests/testHybridMatrix.cpp index 616347e3..69a92a7f 100755 --- a/src/cpp_tests/testHybridMatrix.cpp +++ b/src/cpp_tests/testHybridMatrix.cpp @@ -5,8 +5,10 @@ #include "../math/VectorMath.h" #include "../math/MatrixMath.h" -TEST_CASE("Test HybridMatrix.h","[hybridmatrix]") +TEST_CASE("Test HybridMatrix", "[hybridmatrix]") { + SECTION("Basic operations") + { HybridMatrix mat(100, 250); REQUIRE(mat.nRow() == 100); REQUIRE(mat.nCol() == 250); @@ -32,4 +34,5 @@ TEST_CASE("Test HybridMatrix.h","[hybridmatrix]") REQUIRE(mat(i,j) == ref(i,j)); } } + } // closes SECTION } diff --git a/src/cpp_tests/testHybridVector.cpp b/src/cpp_tests/testHybridVector.cpp index c8e3d73c..8a40ce43 100755 --- a/src/cpp_tests/testHybridVector.cpp +++ b/src/cpp_tests/testHybridVector.cpp @@ -5,7 +5,7 @@ #include "../math/Math.h" #include "../math/VectorMath.h" -TEST_CASE("Test HybridVector.h","[hybridvector]") +TEST_CASE("Test HybridVector", "[hybridvector]") { GapsRandomState randState(123); diff --git a/src/cpp_tests/testMathHelpers.cpp b/src/cpp_tests/testMathHelpers.cpp new file mode 100644 index 00000000..8ac4648f --- /dev/null +++ b/src/cpp_tests/testMathHelpers.cpp @@ -0,0 +1,136 @@ +#include +#include "../testthat-tweak.h" +#include "../data_structures/Matrix.h" +#include "../data_structures/SparseMatrix.h" +#include "../data_structures/Vector.h" +#include "../math/MatrixMath.h" +#include "../math/VectorMath.h" + +// Helpers that the sampler and GapsStatistics rely on but that had no direct +// coverage: elementSq, dot_diff, mean and sparsity. dot_diff in particular runs +// a SIMD loop, so it is exercised on sizes that are not a multiple of SIMD_INC. + +static std::vector sequentialVector(unsigned n) +{ + std::vector vec; + for (unsigned i = 1; i <= n; ++i) // mimic R indices + { + vec.push_back(i); + } + return vec; +} + +TEST_CASE("gaps::elementSq squares each element","[vectormath][elementsq]") +{ + SECTION("known values") + { + Vector v(4); + v[0] = 0.f; v[1] = 2.f; v[2] = -3.f; v[3] = 1.5f; + Vector sq(gaps::elementSq(v)); + + REQUIRE(sq.size() == v.size()); + REQUIRE(sq[0] == 0.f); + REQUIRE(sq[1] == 4.f); + REQUIRE(sq[2] == 9.f); + REQUIRE(sq[3] == Approx(2.25f)); + // the source vector is not modified + REQUIRE(v[2] == -3.f); + } + + SECTION("empty vector") + { + Vector v(0); + Vector sq(gaps::elementSq(v)); + REQUIRE(sq.size() == 0); + } +} + +TEST_CASE("gaps::dot_diff computes sum(a * (b - c))","[vectormath][dotdiff]") +{ + SECTION("known values") + { + Vector a(3), b(3), c(3); + a[0] = 1.f; a[1] = 2.f; a[2] = 3.f; + b[0] = 4.f; b[1] = 5.f; b[2] = 6.f; + c[0] = 1.f; c[1] = 1.f; c[2] = 1.f; + // 1*(4-1) + 2*(5-1) + 3*(6-1) = 3 + 8 + 15 = 26 + REQUIRE(gaps::dot_diff(a, b, c) == Approx(26.f)); + } + + SECTION("agrees with dot(a, b-c) on a size that is not a SIMD multiple") + { + const unsigned n = 13; + Vector a(n), b(n), c(n), diff(n); + for (unsigned i = 0; i < n; ++i) + { + a[i] = static_cast(i) * 0.5f + 1.f; + b[i] = static_cast(n - i); + c[i] = static_cast(i) * 0.25f; + diff[i] = b[i] - c[i]; + } + REQUIRE(gaps::dot_diff(a, b, c) == Approx(gaps::dot(a, diff))); + } + + SECTION("b == c gives zero") + { + Vector a(5), b(5); + for (unsigned i = 0; i < 5; ++i) + { + a[i] = static_cast(i + 1); + b[i] = static_cast(i * 3); + } + REQUIRE(gaps::dot_diff(a, b, b) == Approx(0.f)); + } +} + +TEST_CASE("gaps::mean and gaps::sparsity on a Matrix","[matrixmath][meansparsity]") +{ + // 2x2 with one zero: sum = 6, mean = 6/4, sparsity = 1 - 3/4 + Matrix mat(2, 2); + mat(0,0) = 1.f; mat(0,1) = 2.f; + mat(1,0) = 3.f; mat(1,1) = 0.f; + + SECTION("mean is the sum over the number of cells") + { + REQUIRE(gaps::sum(mat) == Approx(6.f)); + REQUIRE(gaps::mean(mat) == Approx(1.5f)); + } + + SECTION("sparsity is the fraction of zero cells") + { + REQUIRE(gaps::sparsity(mat) == Approx(0.25f)); + } + + SECTION("an all-zero matrix is fully sparse and has mean 0") + { + Matrix zero(3, 3); + REQUIRE(gaps::mean(zero) == Approx(0.f)); + REQUIRE(gaps::sparsity(zero) == Approx(1.f)); + } + + SECTION("a matrix with no zeros has sparsity 0") + { + Matrix full(2, 2); + full(0,0) = 1.f; full(0,1) = 1.f; + full(1,0) = 1.f; full(1,1) = 1.f; + REQUIRE(gaps::sparsity(full) == Approx(0.f)); + REQUIRE(gaps::mean(full) == Approx(1.f)); + } +} + +TEST_CASE("gaps::sparsity agrees between Matrix and SparseMatrix","[matrixmath][sparsityagree]") +{ + Matrix mat(4, 4); + for (unsigned j = 0; j < 4; ++j) + { + for (unsigned i = 0; i < 4; ++i) + { + // leave the diagonal zero -> 4 zeros out of 16 + mat(i,j) = (i == j) ? 0.f : static_cast(i + j + 1); + } + } + SparseMatrix smat(mat, false, false, sequentialVector(0)); + + REQUIRE(gaps::sparsity(mat) == Approx(0.25f)); + REQUIRE(gaps::sparsity(smat) == Approx(gaps::sparsity(mat))); +} diff --git a/src/cpp_tests/testMatrix.cpp b/src/cpp_tests/testMatrix.cpp index e48b6651..17e8f969 100755 --- a/src/cpp_tests/testMatrix.cpp +++ b/src/cpp_tests/testMatrix.cpp @@ -1,13 +1,15 @@ #include #include "../testthat-tweak.h" #include "../data_structures/Matrix.h" -#include "../file_parser/CsvParser.h" -#include "../file_parser/TsvParser.h" -#include "../file_parser/MtxParser.h" +//#include "../file_parser/MtxParser.h" +//#include "../file_parser/CharacterDelimitedParser.h" +#include "../file_parser/FileParser.h" +#include "../math/Math.h" #include "../math/Random.h" #include "../math/VectorMath.h" #include "../math/MatrixMath.h" + static std::vector sequentialVector(unsigned n) { std::vector vec; @@ -57,7 +59,7 @@ unsigned nc, unsigned nIndices, float sum1, float sum2, float sum3) sequentialVector(nIndices)); } -TEST_CASE("Test Writing/Reading Matrices from File") +TEST_CASE("Test Writing/Reading Matrices from File","[matrix][matrixrw]") { // matrix to use for testing Matrix ref(25, 50); @@ -70,29 +72,27 @@ TEST_CASE("Test Writing/Reading Matrices from File") } // write matrix to file - FileParser::writeToTsv("testMatWrite.tsv", ref); FileParser::writeToCsv("testMatWrite.csv", ref); - FileParser::writeToMtx("testMatWrite.mtx", ref); +// FileParser::writeToMtx("testMatWrite.mtx", ref); // read matrices from file Matrix mat(ref, false, false, sequentialVector(0)); - Matrix matTsv("testMatWrite.tsv", false, false, sequentialVector(0)); Matrix matCsv("testMatWrite.csv", false, false, sequentialVector(0)); - Matrix matMtx("testMatWrite.mtx", false, false, sequentialVector(0)); +// Matrix matMtx("testMatWrite.mtx", false, false, sequentialVector(0)); // delete files - std::remove("testMatWrite.tsv"); std::remove("testMatWrite.csv"); - std::remove("testMatWrite.mtx"); +// std::remove("testMatWrite.mtx"); // test matrices REQUIRE(gaps::sum(mat) == gaps::sum(ref)); - REQUIRE(gaps::sum(matTsv) == gaps::sum(ref)); REQUIRE(gaps::sum(matCsv) == gaps::sum(ref)); - REQUIRE(gaps::sum(matMtx) == gaps::sum(ref)); + //REQUIRE(gaps::sum(matMtx) == gaps::sum(ref)); } -TEST_CASE("Test Matrix.h") + + +TEST_CASE("Test Matrix","[matrix][matrixfull]") { GapsRandomState randState(123); GapsRng rng(&randState); @@ -124,19 +124,87 @@ TEST_CASE("Test Matrix.h") } // write matrix to file - FileParser::writeToTsv("testMatWrite.tsv", ref); FileParser::writeToCsv("testMatWrite.csv", ref); - FileParser::writeToMtx("testMatWrite.mtx", ref); + //FileParser::writeToMtx("testMatWrite.mtx", ref); // test testAllConstructorSituations(ref, 10, 25, 5, 4125.f, 1750.f, 325.f); - testAllConstructorSituations("testMatWrite.tsv", 10, 25, 5, 4125.f, 1750.f, 325.f); testAllConstructorSituations("testMatWrite.csv", 10, 25, 5, 4125.f, 1750.f, 325.f); - testAllConstructorSituations("testMatWrite.mtx", 10, 25, 5, 4125.f, 1750.f, 325.f); + //testAllConstructorSituations("testMatWrite.mtx", 10, 25, 5, 4125.f, 1750.f, 325.f); // delete files - std::remove("testMatWrite.tsv"); std::remove("testMatWrite.csv"); - std::remove("testMatWrite.mtx"); + //std::remove("testMatWrite.mtx"); + } +} + + +TEST_CASE("Test Matrix pad","[matrix][matrixpad]") +{ + + SECTION("pad") + { + Matrix mat(100, 250); + float foam=3; + //here + REQUIRE(!mat.empty()); + REQUIRE(mat.nRow() == 100); + REQUIRE(mat.nCol() == 250); + REQUIRE(gaps::isVectorZero(mat.getCol(42))); + mat.pad(foam); + REQUIRE(mat.nRow()*mat.nCol()*foam == gaps::sum(mat)); + } + +} + +TEST_CASE("Test Matrix assign","[matrix][matrixassign]") +{ + SECTION("assign") + { + Matrix mat(100, 250), cpmat(100,250); + float foam=3; + //here + mat.pad(foam); + cpmat=mat; + REQUIRE(mat.nRow()*mat.nCol()*foam == gaps::sum(cpmat)); } + +} + + +TEST_CASE("Test gap Matrix pmax","[matrix][matrixpmax]") +{ + SECTION("pmax") + { + Matrix mat(100, 250), cpmat(100,250); + float foam=3,minfoam=0.3; + //here + for (unsigned i = 0; i < mat.nRow(); ++i) { + mat(i,i)=foam; + } + REQUIRE(minfoam == gaps::min(foam,minfoam)); + REQUIRE(foam == gaps::max(foam,minfoam)); + REQUIRE(minfoam == std::min(foam,minfoam)); + REQUIRE(foam == std::max(foam,minfoam)); + cpmat=gaps::pmax(mat,1,minfoam); + REQUIRE(minfoam == cpmat(0,1)); + REQUIRE(minfoam == gaps::min(cpmat)); + } +} + +// Regression: gaps::nonZeroMean returned sum/0 = NaN on an all-zero matrix, +// poisoning mLambda in the models. +TEST_CASE("nonZeroMean of an all-zero matrix is 0, not NaN","[matrix][nonzeromean-empty]") +{ + Matrix mat(10, 10); // default-constructed: all zero + REQUIRE(gaps::nonZeroMean(mat) == 0.f); // NaN would fail this comparison +} + +// Regression: gaps::min/max(Matrix) dereferenced getCol(0) on a zero-column matrix. +TEST_CASE("min/max of a zero-column matrix return 0","[matrix][minmax-empty]") +{ + Matrix mat(5, 0); + REQUIRE(mat.nCol() == 0); + REQUIRE(gaps::min(mat) == 0.f); + REQUIRE(gaps::max(mat) == 0.f); } diff --git a/src/cpp_tests/testRandom.cpp b/src/cpp_tests/testRandom.cpp index 6749e811..48d5737e 100755 --- a/src/cpp_tests/testRandom.cpp +++ b/src/cpp_tests/testRandom.cpp @@ -2,47 +2,42 @@ #include "../testthat-tweak.h" #include "../math/Random.h" #include "../math/Math.h" +#include "../utils/GapsPrint.h" + +#include +#include #define TEST_APPROX(x) Approx(x).epsilon(0.001) // this is intended to replicated the random stream that happens when // each proposal is creating a new rng and using it a few times -class EmulatedRng -{ -public: - EmulatedRng(unsigned seed) - : randState(seed), rng(&randState), tickRng(&randState), - remaining(tickRng.uniform32(1, 5)) - {} +GapsRandomState testRandState(42); +GapsRng testRng(&testRandState); - uint64_t uniform64() +TEST_CASE("Random Number Generation -- basic","[randomrng][basic]") +{ + SECTION("Make sure uniform is working") { - advance(); - return rng.uniform64(); + REQUIRE(testRng.uniform64() != testRng.uniform64()); + REQUIRE(testRng.uniform64(0,1000) != testRng.uniform64(0,1000)); + REQUIRE(testRng.uniform64(1000,1000) == 1000); + REQUIRE(testRng.uniform64(0,2)<3); + REQUIRE(testRng.uniform32() != testRng.uniform32()); + REQUIRE(testRng.uniform32(0,100) != testRng.uniform32(0,100)); + REQUIRE(testRng.uniform32(1000,1000) == 1000); } - -private: - - void advance() + SECTION("Make sure uniform is working on size_t") { - --remaining; - if (remaining == 0) - { - rng = GapsRng(&randState); - remaining = tickRng.uniform32(1, 5); - } + size_t zero=0, thou=1000; + REQUIRE(testRng.uniform64(zero,thou) != testRng.uniform64(zero,thou)); + REQUIRE(testRng.uniform64(thou,thou) == thou); } - - GapsRandomState randState; - GapsRng rng; - GapsRng tickRng; - unsigned remaining; -}; +} static void requireSmallError(float in, float out, float est, float tol) { - float denom = gaps::max(std::abs(out), 1.f); + float denom = std::max(std::abs(out), 1.f); if (std::abs(est - out) / denom >= tol) { gaps_printf("input: %f, output: %f, error: %f\n", in, out, @@ -51,7 +46,7 @@ static void requireSmallError(float in, float out, float est, float tol) REQUIRE(std::abs(est - out) / denom < tol); } -TEST_CASE("Test error of q_norm lookup table") +TEST_CASE("Test error of q_norm lookup table","[randomrng][q_norm]") { GapsRandomState randState(123); @@ -68,7 +63,7 @@ TEST_CASE("Test error of q_norm lookup table") } } -TEST_CASE("Test error of p_norm lookup table") +TEST_CASE("Test error of p_norm lookup table","[randomrng][p_norm]") { GapsRandomState randState(123); @@ -85,99 +80,78 @@ TEST_CASE("Test error of p_norm lookup table") } } -#if 0 -TEST_CASE("write random file to use in diehard tests") -{ - Archive ar("random_stream.out", ARCHIVE_WRITE); - - EmulatedRng rng(123); - for (unsigned i = 0; i < 1500000; ++i) - { - ar << rng.uniform64(); - } -} -#endif - -#if 0 -TEST_CASE("Test Random.h - Random Number Generation") +// ported from the removed gaps::random:: global API to the current GapsRng object. +// GapsRng is deterministic for a fixed seed, so these are stable, not flaky. +TEST_CASE("Random.h - RNG distributions","[randomrng][distributions]") { - gaps::random::setSeed(0); + GapsRandomState randState(0); + GapsRng rng(&randState); - SECTION("Make sure uniform01 is working") + SECTION("uniform01 produces varying values") { - REQUIRE(gaps::random::uniform() != gaps::random::uniform()); + REQUIRE(rng.uniform() != rng.uniform()); } - SECTION("Test uniform distribution over unit interval") + SECTION("uniform over the unit interval") { - float min = 1.f, max = 0.f; - float sum = 0.f; - unsigned N = 10000; + float mn = 1.f, mx = 0.f, sum = 0.f; + const unsigned N = 10000; for (unsigned i = 0; i < N; ++i) { - min = gaps::min(gaps::random::uniform(), min); - max = gaps::max(gaps::random::uniform(), max); - sum += gaps::random::uniform(); + float u = rng.uniform(); + mn = u < mn ? u : mn; + mx = u > mx ? u : mx; + sum += u; } - REQUIRE(sum / N == Approx(0.5f).epsilon(0.01f)); - REQUIRE(min >= 0.f); - REQUIRE(min < 0.01f); - REQUIRE(max <= 1.f); - REQUIRE(max > 0.99f); + REQUIRE(sum / N == Approx(0.5f).epsilon(0.02f)); + REQUIRE(mn >= 0.f); + REQUIRE(mn < 0.02f); + REQUIRE(mx <= 1.f); + REQUIRE(mx > 0.98f); } - SECTION("Test uniform distribution over general interval") + SECTION("uniform over a general interval") { - // bounds equal - REQUIRE(gaps::random::uniform(4.3f, 4.3f) == 4.3f); - - // full range possible - float min = 10., max = 0.; + REQUIRE(rng.uniform(4.3f, 4.3f) == 4.3f); + float mn = 10.f, mx = 0.f; for (unsigned i = 0; i < 1000; ++i) { - min = gaps::min(gaps::random::uniform(0.f,10.f), min); - max = gaps::max(gaps::random::uniform(0.f,10.f), max); + float u = rng.uniform(0.f, 10.f); + mn = u < mn ? u : mn; + mx = u > mx ? u : mx; } - REQUIRE(min < 0.1f); - REQUIRE(max > 9.9f); - } - - SECTION("Test uniform distribution over 64 bit integers") - { - // TODO + REQUIRE(mn < 0.2f); + REQUIRE(mx > 9.8f); } - SECTION("Test poisson distribution") + SECTION("poisson mean") { - float total = 0.f; - for (unsigned i = 0; i < 10000; ++i) + double total = 0.; + const unsigned N = 10000; + for (unsigned i = 0; i < N; ++i) { - float num = gaps::random::poisson(4.f); + int num = rng.poisson(4.0); + REQUIRE(num >= 0); total += num; - - REQUIRE((int)num == num); // should be integer - REQUIRE(num >= 0.f); // should be non-negative } - float mean = total / 10000.f; - REQUIRE(mean == Approx(4.f).epsilon(0.025f)); + REQUIRE(total / N == Approx(4.0).epsilon(0.03)); } - SECTION("Test exponential distribution") + SECTION("exponential mean") { - float total = 0.f; - for (unsigned i = 0; i < 10000; ++i) + double total = 0.; + const unsigned N = 10000; + for (unsigned i = 0; i < N; ++i) { - float num = gaps::random::exponential(1.f); + float num = rng.exponential(1.f); + REQUIRE(num >= 0.f); total += num; - - REQUIRE(num >= 0.f); // should be non-negative } - float mean = total / 10000.f; - REQUIRE(mean == Approx(1.f).epsilon(0.025f)); + REQUIRE(total / N == Approx(1.0).epsilon(0.03)); } } -TEST_CASE("Test Random.h - Distribution Calculations") +TEST_CASE("Random.h - distribution calculations","[randomrng][distcalc]") { REQUIRE(gaps::d_gamma(0.5f, 1.f, 1.f) == TEST_APPROX(0.607f)); REQUIRE(gaps::p_gamma(0.5f, 1.f, 1.f) == TEST_APPROX(0.394f)); @@ -186,4 +160,47 @@ TEST_CASE("Test Random.h - Distribution Calculations") REQUIRE(gaps::q_norm(0.5f, 0.f, 1.f) == TEST_APPROX(0.000f)); REQUIRE(gaps::p_norm(0.5f, 0.f, 1.f) == TEST_APPROX(0.692f)); } + +// Tooling harness (not a unit test): writes a long random stream to a file for +// external diehard RNG tests. Left disabled; enable manually when needed. +#if 0 +#include "../utils/Archive.h" + +class EmulatedRng +{ +public: + EmulatedRng(unsigned seed) + : randState(seed), rng(&randState), tickRng(&randState), + remaining(tickRng.uniform32(1, 5)) + {} + uint64_t uniform64() + { + advance(); + return rng.uniform64(); + } +private: + void advance() + { + --remaining; + if (remaining == 0) + { + rng = GapsRng(&randState); + remaining = tickRng.uniform32(1, 5); + } + } + GapsRandomState randState; + GapsRng rng; + GapsRng tickRng; + unsigned remaining; +}; + +TEST_CASE("write random file to use in diehard tests") +{ + Archive ar("random_stream.out", ARCHIVE_WRITE); + EmulatedRng rng(123); + for (unsigned i = 0; i < 1500000; ++i) + { + ar << rng.uniform64(); + } +} #endif diff --git a/src/cpp_tests/testSamplerHighLevel.cpp b/src/cpp_tests/testSamplerHighLevel.cpp index 5f2b76cc..cea47ce8 100644 --- a/src/cpp_tests/testSamplerHighLevel.cpp +++ b/src/cpp_tests/testSamplerHighLevel.cpp @@ -4,7 +4,6 @@ #include "../GapsParameters.h" #include "../math/Random.h" -#include "../gibbs_sampler/AsynchronousGibbsSampler.h" #include "../gibbs_sampler/SingleThreadedGibbsSampler.h" #include "../gibbs_sampler/DenseNormalModel.h" #include "../gibbs_sampler/SparseNormalModel.h" @@ -19,43 +18,76 @@ Matrix getDummyData(unsigned nrow, unsigned ncol) { for (unsigned j = 0; j < data.nCol(); ++j) { - // [AI-generated] Build a simple checkerboard-like test matrix with zeros on - // alternating entries. data(i,j) = i * j % 2 == 1 ? 0.f : static_cast(i * j); } } return data; } -template -template +template class GibbsSampler, class DataModel> GibbsSampler initGibbsSampler() { // initialization parameters Matrix data(getDummyData(25, 50)); GapsParameters params(data); - GapsRandomState randState(params.seed); - return GibbsSampler(data, false, false, 0.01f, 100.f, params, randState); + // GapsRng stores a pointer to GapsRandomState (const GapsRandomState *mRandState), + // so randState must outlive the sampler returned by value. Make it static: one + // instance per template instantiation, alive for the program's lifetime. + // (data/params are copied into the model, only randState is held by pointer.) + static GapsRandomState randState(params.seed); + return GibbsSampler(data, false, false, 0.01f, 100.f, params, &randState); } -TEST_CASE("Sampler Construction") +TEST_CASE("Sampler Construction", "[samplerhighlevel][construction]") { + SECTION("Build all sampler variants with default uncertainty") + { // construct samplers using default uncertainty INIT_SAMPLER(sampler1, SingleThreadedGibbsSampler, DenseNormalModel); INIT_SAMPLER(sampler2, SingleThreadedGibbsSampler, SparseNormalModel); - INIT_SAMPLER(sampler3, AsynchronousGibbsSampler, DenseNormalModel); - INIT_SAMPLER(sampler4, AsynchronousGibbsSampler, SparseNormalModel); REQUIRE(sampler1.dataSparsity() == sampler2.dataSparsity()); - REQUIRE(sampler2.dataSparsity() == sampler3.dataSparsity()); - REQUIRE(sampler3.dataSparsity() == sampler4.dataSparsity()); + } // closes SECTION } -TEST_CASE("Sampler Update") +// construct an A/P sampler pair for the given DataModel, run sampling, and +// require the fit (chiSq) to improve -- exercises the full update() path +template +static void requireUpdateDecreasesChiSq() { - // construct samplers using default uncertainty - INIT_SAMPLER(sampler1, SingleThreadedGibbsSampler, DenseNormalModel); - INIT_SAMPLER(sampler2, SingleThreadedGibbsSampler, SparseNormalModel); - INIT_SAMPLER(sampler3, AsynchronousGibbsSampler, DenseNormalModel); - INIT_SAMPLER(sampler4, AsynchronousGibbsSampler, SparseNormalModel); + Matrix data(getDummyData(25, 50)); + GapsRandomState randState(42); + GapsParameters params(data); + SingleThreadedGibbsSampler A(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler P(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + + // chiSq() uses the other matrix, which is only valid after sync() + A.sync(P); P.sync(A); + A.extraInitialization(); P.extraInitialization(); + + double AChiInit = A.chiSq(); + double PChiInit = P.chiSq(); + + // interleave update+sync as the algorithm does (sparse rebuilds its lookup + // tables in sync(), so each sampler must sync after the other matrix changes) + A.update(100); P.sync(A); + P.update(100); A.sync(P); + A.extraInitialization(); P.extraInitialization(); + + REQUIRE(A.chiSq() < AChiInit); + REQUIRE(P.chiSq() < PChiInit); +} + +TEST_CASE("Sampler Update", "[samplerhighlevel][update]") +{ + SECTION("update() improves the fit -- dense model") + { + requireUpdateDecreasesChiSq(); + } + SECTION("update() improves the fit -- sparse model") + { + requireUpdateDecreasesChiSq(); + } } diff --git a/src/cpp_tests/testSerialization.cpp b/src/cpp_tests/testSerialization.cpp old mode 100755 new mode 100644 index dc0f9b8e..63f2e889 --- a/src/cpp_tests/testSerialization.cpp +++ b/src/cpp_tests/testSerialization.cpp @@ -2,14 +2,28 @@ #include "../testthat-tweak.h" #include "../utils/Archive.h" #include "../data_structures/Matrix.h" +#include "../data_structures/HybridVector.h" +#include "../data_structures/SparseVector.h" +#include "../data_structures/HybridMatrix.h" +#include "../data_structures/SparseMatrix.h" #include "../math/Random.h" +#include "../math/MatrixMath.h" #include "../atomic/AtomicDomain.h" -#include "../atomic/ProposalQueue.h" +#include "../GapsParameters.h" +#include "../GapsStatistics.h" +#include "../gibbs_sampler/SingleThreadedGibbsSampler.h" +#include "../gibbs_sampler/DenseNormalModel.h" + +#include +#include +#include // put Archive in it's own scope so it gets destructed (file stream closed) -TEST_CASE("Reading/Writing to an Archive") +TEST_CASE("Reading/Writing to an Archive", "[serialization][archive]") { + SECTION("Write an integer and read it back") + { { Archive ar1("test_ar.temp", ARCHIVE_WRITE); ar1 << 3; @@ -24,10 +38,13 @@ TEST_CASE("Reading/Writing to an Archive") // cleanup directory std::remove("test_ar.temp"); + } // closes SECTION } -TEST_CASE("Serialization of primitive types") +TEST_CASE("Serialization of primitive types", "[serialization][primitives]") { + SECTION("Round-trip read/write of all primitive types") + { // test values unsigned u_read = 0, u_write = 456; uint32_t u32_read = 0, u32_write = 512; @@ -68,10 +85,13 @@ TEST_CASE("Serialization of primitive types") // cleanup directory std::remove("test_ar.temp"); + } // closes SECTION } -TEST_CASE("Vector Serialization") +TEST_CASE("Vector Serialization", "[serialization][vector]") { + SECTION("Round-trip read/write of Vector") + { GapsRandomState randState(123); GapsRng rng(&randState); @@ -101,20 +121,54 @@ TEST_CASE("Vector Serialization") // cleanup directory std::remove("test_ar.temp"); + } // closes SECTION } -TEST_CASE("HybridVector Serialization") +TEST_CASE("HybridVector Serialization", "[serialization][hybridvector]") { + GapsRandomState randState(123); + GapsRng rng(&randState); + + std::vector in_v; + for (unsigned n = 0; n < 100; ++n) + in_v.push_back(n % 3 == 0 ? 0.f : rng.uniform(0.5f, 2.f)); + HybridVector vecWrite(in_v), vecRead(100); + + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << vecWrite; } + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> vecRead; } + REQUIRE(vecRead.size() == vecWrite.size()); + for (unsigned i = 0; i < vecWrite.size(); ++i) + REQUIRE(vecRead[i] == vecWrite[i]); + + std::remove("test_ar.temp"); } -TEST_CASE("SparseVector Serialization") +TEST_CASE("SparseVector Serialization", "[serialization][sparsevector]") { + GapsRandomState randState(123); + GapsRng rng(&randState); + + std::vector in_v; + for (unsigned n = 0; n < 100; ++n) + in_v.push_back(n % 4 == 0 ? rng.uniform(0.5f, 2.f) : 0.f); + SparseVector vecWrite(in_v), vecRead(100); + + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << vecWrite; } + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> vecRead; } + + REQUIRE(vecRead.size() == vecWrite.size()); + Vector denseWrite(vecWrite.getDense()), denseRead(vecRead.getDense()); + for (unsigned i = 0; i < denseWrite.size(); ++i) + REQUIRE(denseRead[i] == denseWrite[i]); + std::remove("test_ar.temp"); } -TEST_CASE("Matrix Serialization") +TEST_CASE("Matrix Serialization", "[serialization][matrix]") { + SECTION("Round-trip read/write of Matrix") + { GapsRandomState randState(123); GapsRng rng(&randState); @@ -151,20 +205,67 @@ TEST_CASE("Matrix Serialization") // cleanup directory std::remove("test_ar.temp"); + } // closes SECTION } -TEST_CASE("HybridMatrix Serialization") +TEST_CASE("HybridMatrix Serialization", "[serialization][hybridmatrix]") { + GapsRandomState randState(123); + GapsRng rng(&randState); + + HybridMatrix matWrite(20, 12), matRead(20, 12); + for (unsigned i = 0; i < 20; ++i) + for (unsigned j = 0; j < 12; ++j) + matWrite.set(i, j, (i + j) % 3 == 0 ? 0.f : rng.uniform(0.5f, 2.f)); + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << matWrite; } + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> matRead; } + + REQUIRE(matRead.nRow() == matWrite.nRow()); + REQUIRE(matRead.nCol() == matWrite.nCol()); + for (unsigned i = 0; i < 20; ++i) + for (unsigned j = 0; j < 12; ++j) + REQUIRE(matRead(i,j) == matWrite(i,j)); + + std::remove("test_ar.temp"); } -TEST_CASE("SparseMatrix Serialization") +TEST_CASE("SparseMatrix Serialization", "[serialization][sparsematrix]") { + GapsRandomState randState(123); + GapsRng rng(&randState); + + Matrix dense(30, 10); + for (unsigned i = 0; i < 30; ++i) + for (unsigned j = 0; j < 10; ++j) + dense(i,j) = (i % 3 == 0) ? rng.uniform(0.5f, 2.f) : 0.f; + SparseMatrix matWrite(dense, false, false, std::vector()); + // build the read target from an all-zero matrix so its columns start empty: + // the round-trip must repopulate them (catches the SparseVector count bug) + Matrix zeros(30, 10); + SparseMatrix matRead(zeros, false, false, std::vector()); + + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << matWrite; } + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> matRead; } + REQUIRE(matRead.nRow() == matWrite.nRow()); + REQUIRE(matRead.nCol() == matWrite.nCol()); + for (unsigned j = 0; j < matWrite.nCol(); ++j) + { + Vector colWrite(matWrite.getCol(j).getDense()); + Vector colRead(matRead.getCol(j).getDense()); + REQUIRE(colRead.size() == colWrite.size()); + for (unsigned i = 0; i < colWrite.size(); ++i) + REQUIRE(colRead[i] == colWrite[i]); + } + + std::remove("test_ar.temp"); } -TEST_CASE("Random Generator Serialization") +TEST_CASE("Random Generator Serialization", "[serialization][random]") { + SECTION("Round-trip read/write of GapsRandomState") + { std::vector randSequence; GapsRandomState randStateWrite(123); @@ -206,245 +307,165 @@ TEST_CASE("Random Generator Serialization") // cleanup directory std::remove("test_ar.temp"); + } // closes SECTION } -TEST_CASE("GibbsSampler Serialization") -{ -#if 0 - Rcpp::Environment env = Rcpp::Environment::global_env(); - std::string csvPath = Rcpp::as(env["gistCsvPath"]); - - GibbsSampler Asampler(csvPath, false, 7, false, std::vector()); - GibbsSampler Psampler(csvPath, true, 7, false, std::vector()); - Asampler.sync(Psampler); - Psampler.sync(Asampler); - - Asampler.update(10000, 1); - - Archive arWrite("test_ar.temp", ARCHIVE_WRITE); - arWrite << Asampler; - arWrite.close(); +// (the old CSV-based "GibbsSampler Serialization" test used the removed monolithic +// GibbsSampler API; it is superseded by [serialization][gibbssampler-roundtrip] +// below, which round-trips SingleThreadedGibbsSampler with the current API.) - GibbsSampler savedAsampler(csvPath, false, 7, false, std::vector()); - Archive arRead("test_ar.temp", ARCHIVE_READ); - arRead >> savedAsampler; - arRead.close(); - - // cleanup directory - std::remove("test_ar.temp"); -#endif -} - -TEST_CASE("GapsParameters Serialization") +TEST_CASE("GapsParameters Serialization", "[serialization][gapsparameters]") { + Matrix data(40, 15); + GapsParameters pWrite(data); + pWrite.seed = 777; + pWrite.nGenes = 40; + pWrite.nSamples = 15; + pWrite.nPatterns = 5; + pWrite.nIterations = 321; + pWrite.alphaA = 0.02f; + pWrite.alphaP = 0.03f; + pWrite.maxGibbsMassA = 50.f; + pWrite.maxGibbsMassP = 60.f; + pWrite.useSparseOptimization = true; + pWrite.checkpointInterval = 42; + + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << pWrite; } + GapsParameters pRead(data); + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> pRead; } + + // these are exactly the fields the serialization operators read/write; a + // mismatch (or a << / >> field-order slip) fails here + REQUIRE(pRead.seed == pWrite.seed); + REQUIRE(pRead.nGenes == pWrite.nGenes); + REQUIRE(pRead.nSamples == pWrite.nSamples); + REQUIRE(pRead.nPatterns == pWrite.nPatterns); + REQUIRE(pRead.nIterations == pWrite.nIterations); + REQUIRE(pRead.alphaA == pWrite.alphaA); + REQUIRE(pRead.alphaP == pWrite.alphaP); + REQUIRE(pRead.maxGibbsMassA == pWrite.maxGibbsMassA); + REQUIRE(pRead.maxGibbsMassP == pWrite.maxGibbsMassP); + REQUIRE(pRead.useSparseOptimization == pWrite.useSparseOptimization); + REQUIRE(pRead.checkpointInterval == pWrite.checkpointInterval); + std::remove("test_ar.temp"); } -TEST_CASE("GapsStatistics Serialization") +TEST_CASE("GapsStatistics Serialization", "[serialization][gapsstatistics]") { + // build two dense samplers with real state so the statistics matrices are + // non-trivial, then round-trip the accumulated statistics + Matrix data(12, 8); + data.pad(15.f); + GapsRandomState randState(42); + GapsParameters params(data); + params.nPatterns = 3; + + SingleThreadedGibbsSampler A(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler P(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + A.sync(P); P.sync(A); + A.extraInitialization(); P.extraInitialization(); + A.update(300); P.update(300); + A.sync(P); P.sync(A); + + GapsStatistics statsWrite(data.nRow(), data.nCol(), params.nPatterns); + for (unsigned i = 0; i < 5; ++i) statsWrite.update(A, P); + Matrix ameanW(statsWrite.Amean()), pmeanW(statsWrite.Pmean()); + REQUIRE(gaps::sum(ameanW) > 0.f); // there is real state to serialize + + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << statsWrite; } + GapsStatistics statsRead(data.nRow(), data.nCol(), params.nPatterns); + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> statsRead; } + + Matrix ameanR(statsRead.Amean()), pmeanR(statsRead.Pmean()); + REQUIRE(ameanR.nRow() == ameanW.nRow()); + REQUIRE(ameanR.nCol() == ameanW.nCol()); + for (unsigned i = 0; i < ameanW.nRow(); ++i) + for (unsigned j = 0; j < ameanW.nCol(); ++j) + REQUIRE(ameanR(i,j) == ameanW(i,j)); + for (unsigned i = 0; i < pmeanW.nRow(); ++i) + for (unsigned j = 0; j < pmeanW.nCol(); ++j) + REQUIRE(pmeanR(i,j) == pmeanW(i,j)); + std::remove("test_ar.temp"); } -#if 0 -TEST_CASE("AtomicDomain Serialization") +TEST_CASE("AtomicDomain Serialization", "[serialization][atomicdomain]") { GapsRandomState randState(123); GapsRng rng(&randState); - - AtomicDomain domainWrite(100000); + AtomicDomain domainWrite(100000); for (unsigned i = 0; i < 1000; ++i) { domainWrite.insert(rng.uniform64(), rng.uniform(0.f, 100.f)); } - { - Archive arWrite("test_ar.temp", ARCHIVE_WRITE); - arWrite << domainWrite; - } - + { Archive ar("test_ar.temp", ARCHIVE_WRITE); ar << domainWrite; } AtomicDomain domainRead(1); - - { - Archive arRead("test_ar.temp", ARCHIVE_READ); - arRead >> domainRead; - } + { Archive ar("test_ar.temp", ARCHIVE_READ); ar >> domainRead; } - REQUIRE(domainWrite.front()->pos == domainRead.front()->pos); - REQUIRE(domainWrite.front()->mass == domainRead.front()->mass); - REQUIRE(domainWrite.size() == domainRead.size()); - REQUIRE(domainWrite.mDomainLength == domainRead.mDomainLength); + REQUIRE(domainRead.size() == domainWrite.size()); + REQUIRE(domainRead.DomainLength() == domainWrite.DomainLength()); + REQUIRE(domainRead.front()->pos() == domainWrite.front()->pos()); + REQUIRE(domainRead.front()->mass() == domainWrite.front()->mass()); - for (unsigned i = 0; i < domainWrite.size(); ++i) + // compare every atom as a (pos, mass) set (robust to storage order) + std::vector > want, got; + for (uint64_t i = 0; i < domainWrite.size(); ++i) { - REQUIRE(domainWrite.mAtoms[i]->pos == domainRead.mAtoms[i]->pos); - REQUIRE(domainWrite.mAtoms[i]->mass == domainRead.mAtoms[i]->mass); + want.push_back(std::make_pair(domainWrite.storedAtom(i)->pos(), + domainWrite.storedAtom(i)->mass())); + got.push_back(std::make_pair(domainRead.storedAtom(i)->pos(), + domainRead.storedAtom(i)->mass())); } + std::sort(want.begin(), want.end()); + std::sort(got.begin(), got.end()); + REQUIRE(want == got); - // cleanup directory std::remove("test_ar.temp"); } -#endif -TEST_CASE("ProposalQueue Serialization") +// Regression: SingleThreadedGibbsSampler's operator>> read the DataModel with >> +// but then chained << (write) for mDomain/mNumBins/.../mAlpha, so a checkpoint +// restore did NOT reload the atomic domain (a resumed run diverged from a fresh +// one). Fixed to use >>. This round-trip catches it: nAtoms() of the restored +// sampler must match the saved one. +TEST_CASE("GibbsSampler serialization round-trip","[serialization][gibbssampler-roundtrip]") { - const unsigned nGenes = 10000; - const unsigned nPatterns = 100; - const unsigned nIterations = 1000; - - GapsRandomState randStateWrite(123); - AtomicDomain domainWrite(nGenes * nPatterns); - ProposalQueue queueWrite(nGenes, nPatterns, &randStateWrite); - queueWrite.setAlpha(0.01f); - queueWrite.setLambda(0.01f); - - for (unsigned i = 0; i < nIterations; ++i) - { - queueWrite.populate(domainWrite, nIterations); - for (unsigned j = 0; j < queueWrite.size(); ++j) - { - switch (queueWrite[j].type) - { - case 'B': - queueWrite.acceptBirth(); - queueWrite[j].atom1->mass = 3.f; - break; - case 'D': - queueWrite.acceptDeath(); - domainWrite.erase(queueWrite[j].atom1->pos); - break; - case 'M': - queueWrite[j].atom1->pos = queueWrite[j].pos; - break; - case 'E': - float mass1 = queueWrite[j].atom1->mass; - queueWrite[j].atom1->mass = queueWrite[j].atom2->mass; - queueWrite[j].atom2->mass = mass1; - break; - } - } - queueWrite.clear(); - } + Matrix data(12, 8); + data.pad(15.f); + GapsRandomState randState(42); + GapsParameters params(data); + params.nPatterns = 3; + + SingleThreadedGibbsSampler A(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler P(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + A.sync(P); P.sync(A); + A.extraInitialization(); P.extraInitialization(); + A.update(500); + REQUIRE(A.nAtoms() > 0); // there is domain state worth serializing { Archive arWrite("test_ar.temp", ARCHIVE_WRITE); - arWrite << randStateWrite << queueWrite << domainWrite; + arWrite << A; } - GapsRandomState randStateRead(456); - AtomicDomain domainRead(nGenes * nPatterns); - ProposalQueue queueRead(nGenes, nPatterns, &randStateRead); - queueRead.setAlpha(100.f); - queueRead.setLambda(100.f); - + SingleThreadedGibbsSampler Aread(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + REQUIRE(Aread.nAtoms() == 0); // freshly constructed: empty domain { Archive arRead("test_ar.temp", ARCHIVE_READ); - arRead >> randStateRead >> queueRead >> domainRead; - } - - GapsRng rngWriteTest(&randStateWrite); - GapsRng rngReadTest(&randStateRead); - - REQUIRE(rngWriteTest.uniform() == rngReadTest.uniform()); - REQUIRE(rngWriteTest.uniform() == rngReadTest.uniform()); - REQUIRE(domainWrite.size() == domainRead.size()); - REQUIRE(queueWrite.size() == 0); - REQUIRE(queueRead.size() == 0); - -#if 0 - REQUIRE(queueWrite.mRng.uniform() == queueRead.mRng.uniform()); - REQUIRE(queueWrite.mRng.uniform() == queueRead.mRng.uniform()); - REQUIRE(queueWrite.mRng.uniform() == queueRead.mRng.uniform()); - REQUIRE(queueWrite.mRng.uniform() == queueRead.mRng.uniform()); - REQUIRE(queueWrite.mMinAtoms == queueRead.mMinAtoms); - REQUIRE(queueWrite.mMaxAtoms == queueRead.mMaxAtoms); - REQUIRE(queueWrite.mBinLength == queueRead.mBinLength); - REQUIRE(queueWrite.mNumCols == queueRead.mNumCols); - REQUIRE(queueWrite.mAlpha == queueRead.mAlpha); - REQUIRE(queueWrite.mDomainLength == queueRead.mDomainLength); - REQUIRE(queueWrite.mNumBins == queueRead.mNumBins); - REQUIRE(queueWrite.mUseCachedRng == queueRead.mUseCachedRng); - for (unsigned i = 0; i < domainWrite.size(); ++i) - { - REQUIRE(domainWrite.mAtoms[i]->pos == domainRead.mAtoms[i]->pos); - REQUIRE(domainWrite.mAtoms[i]->mass == domainRead.mAtoms[i]->mass); + arRead >> Aread; } -#endif - - for (unsigned i = 0; i < nIterations; ++i) - { - queueWrite.populate(domainWrite, nIterations); - queueRead.populate(domainRead, nIterations); - REQUIRE(queueWrite.size() == queueRead.size()); - - for (unsigned j = 0; j < queueWrite.size(); ++j) - { - REQUIRE(queueWrite[j].type == queueRead[j].type); - REQUIRE(queueWrite[j].pos == queueRead[j].pos); - REQUIRE(queueWrite[j].r1 == queueRead[j].r1); - REQUIRE(queueWrite[j].c1 == queueRead[j].c1); - REQUIRE(queueWrite[j].r2 == queueRead[j].r2); - REQUIRE(queueWrite[j].c2 == queueRead[j].c2); - - if (queueWrite[j].atom1 == NULL) - { - bool b = queueRead[j].atom1 == NULL; - REQUIRE(b); // prevent pointer comparison warning in catch.h - } - else - { - REQUIRE(queueWrite[j].atom1->pos == queueRead[j].atom1->pos); - REQUIRE(queueWrite[j].atom1->mass == queueRead[j].atom1->mass); - } - - if (queueWrite[j].atom2 == NULL) - { - bool b = queueRead[j].atom2 == NULL; - REQUIRE(b); // prevent pointer comparison warning in catch.h - } - else - { - REQUIRE(queueWrite[j].atom2->pos == queueRead[j].atom2->pos); - REQUIRE(queueWrite[j].atom2->mass == queueRead[j].atom2->mass); - } - - // process proposal - switch (queueWrite[j].type) - { - case 'B': - queueWrite.acceptBirth(); - queueRead.acceptBirth(); - queueWrite[j].atom1->mass = 3.f; - queueRead[j].atom1->mass = 3.f; - break; - case 'D': - queueWrite.acceptDeath(); - queueRead.acceptDeath(); - domainWrite.erase(queueWrite[j].atom1->pos); - domainRead.erase(queueRead[j].atom1->pos); - break; - case 'M': - queueWrite[j].atom1->pos = queueWrite[j].pos; - queueRead[j].atom1->pos = queueRead[j].pos; - break; - case 'E': - float mass1w = queueWrite[j].atom1->mass; - queueWrite[j].atom1->mass = queueWrite[j].atom2->mass; - queueWrite[j].atom2->mass = mass1w; - - float mass1r = queueRead[j].atom1->mass; - queueRead[j].atom1->mass = queueRead[j].atom2->mass; - queueRead[j].atom2->mass = mass1r; - break; - } - } - queueWrite.clear(); - queueRead.clear(); - } + // with the << bug the domain was never restored, so nAtoms() stayed 0 + REQUIRE(Aread.nAtoms() == A.nAtoms()); + REQUIRE(gaps::sum(Aread.MyMatrix()) == gaps::sum(A.MyMatrix())); - // cleanup directory std::remove("test_ar.temp"); } - - diff --git a/src/cpp_tests/testSparseGibbsSampler.cpp b/src/cpp_tests/testSparseGibbsSampler.cpp index 086c5f8b..f7434839 100755 --- a/src/cpp_tests/testSparseGibbsSampler.cpp +++ b/src/cpp_tests/testSparseGibbsSampler.cpp @@ -1,12 +1,53 @@ #include #include "../testthat-tweak.h" -#include "../gibbs_sampler/AsynchronousGibbsSampler.h" -#include "../gibbs_sampler/DenseStoragePolicy.h" -#include "../gibbs_sampler/SparseStoragePolicy.h" +#include "../gibbs_sampler/SingleThreadedGibbsSampler.h" +#include "../gibbs_sampler/SparseNormalModel.h" +#include "../gibbs_sampler/DenseNormalModel.h" +#include "../gibbs_sampler/AlphaParameters.h" +#include "../math/Math.h" +#include "../math/MatrixMath.h" + +#include #define TEST_APPROX(x) Approx(x).epsilon(0.001f) -TEST_CASE("Test SparseGibbsSampler") +// Test-only subclass that exposes the protected alphaParameters()/mMatrix of the +// DataModel, so the dense-vs-sparse consistency can be checked without changing +// production access levels. +template +class ExposedSampler : public SingleThreadedGibbsSampler +{ +public: + template + ExposedSampler(const DataType &data, bool transpose, bool subsetRows, + float alpha, float maxGibbsMass, const GapsParameters ¶ms, + GapsRandomState *randState) + : SingleThreadedGibbsSampler(data, transpose, subsetRows, alpha, + maxGibbsMass, params, randState) {} + + AlphaParameters a1(unsigned r, unsigned c) + { return this->alphaParameters(r, c); } + AlphaParameters a2(unsigned r1, unsigned c1, unsigned r2, unsigned c2) + { return this->alphaParameters(r1, c1, r2, c2); } + AlphaParameters aC(unsigned r, unsigned c, float ch) + { return this->alphaParametersWithChange(r, c, ch); } + float matrixSum() const { return gaps::sum(this->mMatrix); } +}; + +static void requireAlphaEqual(AlphaParameters sa, AlphaParameters da) +{ + REQUIRE(sa.s >= 0.f); + REQUIRE(da.s >= 0.f); + if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) + { + REQUIRE(sa.s <= gaps::epsilon); + REQUIRE(da.s <= gaps::epsilon); + } + REQUIRE(sa.s == TEST_APPROX(da.s)); + REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); +} + +TEST_CASE("Test SparseGibbsSampler", "[sparsegibbs]") { SECTION("Construct from data matrix") { @@ -21,11 +62,11 @@ TEST_CASE("Test SparseGibbsSampler") GapsRandomState randState(123); GapsParameters params(data); - GibbsSampler ASampler(data, true, false, params.alphaA, + SingleThreadedGibbsSampler ASampler(data, true, false, params.alphaA, params.maxGibbsMassA, params, &randState); - GibbsSampler PSampler(data, false, false, params.alphaP, + SingleThreadedGibbsSampler PSampler(data, false, false, params.alphaP, params.maxGibbsMassP, params, &randState); - + ASampler.sync(PSampler); PSampler.sync(ASampler); @@ -38,7 +79,72 @@ TEST_CASE("Test SparseGibbsSampler") #endif } -#if 0 + SECTION("Update decreases chiSq") + { + Matrix data(25, 50); + for (unsigned i = 0; i < data.nRow(); ++i) + for (unsigned j = 0; j < data.nCol(); ++j) + data(i,j) = i + j + 1.f; + + GapsRandomState randState(42); + GapsParameters params(data); + SingleThreadedGibbsSampler ASampler(data, true, false, params.alphaA, + params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler PSampler(data, false, false, params.alphaP, + params.maxGibbsMassP, params, &randState); + + // chiSq() uses mOtherMatrix, which is only valid after sync() + ASampler.sync(PSampler); + PSampler.sync(ASampler); + ASampler.extraInitialization(); + PSampler.extraInitialization(); + + double AChiInit = ASampler.chiSq(); + double PChiInit = PSampler.chiSq(); + + // interleave update+sync as the algorithm does: each sampler re-syncs to + // the other's updated matrix before updating (sparse rebuilds its lookup + // tables in sync(), so it must sync after the other matrix changes) + ASampler.update(100); + PSampler.sync(ASampler); + PSampler.update(100); + ASampler.sync(PSampler); + ASampler.extraInitialization(); + PSampler.extraInitialization(); + + // sampling improved the fit + REQUIRE(ASampler.chiSq() < AChiInit); + REQUIRE(PSampler.chiSq() < PChiInit); + } + + SECTION("chiSq() before sync() does not crash and matches dense (regression)") + { + // Regression for issue #18: SparseNormalModel::chiSq() dereferenced the + // NULL mOtherMatrix when called before sync(), segfaulting -- whereas + // DenseNormalModel::chiSq() is safe in the same (un-synced) state. Both + // must return the "no fit" chiSq (A*P treated as 0), matching each other. + Matrix data(25, 50); + for (unsigned i = 0; i < data.nRow(); ++i) + for (unsigned j = 0; j < data.nCol(); ++j) + data(i,j) = i + j + 1.f; // all non-zero, all >= 1 -> S = 0.1*D + + GapsRandomState randState(123); + GapsParameters params(data); + SingleThreadedGibbsSampler sparse(data, true, false, + params.alphaA, params.maxGibbsMassA, params, &randState); + SingleThreadedGibbsSampler dense(data, true, false, + params.alphaA, params.maxGibbsMassA, params, &randState); + + // no sync() -> mOtherMatrix is NULL. Must not crash. + float sparseChi = sparse.chiSq(); + float denseChi = dense.chiSq(); + + REQUIRE(std::isfinite(sparseChi)); + // no fit: (D-0)^2/S^2 with S = 0.1*D is 100 per entry, over nRow*nCol entries + REQUIRE(sparseChi == 100.f * data.nRow() * data.nCol()); + REQUIRE(sparseChi == denseChi); + } + SECTION("Test consistency between alpha parameters calculations") { // create the "data" @@ -49,208 +155,99 @@ TEST_CASE("Test SparseGibbsSampler") { for (unsigned j = 0; j < data.nCol(); ++j) { - // [AI-generated] Generate sparse test data by randomly zeroing about half of - // the entries. - data(i,j) = rng.uniform32(1,14) * (rng.uniform() < 0.5f ? 0.f : 1.f); + // continuous values include 0 < D < 1 (exercise the S floor) and + // D > 1 (S = factor*D regime), with ~50% zeros + data(i,j) = rng.uniform(0.f, 3.f) * (rng.uniform() < 0.5f ? 0.f : 1.f); } } - // create pair of sparse gibbs samplers GapsParameters params(data); - GibbsSampler sparse_ASampler(data, true, false, params.alphaA, + + // pair of sparse and pair of dense samplers over the same data + ExposedSampler sparse_A(data, true, false, params.alphaA, params.maxGibbsMassA, params, &randState); - GibbsSampler sparse_PSampler(data, false, false, params.alphaP, + ExposedSampler sparse_P(data, false, false, params.alphaP, params.maxGibbsMassP, params, &randState); - sparse_ASampler.sync(sparse_PSampler); - sparse_PSampler.sync(sparse_ASampler); + sparse_A.sync(sparse_P); + sparse_P.sync(sparse_A); - // create pair of dense gibbs samplers - GibbsSampler dense_ASampler(data, true, false, params.alphaA, + ExposedSampler dense_A(data, true, false, params.alphaA, params.maxGibbsMassA, params, &randState); - GibbsSampler dense_PSampler(data, false, false, params.alphaP, + ExposedSampler dense_P(data, false, false, params.alphaP, params.maxGibbsMassP, params, &randState); - dense_ASampler.sync(dense_PSampler); - dense_PSampler.sync(dense_ASampler); + dense_A.sync(dense_P); + dense_P.sync(dense_A); - // set the A and P matrix to the same thing + // set the A matrix (genes x patterns) to the same thing on both samplers + Matrix AMat(data.nRow(), params.nPatterns); for (unsigned i = 0; i < data.nRow(); ++i) - { for (unsigned k = 0; k < params.nPatterns; ++k) - { - // [AI-generated] Populate matching dense/sparse test matrices with mostly - // nonzero values and occasional zeros. - float val = rng.uniform(0.f, 10.f) * (rng.uniform() < 0.2f ? 0.f : 1.f); - dense_ASampler.mMatrix(i,k) = val; - sparse_ASampler.mMatrix.add(i, k, val); - } - } - REQUIRE(gaps::sum(dense_ASampler.mMatrix) == gaps::sum(sparse_ASampler.mMatrix)); + AMat(i,k) = rng.uniform(0.f, 10.f) * (rng.uniform() < 0.2f ? 0.f : 1.f); + sparse_A.setMatrix(AMat); + dense_A.setMatrix(AMat); + REQUIRE(dense_A.matrixSum() == sparse_A.matrixSum()); + // and the P matrix (samples x patterns) + Matrix PMat(data.nCol(), params.nPatterns); for (unsigned j = 0; j < data.nCol(); ++j) - { for (unsigned k = 0; k < params.nPatterns; ++k) - { - // [AI-generated] Populate matching dense/sparse test matrices with mostly - // nonzero values and occasional zeros. - float val = rng.uniform(0.f, 10.f) * (rng.uniform() < 0.2f ? 0.f : 1.f); - dense_PSampler.mMatrix(j,k) = val; - sparse_PSampler.mMatrix.add(j, k, val); - } - } - REQUIRE(gaps::sum(dense_PSampler.mMatrix) == gaps::sum(sparse_PSampler.mMatrix)); - - // sync them back up - sparse_ASampler.sync(sparse_PSampler); - sparse_PSampler.sync(sparse_ASampler); - dense_ASampler.sync(dense_PSampler); - dense_PSampler.sync(dense_ASampler); - dense_ASampler.extraInitialization(); - dense_PSampler.extraInitialization(); - -///////////////// test that alphaParameters are the same /////////////////////// + PMat(j,k) = rng.uniform(0.f, 10.f) * (rng.uniform() < 0.2f ? 0.f : 1.f); + sparse_P.setMatrix(PMat); + dense_P.setMatrix(PMat); + REQUIRE(dense_P.matrixSum() == sparse_P.matrixSum()); + + // sync them back up (dense needs extraInitialization to build the AP matrix; + // sparse's is a nop, its sync regenerates the lookup tables) + sparse_A.sync(sparse_P); + sparse_P.sync(sparse_A); + dense_A.sync(dense_P); + dense_P.sync(dense_A); + dense_A.extraInitialization(); + dense_P.extraInitialization(); + + // chiSq must also match between sparse and dense. This exercises the S + // floor in chiSq() specifically: the data has 0 < D < 1 entries where the + // unfloored S = D disagrees with the floored S = max(0.1*D, 0.1) (issue #18 + // follow-up). Both must use the single invSSq() model. + REQUIRE(sparse_A.chiSq() == TEST_APPROX(dense_A.chiSq())); + REQUIRE(sparse_P.chiSq() == TEST_APPROX(dense_P.chiSq())); + + // 1D alphaParameters must match between sparse and dense for (unsigned i = 0; i < data.nRow(); ++i) - { for (unsigned k = 0; k < params.nPatterns; ++k) - { - AlphaParameters sa = sparse_ASampler.alphaParameters(i,k); - AlphaParameters da = dense_ASampler.alphaParameters(i,k); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); - } - } - + requireAlphaEqual(sparse_A.a1(i,k), dense_A.a1(i,k)); for (unsigned j = 0; j < data.nCol(); ++j) - { for (unsigned k = 0; k < params.nPatterns; ++k) - { - AlphaParameters sa = sparse_PSampler.alphaParameters(j,k); - AlphaParameters da = dense_PSampler.alphaParameters(j,k); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); - } - } + requireAlphaEqual(sparse_P.a1(j,k), dense_P.a1(j,k)); -///////////// test two dimensional alphaParameters are the same //////////////// + // 2D alphaParameters (and symmetry) must match for (unsigned i = 0; i < data.nRow(); ++i) - { for (unsigned k1 = 0; k1 < params.nPatterns; ++k1) - { for (unsigned k2 = k1+1; k2 < params.nPatterns; ++k2) { - AlphaParameters sa = sparse_ASampler.alphaParameters(i,k1,i,k2); - AlphaParameters da = dense_ASampler.alphaParameters(i,k1,i,k2); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); - - // symmetry - sa = sparse_ASampler.alphaParameters(i,k2,i,k1); - da = dense_ASampler.alphaParameters(i,k2,i,k1); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); + requireAlphaEqual(sparse_A.a2(i,k1,i,k2), dense_A.a2(i,k1,i,k2)); + requireAlphaEqual(sparse_A.a2(i,k2,i,k1), dense_A.a2(i,k2,i,k1)); } - } - } - for (unsigned j = 0; j < data.nCol(); ++j) - { for (unsigned k1 = 0; k1 < params.nPatterns; ++k1) - { for (unsigned k2 = k1+1; k2 < params.nPatterns; ++k2) { - AlphaParameters sa = sparse_PSampler.alphaParameters(j,k1,j,k2); - AlphaParameters da = dense_PSampler.alphaParameters(j,k1,j,k2); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); - - // symmetry - sa = sparse_PSampler.alphaParameters(j,k2,j,k1); - da = dense_PSampler.alphaParameters(j,k2,j,k1); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); + requireAlphaEqual(sparse_P.a2(j,k1,j,k2), dense_P.a2(j,k1,j,k2)); + requireAlphaEqual(sparse_P.a2(j,k2,j,k1), dense_P.a2(j,k2,j,k1)); } - } - } -///////////// test alphaParameters with change are the same //////////////////// + // alphaParametersWithChange must match for (unsigned i = 0; i < data.nRow(); ++i) - { for (unsigned k = 0; k < params.nPatterns; ++k) { float ch = rng.uniform(0.f, 25.f); - AlphaParameters sa = sparse_ASampler.alphaParametersWithChange(i,k,ch); - AlphaParameters da = dense_ASampler.alphaParametersWithChange(i,k,ch); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); + requireAlphaEqual(sparse_A.aC(i,k,ch), dense_A.aC(i,k,ch)); } - } - for (unsigned j = 0; j < data.nCol(); ++j) - { for (unsigned k = 0; k < params.nPatterns; ++k) { float ch = rng.uniform(0.f, 25.f); - AlphaParameters sa = sparse_PSampler.alphaParametersWithChange(j,k,ch); - AlphaParameters da = dense_PSampler.alphaParametersWithChange(j,k,ch); - REQUIRE(sa.s >= 0.f); - REQUIRE(da.s >= 0.f); - if (sa.s <= gaps::epsilon || da.s <= gaps::epsilon) - { - REQUIRE(sa.s <= gaps::epsilon); - REQUIRE(da.s <= gaps::epsilon); - } - REQUIRE(sa.s == TEST_APPROX(da.s)); - REQUIRE(sa.s_mu == TEST_APPROX(da.s_mu)); + requireAlphaEqual(sparse_P.aC(j,k,ch), dense_P.aC(j,k,ch)); } - } } -#endif } diff --git a/src/cpp_tests/testSparseIterator.cpp b/src/cpp_tests/testSparseIterator.cpp index 89a9fb61..0c6aff47 100755 --- a/src/cpp_tests/testSparseIterator.cpp +++ b/src/cpp_tests/testSparseIterator.cpp @@ -9,17 +9,15 @@ #include "../data_structures/SparseIterator.h" #include +#include -TEST_CASE("Test SparseIterator.h - One Dimensional","[sparseiterator][1D]") +TEST_CASE("Test SparseIterator.h - One Dimensional", "[sparseiterator][1d]") { -#if 0 SECTION("Simple Case") { - SparseVector v(10); - v.insert(0, 1.f); - v.insert(4, 5.f); - v.insert(7, 8.f); - v.insert(9, 10.f); + std::vector vin(10, 0.f); + vin[0] = 1.f; vin[4] = 5.f; vin[7] = 8.f; vin[9] = 10.f; + SparseVector v(vin); SparseIterator<1> it(v); REQUIRE(get<1>(it) == 1.f); @@ -32,7 +30,6 @@ TEST_CASE("Test SparseIterator.h - One Dimensional","[sparseiterator][1D]") it.next(); REQUIRE(it.atEnd()); } -#endif SECTION("Test Identical Sums") { @@ -43,8 +40,6 @@ TEST_CASE("Test SparseIterator.h - One Dimensional","[sparseiterator][1D]") { for (unsigned j = 0; j < ref.nCol(); ++j) { - // [AI-generated] Build toy sparse test data by randomly zeroing half of the - // candidate entries. ref(i,j) = (i + j) * (rng.uniform() < 0.5f ? 0.f : 1.f); } } @@ -65,16 +60,13 @@ TEST_CASE("Test SparseIterator.h - One Dimensional","[sparseiterator][1D]") } } -TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") +TEST_CASE("Test SparseIterator.h - Two Dimensional", "[sparseiterator][2d]") { -#if 0 SECTION("Simple Case") { - SparseVector sv(10); - sv.insert(0, 1.f); - sv.insert(4, 5.f); - sv.insert(7, 8.f); - sv.insert(9, 10.f); + std::vector svin(10, 0.f); + svin[0] = 1.f; svin[4] = 5.f; svin[7] = 8.f; svin[9] = 10.f; + SparseVector sv(svin); HybridVector hv(10); hv.add(4, 3.f); @@ -94,15 +86,10 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") SECTION("First overlap happens after 64 entries") { - SparseVector sv(100); - sv.insert(1, 1.f); - sv.insert(2, 2.f); - sv.insert(3, 3.f); - sv.insert(4, 4.f); - sv.insert(5, 5.f); - sv.insert(74, 74.f); - sv.insert(75, 75.f); - sv.insert(76, 76.f); + std::vector svin(100, 0.f); + svin[1]=1.f; svin[2]=2.f; svin[3]=3.f; svin[4]=4.f; svin[5]=5.f; + svin[74]=74.f; svin[75]=75.f; svin[76]=76.f; + SparseVector sv(svin); HybridVector hv(100); hv.add(6, 7.f); @@ -119,7 +106,6 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") SECTION("Test Dot Product with gap") { - SparseVector sv(300); HybridVector hv(300); Vector dv1(300), dv2(300); @@ -130,7 +116,6 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") for (unsigned i = 0; i < 30; ++i) { float val = rng.uniform(50.f,500.f); - sv.insert(i, val); dv1[i] = val; } @@ -144,7 +129,6 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") for (unsigned i = 70; i < 120; i+=3) { float v1 = rng.uniform(50.f,500.f); - sv.insert(i, v1); dv1[i] = v1; float v2 = rng.uniform(50.f,500.f); @@ -156,14 +140,12 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") for (unsigned i = 128; i < 196; ++i) { float val = rng.uniform(5.f,10.f); - sv.insert(i, val); dv1[i] = val; } for (unsigned i = 200; i < 300; ++i) { float v1 = rng.uniform(50.f,500.f); - sv.insert(i, v1); dv1[i] = v1; float v2 = rng.uniform(50.f,500.f); @@ -171,6 +153,9 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") dv2[i] = v2; } + // build the sparse vector from its dense form, then iterate + SparseVector sv(dv1); + // calculate dot product float sdot = 0.f, ddot = 0.f; SparseIterator<2> it(sv, hv); @@ -194,10 +179,11 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") it.next(); } - REQUIRE(ddot == gaps::dot(dv1, dv2)); - REQUIRE(sdot == ddot); + // ddot accumulates in index order, gaps::dot uses SIMD (different order), + // so compare with Approx (float addition is not associative) + REQUIRE(ddot == Approx(gaps::dot(dv1, dv2))); + REQUIRE(sdot == ddot); // sparse iterator uses the same order as ddot -> exact } -#endif // could this fail because of SIMD? SECTION("Test Identical Dot Products") @@ -210,8 +196,6 @@ TEST_CASE("Test SparseIterator.h - Two Dimensional","[sparseiterator][2D]") { for (unsigned j = 0; j < ref.nCol(); ++j) { - // [AI-generated] Build toy sparse test data by randomly zeroing half of the - // candidate entries. ref(i,j) = (i + j) * (rng.uniform() < 0.5f ? 0.f : 1.f); hMat.add(i, j, ref(i,j)); } @@ -245,16 +229,13 @@ static float tripleProduct(const Vector &v1, const Vector &v2, const Vector &v3) return prod; } -TEST_CASE("Test SparseIterator.h - Three Dimensional","[sparseiterator][3D]") +TEST_CASE("Test SparseIterator.h - Three Dimensional", "[sparseiterator][3d]") { -#if 0 SECTION("Simple Case") { - SparseVector sv(10); - sv.insert(0, 1.f); - sv.insert(7, 8.f); - sv.insert(8, 9.f); - sv.insert(9, 10.f); + std::vector svin(10, 0.f); + svin[0] = 1.f; svin[7] = 8.f; svin[8] = 9.f; svin[9] = 10.f; + SparseVector sv(svin); HybridVector hv1(10); hv1.add(4, 3.f); @@ -282,7 +263,6 @@ TEST_CASE("Test SparseIterator.h - Three Dimensional","[sparseiterator][3D]") it.next(); REQUIRE(it.atEnd()); } -#endif SECTION("Test Identical Triple Products") { @@ -294,8 +274,6 @@ TEST_CASE("Test SparseIterator.h - Three Dimensional","[sparseiterator][3D]") { for (unsigned j = 0; j < ref.nCol(); ++j) { - // [AI-generated] Build toy sparse test data by randomly zeroing half of the - // candidate entries. ref(i,j) = (i + j) * (rng.uniform() < 0.5f ? 0.f : 1.f); hMat.add(i, j, ref(i,j)); } diff --git a/src/cpp_tests/testSparseMatrix.cpp b/src/cpp_tests/testSparseMatrix.cpp index 0c4b1929..015f0ee5 100755 --- a/src/cpp_tests/testSparseMatrix.cpp +++ b/src/cpp_tests/testSparseMatrix.cpp @@ -1,8 +1,7 @@ #include #include "../testthat-tweak.h" #include "../data_structures/SparseMatrix.h" -#include "../file_parser/CsvParser.h" -#include "../file_parser/TsvParser.h" +#include "../file_parser/FileParser.h" #include "../file_parser/MtxParser.h" #include "../math/Random.h" #include "../math/VectorMath.h" @@ -57,7 +56,7 @@ unsigned nc, unsigned nIndices, float sum1, float sum2, float sum3) sequentialVector(nIndices)); } -TEST_CASE("Test Writing/Reading Sparse Matrices from File") +TEST_CASE("Test Writing/Reading Sparse Matrices from File","[sparsematrix][sparsematrixrw]") { // matrix to use for testing Matrix ref(25, 50); @@ -67,36 +66,31 @@ TEST_CASE("Test Writing/Reading Sparse Matrices from File") { for (unsigned j = 0; j < ref.nCol(); ++j) { - // [AI-generated] Build toy sparse test data by randomly zeroing half of the - // candidate entries. ref(i,j) = (i + j) * (rng.uniform() < 0.5f ? 0.f : 1.f); } } // write matrix to file - FileParser::writeToTsv("testMatWrite.tsv", ref); FileParser::writeToCsv("testMatWrite.csv", ref); - FileParser::writeToMtx("testMatWrite.mtx", ref); + //FileParser::writeToMtx("testMatWrite.mtx", ref); // read matrices from file SparseMatrix mat(ref, false, false, sequentialVector(0)); - SparseMatrix matTsv("testMatWrite.tsv", false, false, sequentialVector(0)); SparseMatrix matCsv("testMatWrite.csv", false, false, sequentialVector(0)); - SparseMatrix matMtx("testMatWrite.mtx", false, false, sequentialVector(0)); + //SparseMatrix matMtx("testMatWrite.mtx", false, false, sequentialVector(0)); // delete files - std::remove("testMatWrite.tsv"); std::remove("testMatWrite.csv"); - std::remove("testMatWrite.mtx"); + //std::remove("testMatWrite.mtx"); // test matrices REQUIRE(gaps::sum(mat) == gaps::sum(ref)); - REQUIRE(gaps::sum(matTsv) == gaps::sum(ref)); REQUIRE(gaps::sum(matCsv) == gaps::sum(ref)); - REQUIRE(gaps::sum(matMtx) == gaps::sum(ref)); + //REQUIRE(gaps::sum(matMtx) == gaps::sum(ref)); } -TEST_CASE("Test SparseMatrix.h") + +TEST_CASE("Test SparseMatrix.h","[sparsematrix][sparsematrixfull]") { SECTION("Full Constructor") { @@ -110,19 +104,16 @@ TEST_CASE("Test SparseMatrix.h") } // write matrix to file - FileParser::writeToTsv("testMatWrite.tsv", ref); FileParser::writeToCsv("testMatWrite.csv", ref); - FileParser::writeToMtx("testMatWrite.mtx", ref); + //FileParser::writeToMtx("testMatWrite.mtx", ref); // test testAllConstructorSituations(ref, 10, 25, 5, 4125.f, 1750.f, 325.f); - testAllConstructorSituations("testMatWrite.tsv", 10, 25, 5, 4125.f, 1750.f, 325.f); testAllConstructorSituations("testMatWrite.csv", 10, 25, 5, 4125.f, 1750.f, 325.f); - testAllConstructorSituations("testMatWrite.mtx", 10, 25, 5, 4125.f, 1750.f, 325.f); + //testAllConstructorSituations("testMatWrite.mtx", 10, 25, 5, 4125.f, 1750.f, 325.f); // delete files - std::remove("testMatWrite.tsv"); std::remove("testMatWrite.csv"); - std::remove("testMatWrite.mtx"); + //std::remove("testMatWrite.mtx"); } } diff --git a/src/cpp_tests/testSparseVector.cpp b/src/cpp_tests/testSparseVector.cpp index dec77b84..55c0d00c 100755 --- a/src/cpp_tests/testSparseVector.cpp +++ b/src/cpp_tests/testSparseVector.cpp @@ -59,31 +59,28 @@ TEST_CASE("Test SparseVector","[sparsevector]") REQUIRE(v1[i] == v2[i]); } } +} -#if 0 - SECTION("bit flags set correctly") +// Regression: gaps::min/max(SparseVector) used to read the first element before +// checking atEnd(), segfaulting on an empty (all-zero) sparse vector. This arises +// for any all-zero row/column of the data matrix in SparseNormalModel construction. +TEST_CASE("gaps::min/max on empty SparseVector","[sparsevector][emptyminmax]") +{ + SECTION("empty (all-zero) sparse vector returns 0, no crash") { - SparseVector v(10); - v.insert(0, 1.f); - v.insert(4, 5.f); - v.insert(7, 8.f); - v.insert(9, 10.f); - REQUIRE(v.mIndexBitFlags[0] == 0b1010010001); + SparseVector sv(100); // no non-zero elements stored + REQUIRE(gaps::max(sv) == 0.f); + REQUIRE(gaps::min(sv) == 0.f); } - SECTION("values placed correctly") + SECTION("non-empty sparse vector is unaffected by the guard") { - SparseVector v(10); - v.insert(0, 1.f); - v.insert(4, 5.f); - v.insert(7, 8.f); - v.insert(9, 10.f); - - REQUIRE(v.mData.size() == 4); - REQUIRE(v.mData[0] == 1.f); - REQUIRE(v.mData[1] == 5.f); - REQUIRE(v.mData[2] == 8.f); - REQUIRE(v.mData[3] == 10.f); + std::vector in_v(50, 0.f); + in_v[10] = 3.f; + in_v[20] = 1.f; + in_v[30] = 7.f; + SparseVector sv(in_v); + REQUIRE(gaps::max(sv) == 7.f); + REQUIRE(gaps::min(sv) == 1.f); // min over stored non-zero values } -#endif } diff --git a/src/cpp_tests/testVector.cpp b/src/cpp_tests/testVector.cpp index 1362e8aa..4fe5cdc2 100755 --- a/src/cpp_tests/testVector.cpp +++ b/src/cpp_tests/testVector.cpp @@ -17,6 +17,16 @@ TEST_CASE("Test Vector","[vector]") REQUIRE(gaps::sum(v) == 0.f); } + SECTION("Test padding") + { + Vector v(100); + float foam=3; + v.pad(foam); + REQUIRE(v.size() == 100); + REQUIRE(!gaps::isVectorZero(v)); + REQUIRE(100*foam==gaps::sum(v)); + } + SECTION("Test std::vector constructor") { GapsRng rng(&randState); @@ -49,6 +59,18 @@ TEST_CASE("Test Vector","[vector]") } } +// Regression: gaps::min/max/whichMax(Vector) dereferenced v[0] on an empty +// vector (out-of-bounds read). Now guarded to return 0 on size 0. (Same +// empty-container class as the SparseVector fix, issue 12.) +TEST_CASE("gaps::min/max/whichMax on empty Vector","[vector][emptyminmax]") +{ + Vector v(0); + REQUIRE(v.size() == 0); + REQUIRE(gaps::min(v) == 0.f); + REQUIRE(gaps::max(v) == 0.f); + REQUIRE(gaps::whichMax(v) == 0); +} + // optional test used for benchmarking, set to 0 to disable, 1 to enable #if 0 diff --git a/src/data_structures/HybridVector.cpp b/src/data_structures/HybridVector.cpp index b8f99297..790e89b6 100755 --- a/src/data_structures/HybridVector.cpp +++ b/src/data_structures/HybridVector.cpp @@ -51,35 +51,29 @@ unsigned HybridVector::size() const return mSize; } -// can be called from multiple concurrent OpenMP threads bool HybridVector::add(unsigned i, float v) { GAPS_ASSERT(i < mSize); if (mData[i] + v < gaps::epsilon) { - #pragma omp atomic mIndexBitFlags[i / 64] &= ~(1ull << (i % 64)); mData[i] = 0.f; return true; } - #pragma omp atomic mIndexBitFlags[i / 64] |= (1ull << (i % 64)); mData[i] += v; return false; } -// can be called from multiple concurrent OpenMP threads bool HybridVector::set(unsigned i, float v) { GAPS_ASSERT(i < mSize); if (v < gaps::epsilon) { - #pragma omp atomic mIndexBitFlags[i / 64] &= ~(1ull << (i % 64)); mData[i] = 0.f; return true; } - #pragma omp atomic mIndexBitFlags[i / 64] |= (1ull << (i % 64)); mData[i] = v; return false; diff --git a/src/data_structures/Matrix.cpp b/src/data_structures/Matrix.cpp index b867fa19..15a564ce 100755 --- a/src/data_structures/Matrix.cpp +++ b/src/data_structures/Matrix.cpp @@ -26,6 +26,14 @@ void Matrix::pad(float val) } } +void Matrix::padSIMD(float val) +{ + for (unsigned j = 0; j < mNumCols; ++j) + { + mCols[j].padSIMD(val); + } +} + // constructor from data set read in as a matrix Matrix::Matrix(const Matrix &mat, bool genesInCols, bool subsetGenes, std::vector indices) @@ -146,6 +154,7 @@ std::vector indices) } } + unsigned Matrix::nRow() const { return mNumRows; @@ -214,4 +223,4 @@ Archive& operator>>(Archive &ar, Matrix &mat) ar >> mat.mCols[j]; } return ar; -} \ No newline at end of file +} diff --git a/src/data_structures/Matrix.h b/src/data_structures/Matrix.h index 392a7c3c..ea31679a 100755 --- a/src/data_structures/Matrix.h +++ b/src/data_structures/Matrix.h @@ -25,6 +25,7 @@ class Matrix float& operator()(unsigned i, unsigned j); Vector& getCol(unsigned col); const Vector& getCol(unsigned col) const; + void padSIMD(float val); bool empty() const; Matrix getMatrix() const; friend Archive& operator<<(Archive &ar, const Matrix &mat); @@ -35,4 +36,4 @@ class Matrix unsigned mNumCols; }; -#endif // __COGAPS_MATRIX_H__ \ No newline at end of file +#endif // __COGAPS_MATRIX_H__ diff --git a/src/data_structures/MutableMap.h b/src/data_structures/MutableMap.h index eaa24f7d..5ce320b6 100644 --- a/src/data_structures/MutableMap.h +++ b/src/data_structures/MutableMap.h @@ -61,6 +61,7 @@ class MutableMap std::pair insert(const std::pair &val) { + //insert return iterator and success flag std::pair< typename std::map::iterator, bool> result = mMap.insert(val); iterator it(result.first); return std::pair(it, result.second); @@ -78,7 +79,12 @@ class MutableMap void updateKey(iterator it, const K &newKey) { - const_cast((*it).first) = newKey; // TODO cleaner solution that this + // const_cast((*it).first) = newKey; + // TODO cleaner solution that this + std::pair newpair(newKey,it.mIt->second); + mMap.erase(it.mIt); + mMap.insert(newpair); + } iterator begin() diff --git a/src/data_structures/SparseVector.cpp b/src/data_structures/SparseVector.cpp index b6a827a4..586e5096 100755 --- a/src/data_structures/SparseVector.cpp +++ b/src/data_structures/SparseVector.cpp @@ -133,7 +133,16 @@ Archive& operator>>(Archive &ar, SparseVector &vec) { ar >> vec.mIndexBitFlags[i]; } - for (unsigned i = 0; i < vec.mData.size(); ++i) + // the number of stored (non-zero) values is the popcount of the bit flags, + // which were just read; resize mData so a differently-structured (e.g. empty) + // destination is restored correctly rather than left with the wrong count. + unsigned nNonZeroes = 0; + for (unsigned i = 0; i < vec.mIndexBitFlags.size(); ++i) + { + nNonZeroes += __builtin_popcountll(vec.mIndexBitFlags[i]); + } + vec.mData.resize(nNonZeroes); + for (unsigned i = 0; i < nNonZeroes; ++i) { ar >> vec.mData[i]; } diff --git a/src/data_structures/Vector.cpp b/src/data_structures/Vector.cpp index 1444987d..1e1cd02c 100755 --- a/src/data_structures/Vector.cpp +++ b/src/data_structures/Vector.cpp @@ -3,8 +3,12 @@ #include "../utils/Archive.h" #include "../utils/GapsAssert.h" -#define SIMD_PAD(x) (gaps::simd::Index::increment() + \ - gaps::simd::Index::increment() * ((x) / gaps::simd::Index::increment())) +// Minimum effective SIMD width for padding purposes. +// When explicit SIMD is disabled (SIMD_INC == 1), the compiler may still +// auto-vectorize loops with 4-wide NEON on ARM or 4-wide SSE on x86. +// Use 8 to cover AVX2 as well. +#define SIMD_PAD_INC 8 +#define SIMD_PAD(x) (SIMD_PAD_INC + SIMD_PAD_INC * ((x) / SIMD_PAD_INC)) Vector::Vector(unsigned sz) : @@ -27,6 +31,17 @@ mSize(v.size()) } void Vector::pad(float val) +{ + //here was the Msize as start value, error! + for (unsigned i = 0; i < mData.size(); ++i) + { + mData[i] = val; + } +} + +// fill only the SIMD overflow positions beyond mSize with val +// to prevent 0/0 = NaN when SIMD reads past the real data +void Vector::padSIMD(float val) { for (unsigned i = mSize; i < mData.size(); ++i) { @@ -108,4 +123,4 @@ Archive& operator>>(Archive &ar, Vector &vec) ar >> vec.mData[i]; } return ar; -} \ No newline at end of file +} diff --git a/src/data_structures/Vector.h b/src/data_structures/Vector.h index c7621e98..cd31237f 100755 --- a/src/data_structures/Vector.h +++ b/src/data_structures/Vector.h @@ -25,6 +25,7 @@ class Vector float* ptr(); unsigned size() const; void pad(float val); + void padSIMD(float val); void operator+=(const Vector &v); void operator*=(float f); void operator/=(float f); diff --git a/src/gibbs_sampler/AsynchronousGibbsSampler.h b/src/gibbs_sampler/AsynchronousGibbsSampler.h deleted file mode 100755 index 40544860..00000000 --- a/src/gibbs_sampler/AsynchronousGibbsSampler.h +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef __COGAPS_ASYNCHRONOUS_GIBBS_SAMPLER_H__ -#define __COGAPS_ASYNCHRONOUS_GIBBS_SAMPLER_H__ - -#include "../atomic/ConcurrentAtomicDomain.h" -#include "../atomic/ProposalQueue.h" -#include "../data_structures/Matrix.h" -#include "../math/Math.h" -#include "../math/VectorMath.h" -#include "../math/MatrixMath.h" -#include "../GapsParameters.h" -#include "../math/Random.h" - -#include -#include -#include -#include - -//////////////////////////// AsynchronousGibbsSampler Interface //////////////////////////// - -class GapsStatistics; - -template -class AsynchronousGibbsSampler; - -template -Archive& operator<<(Archive &ar, const AsynchronousGibbsSampler &s); - -template -Archive& operator>>(Archive &ar, AsynchronousGibbsSampler &s); - -template -class AsynchronousGibbsSampler : public DataModel -{ -public: - template - AsynchronousGibbsSampler(const DataType &data, bool transpose, bool subsetRows, - float alpha, float maxGibbsMass, const GapsParameters ¶ms, - GapsRandomState *randState); - unsigned nAtoms() const; - float getAverageQueueLength() const; - void update(unsigned nSteps, unsigned nThreads); - friend Archive& operator<< (Archive &ar, const AsynchronousGibbsSampler &s); - friend Archive& operator>> (Archive &ar, AsynchronousGibbsSampler &s); -private: - void birth(const AtomicProposal &prop); - void death(const AtomicProposal &prop); - void move(const AtomicProposal &prop); - void exchange(const AtomicProposal &prop); -#ifdef GAPS_DEBUG - float maximumDrift() const; -#endif - ConcurrentAtomicDomain mDomain; // data structure providing access to atoms - ProposalQueue mQueue; // creates queue of proposals that get evaluated by sampler - float mAvgQueueLength; - float mNumQueueSamples; -}; - -//////////////////// AsynchronousGibbsSampler - templated functions //////////////////////// - -template -template -AsynchronousGibbsSampler::AsynchronousGibbsSampler(const DataType &data, -bool transpose, bool subsetRows, float alpha, float maxGibbsMass, -const GapsParameters ¶ms, GapsRandomState *randState) - : -DataModel(data, transpose, subsetRows, params, alpha, maxGibbsMass), -mDomain(DataModel::nElements()), -mQueue(DataModel::nElements(), DataModel::nPatterns(), randState), -mAvgQueueLength(0), -mNumQueueSamples(0) -{ - mQueue.setAlpha(alpha); - mQueue.setLambda(DataModel::lambda()); -} - -template -unsigned AsynchronousGibbsSampler::nAtoms() const -{ - return mDomain.size(); -} - -template -float AsynchronousGibbsSampler::getAverageQueueLength() const -{ - return mAvgQueueLength; -} - -template -void AsynchronousGibbsSampler::update(unsigned nSteps, unsigned nThreads) -{ - unsigned n = 0; - while (n < nSteps) - { - // create the largest queue possible, without hitting any conflicts - mQueue.populate(mDomain, nSteps - n); - n += mQueue.nProcessed(); - if (n < nSteps) // don't count last one since it might be truncated - { - mNumQueueSamples += 1.f; // record the size of the queue for diagnostics - mAvgQueueLength *= (mNumQueueSamples - 1.f) / mNumQueueSamples; - mAvgQueueLength += static_cast(mQueue.size()) / mNumQueueSamples; - } - // process all proposed updates in parallel - the way the queue is - // populated ensures no race conditions will happen - #pragma omp parallel for num_threads(nThreads) - for (unsigned i = 0; i < mQueue.size(); ++i) - { - switch (mQueue[i].type) - { - case 'B': birth(mQueue[i]); break; - case 'D': death(mQueue[i]); break; - case 'M': move(mQueue[i]); break; - case 'E': exchange(mQueue[i]); break; - } - } - mQueue.clear(); - mDomain.flushEraseCache(); - } - GAPS_ASSERT(n == nSteps); - GAPS_ASSERT(mDomain.isSorted()); - GAPS_ASSERT_MSG(maximumDrift() < 0.01f, "maximum drift: " << maximumDrift()); -} - -// add an atom at a random position, calculate mass either with an -// exponential distribution or with the gibbs mass distribution -template -void AsynchronousGibbsSampler::birth(const AtomicProposal &prop) -{ - // [AI-generated] If the conditional Gibbs distribution is defined, sample from it; - // otherwise use the exponential prior so the birth proposal remains valid. - OptionalFloat mass = DataModel::canUseGibbs(prop.c1) - ? DataModel::sampleBirth(prop.r1, prop.c1, &(prop.rng)) - : prop.rng.exponential(DataModel::lambda()); - // accept mass as long as gibbs succeded and it's non-zero - if (mass.hasValue() && mass.value() >= gaps::epsilon) - { - mQueue.acceptBirth(); - prop.atom1->updateMass(mass.value()); - DataModel::changeMatrix(prop.r1, prop.c1, mass.value()); - return; - } - // otherwise reject birth - mQueue.rejectBirth(); - mDomain.cacheErase(prop.atom1); -} - -// attempt to rebirth an atom in place of the killed atom -template -void AsynchronousGibbsSampler::death(const AtomicProposal &prop) -{ - // determine mass to attempt rebirth with - float rebirthMass = prop.atom1->mass(); // default rebirth mass == no change to atom - AlphaParameters alpha = DataModel::alphaParametersWithChange(prop.r1, prop.c1, - -1.f * prop.atom1->mass()) * DataModel::annealingTemp(); - if (DataModel::canUseGibbs(prop.c1)) - { - OptionalFloat gMass = gibbsMass(alpha, 0.f, DataModel::maxGibbsMass(), &(prop.rng), - DataModel::lambda()); - if (gMass.hasValue()) - { - rebirthMass = gMass.value(); - } - } - // handle accept/reject of the rebirth - float deltaLL = rebirthMass * (alpha.s_mu - alpha.s * rebirthMass / 2.f); - if (std::log(prop.rng.uniform()) < deltaLL) // accept - { - mQueue.rejectDeath(); - if (rebirthMass != prop.atom1->mass()) - { - DataModel::safelyChangeMatrix(prop.r1, prop.c1, rebirthMass - prop.atom1->mass()); - prop.atom1->updateMass(rebirthMass); - } - } - else // reject - { - mQueue.acceptDeath(); - DataModel::safelyChangeMatrix(prop.r1, prop.c1, -1.f * prop.atom1->mass()); - mDomain.cacheErase(prop.atom1); - } -} - -// move mass from src to dest in the atomic domain -template -void AsynchronousGibbsSampler::move(const AtomicProposal &prop) -{ - GAPS_ASSERT(prop.r1 != prop.r2 || prop.c1 != prop.c2); - float deltaLL = DataModel::deltaLogLikelihood(prop.r1, prop.c1, prop.r2, prop.c2, - prop.atom1->mass()); - if (std::log(prop.rng.uniform()) < deltaLL) - { - mDomain.move(prop.atom1, prop.pos); - DataModel::safelyChangeMatrix(prop.r1, prop.c1, -prop.atom1->mass()); - DataModel::changeMatrix(prop.r2, prop.c2, prop.atom1->mass()); - return; - } -} - -// exchange some amount of mass between two positions, note it is possible -// for one of the atoms to be deleted if it's mass becomes too small -template -void AsynchronousGibbsSampler::exchange(const AtomicProposal &prop) -{ - GAPS_ASSERT(prop.r1 != prop.r2 || prop.c1 != prop.c2); - if (DataModel::canUseGibbs(prop.c1, prop.c2)) - { - OptionalFloat mass = DataModel::sampleExchange(prop.r1, prop.c1, prop.atom1->mass(), - prop.r2, prop.c2, prop.atom2->mass(), &(prop.rng)); - float newMass1 = prop.atom1->mass() + mass.value(); - float newMass2 = prop.atom2->mass() - mass.value(); - if (mass.hasValue() && newMass1 > gaps::epsilon && newMass2 > gaps::epsilon) - { - DataModel::safelyChangeMatrix(prop.r1, prop.c1, newMass1 - prop.atom1->mass()); - DataModel::safelyChangeMatrix(prop.r2, prop.c2, newMass2 - prop.atom2->mass()); - prop.atom1->updateMass(newMass1); - prop.atom2->updateMass(newMass2); - return; - } - } -} - -template -Archive& operator<<(Archive &ar, const AsynchronousGibbsSampler &s) -{ - operator<<(ar, static_cast(s)) << s.mDomain << s.mQueue; - return ar; -} - -template -Archive& operator>>(Archive &ar, AsynchronousGibbsSampler &s) -{ - operator>>(ar, static_cast(s)) >> s.mDomain >> s.mQueue; - return ar; -} - -#ifdef GAPS_DEBUG -template -float AsynchronousGibbsSampler::maximumDrift() const -{ - if (mDomain.size() == 0) - { - return gaps::sum(DataModel::mMatrix); - } - const ConcurrentAtom *atom = mDomain.front(); - uint64_t binLength = std::numeric_limits::max() / DataModel::nElements(); - unsigned row = (atom->pos() / binLength) / DataModel::nPatterns(); - unsigned col = (atom->pos() / binLength) % DataModel::nPatterns(); - float mass = atom->mass(); - float maxDrift = 0.f; - while (atom->hasRight()) - { - atom = atom->right(); - unsigned newRow = (atom->pos() / binLength) / DataModel::nPatterns(); - unsigned newCol = (atom->pos() / binLength) % DataModel::nPatterns(); - if (row == newRow && col == newCol) - { - mass += atom->mass(); - } - else - { - float actual = DataModel::mMatrix(row, col); - //float drift = (actual > 1.f) ? std::abs(actual - mass) / actual : actual; - float drift = std::abs(actual - mass); - // [AI-generated] Track the largest mismatch between atomic and matrix mass. - maxDrift = (drift > maxDrift) ? drift : maxDrift; - mass = atom->mass(); - row = newRow; - col = newCol; - } - } - return maxDrift; -} -#endif // GAPS_DEBUG - -#endif // __COGAPS_ASYNCHRONOUS_GIBBS_SAMPLER_H__ \ No newline at end of file diff --git a/src/gibbs_sampler/DenseNormalModel.cpp b/src/gibbs_sampler/DenseNormalModel.cpp index 9ef5125a..f8584194 100755 --- a/src/gibbs_sampler/DenseNormalModel.cpp +++ b/src/gibbs_sampler/DenseNormalModel.cpp @@ -17,13 +17,12 @@ void DenseNormalModel::setAnnealingTemp(float temp) } // copy transpose of other AP matrix -void DenseNormalModel::sync(const DenseNormalModel &model, unsigned nThreads) +void DenseNormalModel::sync(const DenseNormalModel &model) { GAPS_ASSERT(model.mAPMatrix.nRow() == mAPMatrix.nCol()); GAPS_ASSERT(model.mAPMatrix.nCol() == mAPMatrix.nRow()); unsigned nc = model.mAPMatrix.nCol(); unsigned nr = model.mAPMatrix.nRow(); - #pragma omp parallel for num_threads(nThreads) for (unsigned j = 0; j < nc; ++j) { for (unsigned i = 0; i < nr; ++i) @@ -116,7 +115,7 @@ void DenseNormalModel::changeMatrix(unsigned row, unsigned col, float delta) void DenseNormalModel::safelyChangeMatrix(unsigned row, unsigned col, float delta) { - float newVal = gaps::max(mMatrix(row, col) + delta, 0.f); + float newVal = std::max(mMatrix(row, col) + delta, 0.f); updateAPMatrix(row, col, newVal - mMatrix(row, col)); mMatrix(row, col) = newVal; GAPS_ASSERT(mMatrix(row, col) >= 0.f); diff --git a/src/gibbs_sampler/DenseNormalModel.h b/src/gibbs_sampler/DenseNormalModel.h index c717cd1e..74c07376 100755 --- a/src/gibbs_sampler/DenseNormalModel.h +++ b/src/gibbs_sampler/DenseNormalModel.h @@ -1,98 +1,125 @@ -#ifndef __COGAPS_DENSE_NORMAL_MODEL_H__ -#define __COGAPS_DENSE_NORMAL_MODEL_H__ - -#include "AlphaParameters.h" -#include "../GapsParameters.h" -#include "../data_structures/Matrix.h" -#include "../math/MatrixMath.h" -#include "../utils/GapsPrint.h" - -#include - -class GapsStatistics; -class Archive; - -class DenseNormalModel -{ -public: - template - DenseNormalModel(const DataType &data, bool transpose, bool subsetRows, - const GapsParameters ¶ms, float alpha, float maxGibbsMass); - template - void setUncertainty(const DataType &unc, bool transpose, bool subsetRows, - const GapsParameters ¶ms); - void setMatrix(const Matrix &mat); - void setAnnealingTemp(float temp); - void sync(const DenseNormalModel &model, unsigned nThreads=1); - void extraInitialization(); - float chiSq() const; - float dataSparsity() const; - friend Archive& operator<<(Archive &ar, const DenseNormalModel &m); - friend Archive& operator>>(Archive &ar, DenseNormalModel &m); -protected: - friend class GapsStatistics; - uint64_t nElements() const; - uint64_t nPatterns() const; - float annealingTemp() const; - float lambda() const; - float maxGibbsMass() const; - bool canUseGibbs(unsigned col) const; - bool canUseGibbs(unsigned c1, unsigned c2) const; - void changeMatrix(unsigned row, unsigned col, float delta); - void safelyChangeMatrix(unsigned row, unsigned col, float delta); - float deltaLogLikelihood(unsigned r1, unsigned c1, unsigned r2, unsigned c2, float mass); - OptionalFloat sampleBirth(unsigned row, unsigned col, GapsRng *rng); - OptionalFloat sampleDeathAndRebirth(unsigned row, unsigned col, float delta, GapsRng *rng); - OptionalFloat sampleExchange(unsigned r1, unsigned c1, float m1, unsigned r2, - unsigned c2, float m2, GapsRng *rng); -//private: // TODO - DenseNormalModel(const DenseNormalModel&); // = delete (no c++11) - DenseNormalModel& operator=(const DenseNormalModel&); // = delete (no c++11) - AlphaParameters alphaParameters(unsigned row, unsigned col); - AlphaParameters alphaParameters(unsigned r1, unsigned c1, unsigned r2, unsigned c2); - AlphaParameters alphaParametersWithChange(unsigned row, unsigned col, float ch); - void updateAPMatrix(unsigned row, unsigned col, float delta); - - Matrix mDMatrix; // samples by genes for A, genes by samples for P - Matrix mMatrix; // genes by patterns for A, samples by patterns for P - const Matrix *mOtherMatrix; // pointer to P if this is A, and vice versa - Matrix mSMatrix; // uncertainty values for each data point - Matrix mAPMatrix; // cached product of A and P - float mMaxGibbsMass; - float mAnnealingTemp; - float mLambda; -}; - -template -DenseNormalModel::DenseNormalModel(const DataType &data, bool transpose, -bool subsetRows, const GapsParameters ¶ms, float alpha, float maxGibbsMass) - : -mDMatrix(data, transpose, subsetRows, params.dataIndicesSubset), -mMatrix(mDMatrix.nCol(), params.nPatterns), -mOtherMatrix(NULL), -mSMatrix(gaps::pmax(mDMatrix, 0.1f)), -mAPMatrix(mDMatrix.nRow(), mDMatrix.nCol()), -mMaxGibbsMass(maxGibbsMass), -mAnnealingTemp(1.f), -mLambda(0.f) -{ - float meanD = gaps::nonZeroMean(mDMatrix); - mLambda = alpha * std::sqrt(nPatterns() / meanD); - mMaxGibbsMass = mMaxGibbsMass / mLambda; - - if (gaps::max(mDMatrix) > 50.f) - { - gaps_printf("\nWarning: Large values detected, is data log transformed?\n"); - } - mSMatrix.pad(1.f); // so that SIMD operations don't divide by zero -} - -template -void DenseNormalModel::setUncertainty(const DataType &unc, bool transpose, -bool subsetRows, const GapsParameters ¶ms) -{ - mSMatrix = Matrix(unc, transpose, subsetRows, params.dataIndicesSubset); - mSMatrix.pad(1.f); // so that SIMD operations don't divide by zero -} - -#endif // __COGAPS_DENSE_STORAGE_POLICY_H__ \ No newline at end of file +#ifndef __COGAPS_DENSE_NORMAL_MODEL_H__ +#define __COGAPS_DENSE_NORMAL_MODEL_H__ + +#include "AlphaParameters.h" +#include "../GapsParameters.h" +#include "../data_structures/Matrix.h" +#include "../math/MatrixMath.h" +#include "../utils/GapsPrint.h" + +#include + +class GapsStatistics; +class Archive; + +class DenseNormalModel +{ +protected: + friend class GapsStatistics; + uint64_t nElements() const; + uint64_t nPatterns() const; + float annealingTemp() const; + float lambda() const; + float maxGibbsMass() const; + bool canUseGibbs(unsigned col) const; + bool canUseGibbs(unsigned c1, unsigned c2) const; + void changeMatrix(unsigned row, unsigned col, float delta); + void safelyChangeMatrix(unsigned row, unsigned col, float delta); + float deltaLogLikelihood(unsigned r1, unsigned c1, unsigned r2, unsigned c2, float mass); + OptionalFloat sampleBirth(unsigned row, unsigned col, GapsRng *rng); + OptionalFloat sampleDeathAndRebirth(unsigned row, unsigned col, float delta, GapsRng *rng); + OptionalFloat sampleExchange(unsigned r1, unsigned c1, float m1, unsigned r2, + unsigned c2, float m2, GapsRng *rng); +//private: // TODO + // P means transpose = TRUE + Matrix mDMatrix; // samples by genes for A, genes by samples for P + Matrix mMatrix; // genes by patterns for A, samples by patterns for P + const Matrix *mOtherMatrix; // pointer to P if this is A, and vice versa + Matrix mSMatrix; // uncertainty values for each data point + Matrix mAPMatrix; // cached product of A and P + //GAPS_ASSERT(mMatrix.nRow() == mAPMatrix.nCol()); + + DenseNormalModel(const DenseNormalModel&); // = delete (no c++11) + DenseNormalModel& operator=(const DenseNormalModel&); // = delete (no c++11) + AlphaParameters alphaParameters(unsigned row, unsigned col); + AlphaParameters alphaParameters(unsigned r1, unsigned c1, unsigned r2, unsigned c2); + AlphaParameters alphaParametersWithChange(unsigned row, unsigned col, float ch); + void updateAPMatrix(unsigned row, unsigned col, float delta); + + float mMaxGibbsMass; + float mAnnealingTemp; + float mLambda; +public: + template + DenseNormalModel(const DataType &data, bool transpose, bool subsetRows, + const GapsParameters ¶ms, float alpha, float maxGibbsMass); + template + void setUncertainty(const DataType &unc, bool transpose, bool subsetRows, + const GapsParameters ¶ms); + void setMatrix(const Matrix &mat); + void setAnnealingTemp(float temp); + void sync(const DenseNormalModel &model); + void extraInitialization(); + float chiSq() const; + float dataSparsity() const; + const Matrix & APMatrix () const + { + return mAPMatrix; + }; + const Matrix & MyMatrix () const + { + return mMatrix; + }; + const Matrix & UMatrix () const + { + return mSMatrix; + }; + friend Archive& operator<<(Archive &ar, const DenseNormalModel &m); + friend Archive& operator>>(Archive &ar, DenseNormalModel &m); +}; + + +template +DenseNormalModel::DenseNormalModel(const DataType &data, bool transpose, +bool subsetRows, const GapsParameters ¶ms, float alpha, float maxGibbsMass) + : +mDMatrix(data, transpose, subsetRows, params.dataIndicesSubset), +mMatrix(mDMatrix.nCol(), params.nPatterns), +mOtherMatrix(NULL), +mSMatrix(mDMatrix.nRow(), mDMatrix.nCol()), +mAPMatrix(mDMatrix.nRow(), mDMatrix.nCol()), +mMaxGibbsMass(maxGibbsMass), +mAnnealingTemp(1.f), +mLambda(0.f) +{ + float meanD = gaps::nonZeroMean(mDMatrix); + float factor=0.1f; //it is like 42 but for variance + + mLambda = alpha * std::sqrt(nPatterns() / meanD); + mMaxGibbsMass = mMaxGibbsMass / mLambda; + + if (gaps::max(mDMatrix) > 50.f) + { + gaps_printf("\nWarning: Large values detected, is data log transformed?\n"); + } + // uncertainty model: relative error S = factor*D, floored at factor so that + // zeros (and any D < 1) get S = factor. This matches the SparseNormalModel + // assumption (S = D for observed / 1 for zero, scaled by mBeta = 1/factor^2), + // so sparseOptimization gives the same result as the dense sampler. + // (mLambda is the atom-size scale used for mMaxGibbsMass only, NOT for S.) + mSMatrix = gaps::pmax(mDMatrix, factor, factor); // = max(factor*D, factor) +} + +template +void DenseNormalModel::setUncertainty(const DataType &unc, bool transpose, +bool subsetRows, const GapsParameters ¶ms) +{ + mSMatrix = Matrix(unc, transpose, subsetRows, params.dataIndicesSubset); + // Only the SIMD padding may be overwritten -- pad() would set *every* element + // to 1.f and so discard the uncertainty the caller passed in. Padding lanes + // get 1.f so that the SIMD loops divide by 1, not by 0. + mSMatrix.padSIMD(1.f); +} + + +#endif // __COGAPS_DENSE_STORAGE_POLICY_H__ diff --git a/src/gibbs_sampler/SingleThreadedGibbsSampler.h b/src/gibbs_sampler/SingleThreadedGibbsSampler.h index d09db3a7..eafa35fb 100755 --- a/src/gibbs_sampler/SingleThreadedGibbsSampler.h +++ b/src/gibbs_sampler/SingleThreadedGibbsSampler.h @@ -41,8 +41,7 @@ class SingleThreadedGibbsSampler : public DataModel float alpha, float maxGibbsMass, const GapsParameters ¶ms, GapsRandomState *randState); unsigned nAtoms() const; - float getAverageQueueLength() const; - void update(unsigned nSteps, unsigned nThreads); + void update(unsigned nSteps); friend Archive& operator<< (Archive &ar, const SingleThreadedGibbsSampler &s); friend Archive& operator>> (Archive &ar, SingleThreadedGibbsSampler &s); private: @@ -57,7 +56,11 @@ class SingleThreadedGibbsSampler : public DataModel uint64_t mNumBins; uint64_t mBinLength; uint64_t mNumPatterns; - double mDomainLength; // length of entire atomic domain + double mdDomainLength; + // doudle length of entire atomic domain + // actually, if we need uint64_t value, we call mDomain.DomainLength() + // the dmDomainLength is not to kill a old field + // double mAlpha; }; @@ -75,7 +78,7 @@ mRng(randState), mNumBins(DataModel::nElements()), mBinLength(std::numeric_limits::max() / (DataModel::nElements())), mNumPatterns(DataModel::nPatterns()), -mDomainLength(mBinLength * DataModel::nElements()), +mdDomainLength(mBinLength * DataModel::nElements()), mAlpha(alpha) {} @@ -85,12 +88,6 @@ unsigned SingleThreadedGibbsSampler::nAtoms() const return mDomain.size(); } -template -float SingleThreadedGibbsSampler::getAverageQueueLength() const -{ - return 0.f; -} - template char SingleThreadedGibbsSampler::getUpdateType() const { @@ -108,8 +105,8 @@ char SingleThreadedGibbsSampler::getUpdateType() const if (u1 < 0.5f) { double nAtoms = static_cast(mDomain.size()); - double numer = nAtoms * mDomainLength; - float deathProb = numer / (numer + mAlpha * mNumBins * (mDomainLength - nAtoms)); + double numer = nAtoms * mdDomainLength; //here, we need double version of the domain length + float deathProb = numer / (numer + mAlpha * mNumBins * (mdDomainLength - nAtoms)); // [AI-generated] Within the birth/death branch, choose death with probability implied // by the atom-count prior; otherwise propose birth. return mRng.uniform() < deathProb ? 'D' : 'B'; @@ -119,7 +116,7 @@ char SingleThreadedGibbsSampler::getUpdateType() const } template -void SingleThreadedGibbsSampler::update(unsigned nSteps, unsigned nThreads) // NOLINT +void SingleThreadedGibbsSampler::update(unsigned nSteps) { // [AI-generated] Sequential sampler path: each proposal is generated, evaluated, and // applied before the next proposal is drawn. @@ -215,8 +212,7 @@ void SingleThreadedGibbsSampler::move() // [AI-generated] Bound the move by neighboring atom positions; use domain endpoints for // edge atoms. uint64_t lbound = hood.hasLeft() ? hood.left->pos() : 0; - uint64_t rbound = hood.hasRight() ? hood.right->pos() : - static_cast(mDomainLength); + uint64_t rbound = hood.hasRight() ? hood.right->pos() : mDomain.DomainLength(); // [AI-generated] Select the new atomic position and map old/new positions to matrix indices. uint64_t pos = mRng.uniform64(lbound + 1, rbound - 1); @@ -283,16 +279,18 @@ template Archive& operator<<(Archive &ar, const SingleThreadedGibbsSampler &s) { operator<<(ar, static_cast(s)) << s.mDomain << s.mNumBins - << s.mBinLength << s.mNumPatterns << s.mDomainLength << s.mAlpha; + << s.mBinLength << s.mNumPatterns << s.mdDomainLength << s.mAlpha; return ar; } template Archive& operator>>(Archive &ar, SingleThreadedGibbsSampler &s) { - operator>>(ar, static_cast(s)) << s.mDomain << s.mNumBins - << s.mBinLength << s.mNumPatterns << s.mDomainLength << s.mAlpha; + operator>>(ar, static_cast(s)) >> s.mDomain >> s.mNumBins + >> s.mBinLength >> s.mNumPatterns >> s.mdDomainLength >> s.mAlpha; return ar; } + + #endif // __COGAPS_SINGLE_THREADED_GIBBS_SAMPLER_H__ diff --git a/src/gibbs_sampler/SparseNegativeBinomialModel.cpp b/src/gibbs_sampler/SparseNegativeBinomialModel.cpp deleted file mode 100755 index e69de29b..00000000 diff --git a/src/gibbs_sampler/SparseNegativeBinomialModel.h b/src/gibbs_sampler/SparseNegativeBinomialModel.h deleted file mode 100755 index e69de29b..00000000 diff --git a/src/gibbs_sampler/SparseNormalModel.cpp b/src/gibbs_sampler/SparseNormalModel.cpp index 6805b613..8d481724 100755 --- a/src/gibbs_sampler/SparseNormalModel.cpp +++ b/src/gibbs_sampler/SparseNormalModel.cpp @@ -16,6 +16,20 @@ #define COUNT_BITS(u) __builtin_popcountll(u) #define GET_FIRST_SET_BIT(u) (__builtin_ffsll(u) - 1) +// The single uncertainty model shared by every calculation in this file. The +// standard deviation is S = factor * max(D, 1) (relative error floored at +// factor, factor = 0.1), matching DenseNormalModel's pmax(D, factor, factor). +// The constant factor is pulled out into mBeta (= 1/factor^2 = 100), so every +// per-entry term below is scaled by mBeta and this helper returns only the +// data-dependent 1/max(D,1)^2. Keeping it in one place guarantees chiSq() and +// all three alphaParameters() use identical uncertainty. NOTE: callers must apply +// the mBeta factor (they already return AlphaParameters(...) * mBeta / chisq * mBeta). +static inline float invSSq(float d) +{ + float sraw = gaps::max(d, 1.f); + return 1.f / (sraw * sraw); +} + void SparseNormalModel::setMatrix(const Matrix &mat) { mMatrix = mat; @@ -26,7 +40,7 @@ void SparseNormalModel::setAnnealingTemp(float temp) mAnnealingTemp = temp; } -void SparseNormalModel::sync(const SparseNormalModel &model, unsigned nThreads) // NOLINT +void SparseNormalModel::sync(const SparseNormalModel &model) { mOtherMatrix = &(model.mMatrix); generateLookupTables(); @@ -40,6 +54,28 @@ void SparseNormalModel::extraInitialization() float SparseNormalModel::chiSq() const { + // Before sync() there is no other factor matrix, so the A*P product is zero. + // Dereferencing the NULL mOtherMatrix here used to segfault, whereas + // DenseNormalModel::chiSq() is safe in the same state (its AP matrix is + // zero-initialised). Return the matching "no fit" chiSq: with A*P = 0 the + // first loop below contributes nothing, and each stored data value D + // contributes (D-0)^2/S^2 = D^2 * invSSq(D), scaled by mBeta. + if (mOtherMatrix == NULL) + { + float chisq = 0.f; + for (unsigned j = 0; j < mDMatrix.nCol(); ++j) + { + SparseIterator<1> it(mDMatrix.getCol(j)); + while (!it.atEnd()) + { + float d = get<1>(it); + chisq += d * d * invSSq(d); + it.next(); + } + } + return chisq * mBeta; + } + float chisq = 0.f; for (unsigned j = 0; j < mDMatrix.nCol(); ++j) { @@ -53,8 +89,12 @@ float SparseNormalModel::chiSq() const while (!it.atEnd()) { float dot = gaps::dot(mMatrix.getRow(j), mOtherMatrix->getRow(it.getIndex())); - float dsq = get<1>(it) * get<1>(it); - chisq += 1 + dot * (dot - 2 * get<1>(it) - dsq * dot) / dsq; + float d = get<1>(it); + float invS2 = invSSq(d); + // the first loop added this entry's A*P^2 at the S=factor (zero) weight; + // correct it to the floored residual (D - A*P)^2 * invS2 for this stored + // non-zero entry: add (D^2 - 2*D*AP)*invS2 + AP^2*(invS2 - 1). + chisq += d * d * invS2 - 2.f * d * dot * invS2 + dot * dot * (invS2 - 1.f); it.next(); } } @@ -182,11 +222,12 @@ AlphaParameters SparseNormalModel::alphaParameters(unsigned row, unsigned col) float v_val = V[v_ndx]; float d_val = data[sparseIndex++]; - // compute terms for s and s_mu - float term1 = v_val / d_val; - float term2 = v_val - term1 / d_val; - s += term1 * term1 - v_val * v_val; - s_mu += term1 + term2 * gaps::dot(mMatrix.getRow(row), + // floored uncertainty (see invSSq); the data value d_val is kept in + // the residual term, only the weight uses the floored S. invS2 = 1/S^2. + float invS2 = invSSq(d_val); + float term2 = v_val * (1.f - invS2); + s += v_val * v_val * (invS2 - 1.f); + s_mu += v_val * d_val * invS2 + term2 * gaps::dot(mMatrix.getRow(row), mOtherMatrix->getRow(v_ndx)); } sparseIndex += COUNT_BITS(d_flags); // skip over any remaining indices @@ -227,11 +268,11 @@ unsigned col, float ch) float v_val = V[v_ndx]; float d_val = data[sparseIndex++]; - // compute terms for s and s_mu - float term1 = v_val / d_val; - float term2 = v_val - term1 / d_val; - s += term1 * term1 - v_val * v_val; - s_mu += term1 + term2 * gaps::dot(mMatrix.getRow(row), + // floored uncertainty (see invSSq) + float invS2 = invSSq(d_val); + float term2 = v_val * (1.f - invS2); + s += v_val * v_val * (invS2 - 1.f); + s_mu += v_val * d_val * invS2 + term2 * gaps::dot(mMatrix.getRow(row), mOtherMatrix->getRow(v_ndx)); s_mu += term2 * mOtherMatrix->operator()(v_ndx, col) * ch; } @@ -278,13 +319,15 @@ unsigned r2, unsigned c2) float v2_val = V2[v_ndx]; float d_val = data[sparseIndex++]; - float d_recip = 1.f / d_val; - float term1 = 1.f - d_recip * d_recip; + // floored uncertainty (see invSSq); term1 = 1 - 1/S^2, and the + // data term is D/S^2 = d_val * invS2 + float invS2 = invSSq(d_val); + float term1 = 1.f - invS2; float v_diff = v1_val - v2_val; float ap = gaps::dot(mMatrix.getRow(r1), mOtherMatrix->getRow(v_ndx)); s -= v_diff * v_diff * term1; - s_mu += v_diff * (ap * term1 + d_recip); + s_mu += v_diff * (ap * term1 + d_val * invS2); } sparseIndex += COUNT_BITS(d_flags); } diff --git a/src/gibbs_sampler/SparseNormalModel.h b/src/gibbs_sampler/SparseNormalModel.h index 6389ac68..f8ab5600 100755 --- a/src/gibbs_sampler/SparseNormalModel.h +++ b/src/gibbs_sampler/SparseNormalModel.h @@ -24,7 +24,7 @@ class SparseNormalModel const GapsParameters ¶ms); void setMatrix(const Matrix &mat); void setAnnealingTemp(float temp); - void sync(const SparseNormalModel &model, unsigned nThreads=1); + void sync(const SparseNormalModel &model); void extraInitialization(); float chiSq() const; float dataSparsity() const; diff --git a/src/math/MatrixMath.cpp b/src/math/MatrixMath.cpp index f236f97f..c5a09f69 100755 --- a/src/math/MatrixMath.cpp +++ b/src/math/MatrixMath.cpp @@ -51,6 +51,7 @@ float gaps::nonZeroMean(const Matrix &mat) } } } + if (nNonZeroes == 0) return 0.f; // all-zero matrix: avoid 0/0 = NaN return sum / static_cast(nNonZeroes); } @@ -68,41 +69,56 @@ float gaps::nonZeroMean(const SparseMatrix &mat) it.next(); } } + if (nNonZeroes == 0) return 0.f; // all-zero matrix: avoid 0/0 = NaN return sum / static_cast(nNonZeroes); } -Matrix gaps::pmax(Matrix mat, float p) +Matrix gaps::pmax(const Matrix & mat, float factor, float min_threshold) { + Matrix rmat(mat.nRow(), mat.nCol()); for (unsigned j = 0; j < mat.nCol(); ++j) { for (unsigned i = 0; i < mat.nRow(); ++i) { - mat(i,j) = gaps::max(mat(i,j) * p, p); + rmat(i,j) = std::max(mat(i,j) * factor, min_threshold); } } - return mat; + // SIMD loops in alphaParameters read past mSize into padding positions; + // set them to a positive value so that the denominator (S^2) is never + // zero there, preventing 0/0 = NaN that silences all sampleBirth() calls + rmat.padSIMD(min_threshold); + return rmat; } -Matrix operator*(Matrix mat, float f) +//overload threshold=factor for back compatibility +Matrix gaps::pmax(const Matrix & mat, float factor) { + return gaps::pmax(mat, factor, factor); +} + + +Matrix operator*(const Matrix & mat, float f) +{ + Matrix rmat(mat.nRow(), mat.nCol()); for (unsigned j = 0; j < mat.nCol(); ++j) { for (unsigned i = 0; i < mat.nRow(); ++i) { - mat(i,j) *= f; + rmat(i,j) = f * mat(i,j); } } - return mat; + return rmat; } -Matrix operator/(Matrix mat, float f) +Matrix operator/(const Matrix & mat, float f) { + Matrix rmat(mat.nRow(), mat.nCol()); for (unsigned j = 0; j < mat.nCol(); ++j) { for (unsigned i = 0; i < mat.nRow(); ++i) { - mat(i,j) /= f; + rmat(i,j) = mat(i,j) / f; } } - return mat; -} \ No newline at end of file + return rmat; +} diff --git a/src/math/MatrixMath.h b/src/math/MatrixMath.h index 0603a545..4dfa28ba 100755 --- a/src/math/MatrixMath.h +++ b/src/math/MatrixMath.h @@ -24,17 +24,19 @@ namespace gaps float sum(const MatrixType &mat); template float mean(const MatrixType &mat); - Matrix pmax(Matrix mat, float p); + Matrix pmax(const Matrix & mat, float f, float min_thr); + Matrix pmax(const Matrix & mat, float f); } // namespace gaps -Matrix operator*(Matrix mat, float f); -Matrix operator/(Matrix mat, float f); +Matrix operator*(const Matrix & mat, float f); +Matrix operator/(const Matrix & mat, float f); template float gaps::min(const MatrixType &mat) { - float mn = 0.f; - for (unsigned i = 0; i < mat.nCol(); ++i) + if (mat.nCol() == 0) return 0.f; // empty matrix + float mn = gaps::min(mat.getCol(0)); + for (unsigned i = 1; i < mat.nCol(); ++i) { float cmin = gaps::min(mat.getCol(i)); // [AI-generated] Keep the smallest column minimum seen so far. @@ -46,8 +48,9 @@ float gaps::min(const MatrixType &mat) template float gaps::max(const MatrixType &mat) { - float mx = 0.f; - for (unsigned i = 0; i < mat.nCol(); ++i) + if (mat.nCol() == 0) return 0.f; // empty matrix + float mx = gaps::max(mat.getCol(0)); + for (unsigned i = 1; i < mat.nCol(); ++i) { float cmax = gaps::max(mat.getCol(i)); // [AI-generated] Keep the largest column maximum seen so far. @@ -73,4 +76,4 @@ float gaps::mean(const MatrixType &mat) return gaps::sum(mat) / (mat.nRow() * mat.nCol()); } -#endif \ No newline at end of file +#endif diff --git a/src/math/Random.cpp b/src/math/Random.cpp index 4fcadcf5..ed67b474 100755 --- a/src/math/Random.cpp +++ b/src/math/Random.cpp @@ -80,6 +80,7 @@ uint32_t GapsRng::uniform32() // inclusive of a and b uint32_t GapsRng::uniform32(uint32_t a, uint32_t b) { + GAPS_ASSERT(a<=b); if (b == a) { return a; @@ -88,6 +89,7 @@ uint32_t GapsRng::uniform32(uint32_t a, uint32_t b) uint32_t x = uniform32(); uint32_t iPart = std::numeric_limits::max() / range; while (x >= range * iPart) + //[a,b] is mapped to intervals of iPart length; everything upper than range * iPart maps to nowhere { x = uniform32(); } @@ -107,6 +109,7 @@ uint64_t GapsRng::uniform64() // inclusive of a and b uint64_t GapsRng::uniform64(uint64_t a, uint64_t b) { + GAPS_ASSERT(a<=b); if (b == a) { return a; @@ -114,7 +117,8 @@ uint64_t GapsRng::uniform64(uint64_t a, uint64_t b) uint64_t range = b + 1 - a; uint64_t x = uniform64(); uint64_t iPart = std::numeric_limits::max() / range; - while (x >= range * iPart) + while (x >= range * iPart) + //[a,b] is mapped to intervals of iPart length; everything upper than range * iPart maps to nowhere { x = uniform64(); } @@ -363,3 +367,4 @@ Archive& operator>>(Archive &ar, GapsRandomState &s) ar >> s.mSeeder; return ar; } + diff --git a/src/math/VectorMath.cpp b/src/math/VectorMath.cpp index 14745937..a085d894 100755 --- a/src/math/VectorMath.cpp +++ b/src/math/VectorMath.cpp @@ -4,7 +4,8 @@ float gaps::min(const Vector &v) { - float mn = 0.f; + if (v.size() == 0) return 0.f; // empty vector + float mn = v[0]; for (unsigned i = 0; i < v.size(); ++i) { // [AI-generated] Keep the smaller value seen so far. @@ -15,7 +16,8 @@ float gaps::min(const Vector &v) float gaps::min(const HybridVector &v) { - float mn = 0.f; + if (v.size() == 0) return 0.f; // empty vector + float mn = v[0]; for (unsigned i = 0; i < v.size(); ++i) { // [AI-generated] Keep the smaller value seen so far. @@ -26,8 +28,9 @@ float gaps::min(const HybridVector &v) float gaps::min(const SparseVector &v) { - float mn = 0.f; SparseIterator<1> it(v); + if (it.atEnd()) return 0.f; // empty (all-zero) sparse vector + float mn = get<1>(it); while (!it.atEnd()) { // [AI-generated] Keep the smaller nonzero sparse value seen so far. @@ -39,8 +42,9 @@ float gaps::min(const SparseVector &v) float gaps::max(const Vector &v) { - float mx = 0.f; - for (unsigned i = 0; i < v.size(); ++i) + if (v.size() == 0) return 0.f; // empty vector + float mx = v[0]; + for (unsigned i = 1; i < v.size(); ++i) { // [AI-generated] Keep the larger value seen so far. mx = (v[i] > mx) ? v[i] : mx; @@ -50,8 +54,9 @@ float gaps::max(const Vector &v) float gaps::max(const HybridVector &v) { - float mx = 0.f; - for (unsigned i = 0; i < v.size(); ++i) + if (v.size() == 0) return 0.f; // empty vector + float mx = v[0]; + for (unsigned i = 1; i < v.size(); ++i) { // [AI-generated] Keep the larger value seen so far. mx = (v[i] > mx) ? v[i] : mx; @@ -61,8 +66,9 @@ float gaps::max(const HybridVector &v) float gaps::max(const SparseVector &v) { - float mx = 0.f; SparseIterator<1> it(v); + if (it.atEnd()) return 0.f; // empty (all-zero) sparse vector + float mx = get<1>(it); while (!it.atEnd()) { // [AI-generated] Keep the larger nonzero sparse value seen so far. @@ -74,8 +80,9 @@ float gaps::max(const SparseVector &v) unsigned gaps::whichMax(const Vector &v) { + if (v.size() == 0) return 0; // empty vector unsigned ndx = 0; - float mx = 0.f; + float mx = v[0]; for (unsigned i = 0; i < v.size(); ++i) { // [AI-generated] Track the index and value of the largest entry seen so far. @@ -134,13 +141,14 @@ bool gaps::isVectorZero(const HybridVector &v) return v.empty(); } -Vector gaps::elementSq(Vector v) +Vector gaps::elementSq(const Vector & v) { + Vector res(v.size()); for (unsigned i = 0; i < v.size(); ++i) { - v[i] *= v[i]; + res[i] = v[i] * v[i]; } - return v; + return res; } Vector operator*(Vector v, float f) @@ -175,11 +183,17 @@ Vector operator/(const HybridVector &hv, float f) return v; } -Vector gaps::pmax(Vector v, float p) +Vector gaps::pmax(const Vector & v, float f, float min_thr) { + Vector res(v.size()); for (unsigned i = 0; i < v.size(); ++i) { - v[i] = gaps::max(v[i] * p, p); + res[i] = std::max(v[i] * f, min_thr); } - return v; -} \ No newline at end of file + res.padSIMD(min_thr); + return res; +} + +Vector gaps::pmax(const Vector & v, float f) { + return (gaps::pmax(v,f,f)); +} diff --git a/src/math/VectorMath.h b/src/math/VectorMath.h index f489969d..e2dda706 100755 --- a/src/math/VectorMath.h +++ b/src/math/VectorMath.h @@ -5,6 +5,7 @@ #include "../data_structures/HybridVector.h" #include "../data_structures/SparseVector.h" #include "../utils/GapsAssert.h" +#include "Math.h" #include "SIMD.h" namespace gaps @@ -21,8 +22,9 @@ namespace gaps float sum(const SparseVector &v); bool isVectorZero(const Vector &v); bool isVectorZero(const HybridVector &v); - Vector elementSq(Vector v); - Vector pmax(Vector v, float p); + Vector elementSq(const Vector &v); + Vector pmax(const Vector & v, float factor, float min_threshold); + Vector pmax(const Vector & v, float factor); template float dot(const VectorType &a, const VectorType &b); template @@ -153,4 +155,4 @@ float gaps::dot_diff(const VectorType &a, const VectorType &b, const VectorType return packedDot.scalar(); } -#endif // __COGAPS_VECTOR_MATH_H__ \ No newline at end of file +#endif // __COGAPS_VECTOR_MATH_H__ diff --git a/src/test-runner.cpp b/src/test-runner.cpp index 0a347900..66bb45bb 100755 --- a/src/test-runner.cpp +++ b/src/test-runner.cpp @@ -8,53 +8,75 @@ #include #include +#include #include #define TESTTHAT_TEST_RUNNER #include +// Configure the shared Catch session: pick the reporter, restrict to a tag or +// test name if one was given, and send the report to a file when `output` is +// non-empty. An empty `output` leaves Catch writing to stdout, which is what +// makes the plain console call still behave the way it always has. +static int runCatchSession(const std::string &tag, const std::string &reporter, +const std::string &output) +{ + Catch::Session& session = testthat::catchSession(); + Catch::ConfigData cfg; + + //resetting what a previous call may have left in the live config; + //catch2 has reporterName rather than reporterNames, here we are inside catch + session.configData().testsOrTags.clear(); + session.configData().reporterNames.clear(); + + if (!tag.empty()) + { + cfg.testsOrTags.push_back(tag); + } + cfg.reporterNames.push_back(reporter); + if (!output.empty()) + { + // Catch opens a FileStream for this; without it the report goes to + // stdout, i.e. straight past R's output handling + cfg.outputFilename = output; + } + session.useConfigData(cfg); + + int numFailed = session.run(); + // [AI-generated] Cap the process exit code at 255, the largest portable single-byte status. + return (numFailed < 0xFF ? numFailed : 0xFF); +} + +// reporter: "console" (human readable, the default) or "xml" (machine readable, +// for tests/testthat/test_cpp.R). +// output: file to write the report to; "" (the default) writes to stdout, which +// is what you want interactively. See src/cpp_tests/README.md. // [[Rcpp::export]] -int run_catch_unit_tests(Rcpp::String reporter="console") +int run_catch_unit_tests(Rcpp::String reporter="console", Rcpp::String output="") { - Catch::Session& session = testthat::catchSession(); - Catch::ConfigData cfg; - //we will use it to write to config as - //session.useConfigData(cfg); - session.configData().testsOrTags.clear(); - //resetting reporterNames - //catch2 has reporterName rather than reporterNames, - //here we are inside catch - session.configData().reporterNames.clear(); - //and write "console" there; - //catch 2 would require, but we are in catch - //write reporter - cfg.reporterNames.push_back(reporter); - session.useConfigData(cfg); - //the next line does not work here.. actually, works only one time - //session.configData().reporterNames.push_back(reporter); - int numFailed = session.run(); - // [AI-generated] Cap the process exit code at 255, the largest portable single-byte status. - return (numFailed < 0xFF ? numFailed : 0xFF); + return runCatchSession("", reporter, output); } // [[Rcpp::export]] -int run_catch_unit_tests_by_tag(Rcpp::String tag="",Rcpp::String reporter="console") +int run_catch_unit_tests_by_tag(Rcpp::String tag="", +Rcpp::String reporter="console", Rcpp::String output="") { - Catch::Session& session = testthat::catchSession(); - Catch::ConfigData cfg; - //we will use it to write to config as - //session.useConfigData(cfg); - //empty testsOrTags - session.configData().testsOrTags.clear(); - //add new tag [tag] - //session.configData().testsOrTags.push_back(tag.get_cstring()); - cfg.testsOrTags.push_back(tag.get_cstring()); - //empty reporternames - session.configData().reporterNames.clear(); - //and write "console" there - //session.configData().reporterNames.push_back("console"); - cfg.reporterNames.push_back(reporter); - session.useConfigData(cfg); - int numFailed = session.run(); - // [AI-generated] Cap the process exit code at 255, the largest portable single-byte status. - return (numFailed < 0xFF ? numFailed : 0xFF); + return runCatchSession(tag, reporter, output); +} + +// Names of every TEST_CASE compiled into the package. Used to tell "all C++ +// tests passed" apart from "no C++ tests were built at all" -- with +// --disable-cpp-tests, or on Windows where Makevars.win lists no cpp_tests +// objects, the suite is empty and would otherwise report success vacuously. +// [[Rcpp::export]] +Rcpp::CharacterVector catch_test_case_names() +{ + std::vector const &all = + Catch::getRegistryHub().getTestCaseRegistry().getAllTests(); + + Rcpp::CharacterVector names(all.size()); + for (unsigned i = 0; i < all.size(); ++i) + { + names[i] = all[i].name; + } + return names; } diff --git a/src/testthat-tweak.h b/src/testthat-tweak.h index 620d44b2..deecbbde 100644 --- a/src/testthat-tweak.h +++ b/src/testthat-tweak.h @@ -17,5 +17,15 @@ #define SECTION CATCH_SECTION #define CHECK CATCH_CHECK #define REQUIRE CATCH_CHECK +#define REQUIRE_THROWS CATCH_CHECK_THROWS + +//from +//# define context(__X__) CATCH_TEST_CASE(__X__ " | " __FILE__) +//# define test_that CATCH_SECTION +//# define expect_true CATCH_CHECK +//# define expect_false CATCH_CHECK_FALSE +//# define expect_error CATCH_CHECK_THROWS +//# define expect_error_as CATCH_CHECK_THROWS_AS + #endif //TESTTHAT_TWEAK_HPP \ No newline at end of file diff --git a/src/utils/GapsAssert.h b/src/utils/GapsAssert.h index f485059d..9899c2f3 100755 --- a/src/utils/GapsAssert.h +++ b/src/utils/GapsAssert.h @@ -3,11 +3,17 @@ #include "GapsPrint.h" +#define GAPS_REAL_ASSERT + +#ifdef GAPS_DEBUG +#define GAPS_REAL_ASSERT +#endif + #ifdef __GAPS_R_BUILD__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +//#pragma GCC diagnostic push +//#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include -#pragma GCC diagnostic pop +//#pragma GCC diagnostic pop #endif #ifndef __GAPS_R_BUILD__ @@ -21,28 +27,31 @@ #define gaps_stop() std::exit(0) #endif -// NOLINTNEXTLINE -#define GAPS_ERROR(msg) do {gaps_cout << "error: " << msg << '\n'; gaps_stop();} while(0) - -#ifdef GAPS_DEBUG - #define GAPS_ASSERT(cond) \ - do { \ - if (!(cond)) \ - { \ - gaps_printf("assert failed %s %d\n", __FILE__, __LINE__); \ - gaps_stop(); \ - } \ - } while(0) - - #define GAPS_ASSERT_MSG(cond, msg) \ - do { \ - if (!(cond)) \ - { \ - gaps_cout << "assert failed " << __FILE__ << " " << \ - __LINE__ << ", " << msg << '\n'; \ - gaps_stop(); \ - } \ - } while(0) +#ifdef GAPS_REAL_ASSERT + + #define GAPS_ERROR(msg) \ + { \ + std::cout << "error: " << msg << '\n' \ + << __FILE__ << __LINE__ << std::flush; \ + gaps_stop(); \ + } + + + #define GAPS_ASSERT(cond) \ + if (!(cond)) \ + { \ + std::cout<< "GAPS assert failed \nat " << \ + __FILE__":" << __LINE__ << '\n' << std::flush; \ + gaps_stop(); \ + } \ + + #define GAPS_ASSERT_MSG(cond, msg) \ + if (!(cond)) \ + { \ + std::cout << msg <<"\nat "<<__FILE__ << ":" << \ + __LINE__ << '\n' << std::flush; \ + gaps_stop(); \ + } #define DEBUG_PING gaps_printf("here %s %d\n", __FILE__, __LINE__); #else @@ -51,4 +60,4 @@ #define DEBUG_PING #endif -#endif // __COGAPS_GAPS_ASSERT_H__ \ No newline at end of file +#endif // __COGAPS_GAPS_ASSERT_H__ diff --git a/src/utils/GlobalConfig.h b/src/utils/GlobalConfig.h index c5549821..99e02c53 100755 --- a/src/utils/GlobalConfig.h +++ b/src/utils/GlobalConfig.h @@ -9,10 +9,6 @@ #define __x86_64__ 1 #endif -#ifdef _OPENMP - #define __GAPS_OPENMP__ -#endif - #ifdef __GAPS_R_BUILD__ #define gaps_check_interrupt(x) Rcpp::checkUserInterrupt(x) #else @@ -44,11 +40,7 @@ inline std::string buildReport() std::string simd = "SIMD not enabled\n"; #endif -#ifdef __GAPS_OPENMP__ - std::string openmp = "Compiled with OpenMP\n"; -#else - std::string openmp = "Compiler did not support OpenMP\n"; -#endif + std::string openmp = "OpenMP: disabled (async sampler removed)\n"; return compiler + simd + openmp; } diff --git a/tests/testthat/test_DistributedCogaps.R b/tests/testthat/test_DistributedCogaps.R index a530ec07..73616535 100644 --- a/tests/testthat/test_DistributedCogaps.R +++ b/tests/testthat/test_DistributedCogaps.R @@ -7,7 +7,9 @@ test_that("featureLoadings and sampleFactors are not all 0s in single-cell", { params <- setDistributedParams(params, nSets = 2) data(GIST) - cg <- CoGAPS(GIST.matrix, params=params) + # distributed CoGAPS wants on-disk data; an in-memory matrix warns. GIST.mtx + # holds the same data as GIST.matrix, so the dimension checks below still hold. + cg <- CoGAPS(system.file("extdata/GIST.mtx", package="CoGAPS"), params=params) featureLoadings <- cg@featureLoadings sampleFactors <- cg@sampleFactors @@ -31,7 +33,9 @@ test_that("featureLoadings and sampleFactors are not all 0s in genome-wide", { params <- setDistributedParams(params, nSets = 2) data(GIST) - cg <- CoGAPS(GIST.matrix, params=params) + # distributed CoGAPS wants on-disk data; an in-memory matrix warns. GIST.mtx + # holds the same data as GIST.matrix, so the dimension checks below still hold. + cg <- CoGAPS(system.file("extdata/GIST.mtx", package="CoGAPS"), params=params) featureLoadings <- cg@featureLoadings sampleFactors <- cg@sampleFactors diff --git a/tests/testthat/test_checkpoints.R b/tests/testthat/test_checkpoints.R index d968a17e..87852e35 100755 --- a/tests/testthat/test_checkpoints.R +++ b/tests/testthat/test_checkpoints.R @@ -1,17 +1,17 @@ -context("CoGAPS") - -test_that("Checkpoint System", -{ - if (CoGAPS::checkpointsEnabled()) - { - data(GIST) - run1 <- CoGAPS(GIST.matrix, checkpointInterval=51, seed=22, - checkpointOutFile="test.out", messages=FALSE, nIterations=100) - run2 <- CoGAPS(GIST.matrix, checkpointInFile="test.out", messages=FALSE, - nIterations=100, seed=33) - file.remove("test.out") - - expect_true(all.equal(run1@featureLoadings, run2@featureLoadings)) - expect_true(all.equal(run1@sampleFactors, run2@sampleFactors)) - } +context("CoGAPS") + +test_that("Checkpoint System", +{ + if (CoGAPS::checkpointsEnabled()) + { + data(GIST) + run1 <- CoGAPS(GIST.matrix, nPatterns=7, checkpointInterval=51, seed=22, + checkpointOutFile="test.out", messages=FALSE, nIterations=100) + run2 <- CoGAPS(GIST.matrix, nPatterns=7, checkpointInFile="test.out", messages=FALSE, + nIterations=100, seed=33) + file.remove("test.out") + + expect_true(all.equal(run1@featureLoadings, run2@featureLoadings)) + expect_true(all.equal(run1@sampleFactors, run2@sampleFactors)) + } }) \ No newline at end of file diff --git a/tests/testthat/test_chisq.R b/tests/testthat/test_chisq.R index 7ed47ad6..901170c9 100644 --- a/tests/testthat/test_chisq.R +++ b/tests/testthat/test_chisq.R @@ -1,17 +1,54 @@ -test_that('chi-square reported by CoGAPS mathches manually calculated (w/uncertainty)',{ +context("CoGAPS") + +# Recompute the mean chi-square directly from the returned mean factor matrices +# and the default uncertainty model, and check it against the value CoGAPS +# reports (GapsStatistics::meanChiSq). +# +# The reconstruction is M = A %*% t(P) where A = featureLoadings (mean of the A +# matrix) and P = sampleFactors (mean of the P matrix). The uncertainty is the +# relative-error model floored at 0.1: S = max(0.1 * D, 0.1). This model is the +# same for the dense and sparse samplers (see DenseNormalModel / SparseNormalModel +# and issues #17/#19), so the identical recomputation must hold in both modes. +manualMeanChiSq <- function(res, D) +{ + A <- res@featureLoadings # genes x patterns (mean of A) + P <- res@sampleFactors # samples x patterns (mean of P) + S <- pmax(0.1 * D, 0.1) + sum(((D - A %*% t(P)) / S)^2) +} + +test_that("chi-square reported by CoGAPS matches manually calculated (w/uncertainty)", +{ + data(GIST) + D <- as.matrix(GIST.matrix) + + # dense sampler + res <- CoGAPS(GIST.matrix, nPatterns=5, nIterations=500, outputFrequency=100, + seed=42, messages=FALSE) + expect_equal(getMeanChiSq(res), manualMeanChiSq(res, D), tolerance=1e-3) + + # sparse sampler -- same uncertainty model, so the same recomputation applies + res_sp <- CoGAPS(GIST.matrix, nPatterns=5, nIterations=500, outputFrequency=100, + seed=42, messages=FALSE, sparseOptimization=TRUE) + expect_equal(getMeanChiSq(res_sp), manualMeanChiSq(res_sp, D), tolerance=1e-3) +}) + +# Explicit user-supplied uncertainty: CoGAPS must use the matrix passed in +# 'uncertainty=' verbatim, so the reported chi-square has to match the same sum +# recomputed against that matrix. +test_that("chi-square reported by CoGAPS matches manually calculated (explicit uncertainty)", +{ data(GIST) - data <- GIST.data_frame - unc <- 0.1*as.matrix(data) - res <- CoGAPS(data, nIterations=1000, uncertainty=unc, nPatterns=3, + D <- GIST.data_frame + unc <- 0.1 * as.matrix(D) + res <- CoGAPS(D, nIterations=1000, uncertainty=unc, nPatterns=3, seed=1, messages=FALSE, sparseOptimization=FALSE) reported <- getMeanChiSq(res) A <- getAmplitudeMatrix(res) P <- getPatternMatrix(res) - M <- A %*% t(P) + calculated <- sum(((D - A %*% t(P)) / unc)^2) - calculated <- sum(((data - M)/unc)^2) - #Set the tolerance to float precision times matrix size - sprintf("difference: %.17f", abs(reported - calculated)) - expect_equal(reported, calculated, tolerance = (1e-7)*prod(dim(data))) + # tolerance: float precision scaled by the number of accumulated terms + expect_equal(reported, calculated, tolerance = (1e-7) * prod(dim(D))) }) diff --git a/tests/testthat/test_cpp.R b/tests/testthat/test_cpp.R index 021e2184..48a805f5 100755 --- a/tests/testthat/test_cpp.R +++ b/tests/testthat/test_cpp.R @@ -1,12 +1,77 @@ -context("C++") - -test_that("Catch unit tests pass", -{ - data(GIST) - gistCsvPath <<- system.file("extdata/GIST.csv", package="CoGAPS") - gistTsvPath <<- system.file("extdata/GIST.tsv", package="CoGAPS") - gistMtxPath <<- system.file("extdata/GIST.mtx", package="CoGAPS") - gistGctPath <<- system.file("extdata/GIST.gct", package="CoGAPS") - run_catch_unit_tests() - expect(TRUE, failure_message="NEVER REACHED") -}) +context("C++") + +# The Catch2 C++ suite is run from here so that a C++ failure fails +# `R CMD check`. The report is written as XML to a temporary file and parsed, so +# that every C++ TEST_CASE shows up as its own testthat expectation instead of +# the whole suite collapsing into a single pass/fail. Writing to a file also +# keeps Catch's output out of stdout -- it is emitted by C++, so testthat cannot +# capture it. +# +# To run the same tests interactively, with the ordinary human-readable output, +# use the console form (see src/README.md): +# +# CoGAPS:::run_catch_unit_tests() +# CoGAPS:::run_catch_unit_tests_by_tag("[vector]") + +test_that("the C++ unit tests are compiled into this build", +{ + cases <- CoGAPS:::catch_test_case_names() + if (length(cases) == 0) + skip(paste("no C++ test cases are registered -- this build was made", + "with --disable-cpp-tests, or on Windows, where", + "src/Makevars.win lists no cpp_tests objects")) + expect_gt(length(cases), 0) +}) + +test_that("Catch unit tests pass", +{ + cases <- CoGAPS:::catch_test_case_names() + if (length(cases) == 0) + skip("no C++ test cases are registered") + + # the C++ file-parser tests read these data paths from the global environment + gistCsvPath <<- system.file("extdata/GIST.csv", package="CoGAPS") + gistTsvPath <<- system.file("extdata/GIST.tsv", package="CoGAPS") + gistMtxPath <<- system.file("extdata/GIST.mtx", package="CoGAPS") + gistGctPath <<- system.file("extdata/GIST.gct", package="CoGAPS") + + reportFile <- tempfile(fileext=".xml") + on.exit(unlink(reportFile), add=TRUE) + + # returns the number of failed assertions (capped at 255); 0 == all pass + nFailed <- CoGAPS:::run_catch_unit_tests(reporter="xml", output=reportFile) + + # xml2 is only a suggested dependency, so fall back to the summary check + if (!requireNamespace("xml2", quietly=TRUE)) + { + expect_equal(nFailed, 0L) + return(invisible(NULL)) + } + + report <- xml2::read_xml(reportFile) + testCases <- xml2::xml_find_all(report, "//TestCase") + expect_equal(length(testCases), length(cases)) + + for (tc in testCases) + { + name <- xml2::xml_attr(tc, "name") + # OverallResult (singular) is the per-case verdict; OverallResults + # (plural) is the assertion tally on sections and on the whole group + verdict <- xml2::xml_find_first(tc, "./OverallResult") + passed <- identical(xml2::xml_attr(verdict, "success"), "true") + + if (!passed) + { + where <- paste0(xml2::xml_attr(tc, "filename"), ":", + xml2::xml_attr(tc, "line")) + fail(paste0("C++ test case failed: ", name, " (", where, ")")) + } + else + { + succeed() + } + } + + # belt and braces: the tally has to agree with the per-case verdicts + expect_equal(nFailed, 0L) +}) diff --git a/tests/testthat/test_deprecated_wrappers.R b/tests/testthat/test_deprecated_wrappers.R new file mode 100644 index 00000000..3f438570 --- /dev/null +++ b/tests/testthat/test_deprecated_wrappers.R @@ -0,0 +1,58 @@ +context("CoGAPS") + +# scCoGAPS() and GWCoGAPS() are deprecated wrappers around +# CoGAPS(..., distributed=). They are still exported, so they have to keep +# working. Neither was called anywhere in the test suite or in a runnable +# example, which is how they came to be broken unnoticed: once nPatterns became +# mandatory, their `params=new("CogapsParams")` default could no longer be +# evaluated, and every call failed -- including calls that passed nPatterns, +# because the wrapper touches `params` before forwarding `...`. + +# nSets cannot be passed through `...` -- it has to go through +# setDistributedParams() -- so the no-params cases rely on the default. +gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") + +test_that("scCoGAPS accepts nPatterns without an explicit params object", +{ + expect_warning(res <- CoGAPS::scCoGAPS(gistMtxPath, nPatterns=2, + nIterations=50, seed=1, messages=FALSE, + BPPARAM=BiocParallel::SerialParam()), "deprecated") + expect_true(is(res, "CogapsResult")) + expect_equal(ncol(res@featureLoadings), 2) +}) + +test_that("GWCoGAPS accepts nPatterns without an explicit params object", +{ + expect_warning(res <- CoGAPS::GWCoGAPS(gistMtxPath, nPatterns=2, + nIterations=50, seed=1, messages=FALSE, + BPPARAM=BiocParallel::SerialParam()), "deprecated") + expect_true(is(res, "CogapsResult")) + expect_equal(ncol(res@featureLoadings), 2) +}) + +test_that("the wrappers still accept an explicit params object", +{ + params <- CogapsParams(nPatterns=2) + params <- setDistributedParams(params, nSets=2) + + expect_warning(res <- CoGAPS::scCoGAPS(gistMtxPath, params=params, + nIterations=50, seed=1, messages=FALSE, + BPPARAM=BiocParallel::SerialParam()), "deprecated") + expect_equal(ncol(res@featureLoadings), 2) +}) + +test_that("the wrappers set the distributed mode they are named after", +{ + params <- CogapsParams(nPatterns=2) + params <- setDistributedParams(params, nSets=2) + + expect_warning(sc <- CoGAPS::scCoGAPS(gistMtxPath, params=params, + nIterations=50, seed=1, messages=FALSE, + BPPARAM=BiocParallel::SerialParam()), "deprecated") + expect_equal(sc@metadata$params@distributed, "single-cell") + + expect_warning(gw <- CoGAPS::GWCoGAPS(gistMtxPath, params=params, + nIterations=50, seed=1, messages=FALSE, + BPPARAM=BiocParallel::SerialParam()), "deprecated") + expect_equal(gw@metadata$params@distributed, "genome-wide") +}) diff --git a/tests/testthat/test_non_zero_A_and_P.R b/tests/testthat/test_non_zero_A_and_P.R new file mode 100644 index 00000000..1208b717 --- /dev/null +++ b/tests/testthat/test_non_zero_A_and_P.R @@ -0,0 +1,17 @@ +context("CoGAPS") + + +test_that("CoGAPS returns non-zero-populated patterns and loadings", +{ + data(GIST) + testDataFrame <- GIST.data_frame + nPat<-5 + res <- CoGAPS(testDataFrame, nPatterns=nPat, nIterations=100, outputFrequency=50, seed=42, messages=TRUE) + expect_equal(nrow(res@featureLoadings), 1363) + expect_equal(ncol(res@featureLoadings), nPat) + expect_equal(nrow(res@sampleFactors), 9) + expect_equal(ncol(res@sampleFactors), nPat) + expect_true(all(apply(res@sampleFactors,2,sum) != 0)) + expect_true(all(apply(res@featureLoadings,2,sum) != 0)) +}) + diff --git a/tests/testthat/test_output_across_modes.R b/tests/testthat/test_output_across_modes.R index ecb749c7..8d20410e 100644 --- a/tests/testthat/test_output_across_modes.R +++ b/tests/testthat/test_output_across_modes.R @@ -12,13 +12,14 @@ test_that("equal A and P dimensions in sparse vs standard", { dim(res_sparse@sampleFactors)) }) +# distributed CoGAPS needs on-disk data (mtx/tsv/csv/gct); passing an in-memory +# matrix warns. Use the packaged GIST.mtx, which is the same data as GIST.data_frame. test_that("equal A and P dimensions in sc vs standard", { data(GIST) + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") res_standard <- CoGAPS(GIST.data_frame, nPatterns=2, nIterations=100, seed=1, messages=FALSE) - params <- CogapsParams(nPatterns=2) - params <- setDistributedParams(params, nSets=2) - res_sc <- CoGAPS(GIST.data_frame, params=params, nIterations=100, seed=1, + res_sc <- CoGAPS(gistMtxPath, nPatterns=2, nIterations=100, seed=1, messages=FALSE, distributed="single-cell") expect_equal(dim(res_standard@featureLoadings), dim(res_sc@featureLoadings)) expect_equal(dim(res_standard@sampleFactors), dim(res_sc@sampleFactors)) @@ -26,11 +27,10 @@ test_that("equal A and P dimensions in sc vs standard", { test_that("equal A and P dimensions in gw vs standard", { data(GIST) + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") res_standard <- CoGAPS(GIST.data_frame, nPatterns=2, nIterations=100, seed=1, messages=FALSE) - params <- CogapsParams(nPatterns=2) - params <- setDistributedParams(params, nSets=2) - res_gw <- CoGAPS(GIST.data_frame, params=params, nIterations=100, seed=1, + res_gw <- CoGAPS(gistMtxPath, nPatterns=2, nIterations=100, seed=1, messages=FALSE, distributed="genome-wide") expect_equal(dim(res_standard@featureLoadings), dim(res_gw@featureLoadings)) expect_equal(dim(res_standard@sampleFactors), dim(res_gw@sampleFactors)) diff --git a/tests/testthat/test_parameters.R b/tests/testthat/test_parameters.R index 17346dff..ec8aaf0e 100755 --- a/tests/testthat/test_parameters.R +++ b/tests/testthat/test_parameters.R @@ -1,8 +1,16 @@ -context("CoGAPS") - -test_that("Npatterns are required input", -{ - data(GIST) - expect_error(CoGAPS(data=GIST.data_frame, nIterations=100, messages=FALSE), "nPatterns") - expect_no_error(CoGAPS(data=GIST.data_frame, nPatterns=7, nIterations=100, messages=FALSE)) +context("CoGAPS") + +test_that("Npatterns are required input", +{ + data(GIST) + expect_error(CoGAPS(data=GIST.data_frame, nIterations=100, messages=FALSE), "nPatterns") + expect_no_error(CoGAPS(data=GIST.data_frame, nPatterns=7, nIterations=100, messages=FALSE)) +}) + +test_that("CogapsParams class", +{ + params<-new("CogapsParams", nPatterns=7) + cat("\n") + print(params) + expect_true("CogapsParams" %in% class(params)) }) \ No newline at end of file diff --git a/tests/testthat/test_patternMarkers.R b/tests/testthat/test_patternMarkers.R index 770facd0..d3137ae0 100644 --- a/tests/testthat/test_patternMarkers.R +++ b/tests/testthat/test_patternMarkers.R @@ -146,10 +146,13 @@ test_that("patternMarkers works with lp", { data(GIST) res <- CoGAPS(GIST.data_frame, nIterations=100, nPatterns=7, seed=1, messages=FALSE, sparseOptimization=TRUE) - expect_no_error(patternMarkers(res, lp=list(my_lp=c(1,0,0,0,0)))) + # lp must have one entry per pattern, i.e. length nPatterns + expect_no_error(patternMarkers(res, lp=list(my_lp=c(1,0,0,0,0,0,0)))) + # wrong length -> warning expect_warning(patternMarkers(res, lp=list(my_lp=c(1,0,0,0)), threshold = "all")) - expect_error(patternMarkers(res, lp=list(my_lp=c(2,0,0,0,0)), + # entry greater than 1 -> error + expect_error(patternMarkers(res, lp=list(my_lp=c(2,0,0,0,0,0,0)), threshold = "all")) }) diff --git a/tests/testthat/test_result_accessors.R b/tests/testthat/test_result_accessors.R new file mode 100644 index 00000000..142f9ee2 --- /dev/null +++ b/tests/testthat/test_result_accessors.R @@ -0,0 +1,127 @@ +context("CoGAPS") + +# Accessors and analysis methods on CogapsResult that the suite did not touch: +# getFeatureLoadings, getVersion, getOriginalParameters, calcZ, reconstructGene, +# binaryA, calcCoGAPSStat, toCSV/fromCSV. + +data(GIST) +# 1000 iterations: with fewer, part of the standard-deviation matrix is still +# exactly zero and calcZ warns about it, which would make these tests noisy. +res <- CoGAPS(GIST.matrix, nPatterns=3, nIterations=1000, seed=1, messages=FALSE) + +test_that("the matrix accessors return the corresponding slots", +{ + expect_identical(getFeatureLoadings(res), res@featureLoadings) + expect_identical(getSampleFactors(res), res@sampleFactors) + expect_identical(getAmplitudeMatrix(res), res@featureLoadings) + expect_identical(getPatternMatrix(res), res@sampleFactors) + + expect_equal(nrow(getFeatureLoadings(res)), nrow(GIST.matrix)) + expect_equal(ncol(getFeatureLoadings(res)), 3) +}) + +test_that("getVersion and getOriginalParameters describe the run", +{ + # getVersion returns the package_version recorded in the result metadata + expect_true(is(getVersion(res), "numeric_version")) + expect_equal(format(getVersion(res)), as.character(packageVersion("CoGAPS"))) + params <- getOriginalParameters(res) + expect_true(is(params, "CogapsParams")) + expect_equal(params@nPatterns, 3) + expect_equal(params@seed, 1) +}) + +test_that("calcZ returns mean/stddev z-scores of the requested matrix", +{ + zA <- calcZ(res, "featureLoadings") + zP <- calcZ(res, "sampleFactors") + expect_equal(dim(zA), dim(res@featureLoadings)) + expect_equal(dim(zP), dim(res@sampleFactors)) + expect_true(all(is.finite(zA))) + expect_true(all(is.finite(zP))) + expect_error(calcZ(res, "notAMatrix"), "whichMatrix") +}) + +test_that("calcZ warns when the standard deviation matrix contains zeros", +{ + # short runs leave exact zeros in the sd matrix, which would divide by zero + zeroed <- res + zeroed@loadingStdDev[1, 1] <- 0 + expect_warning(calcZ(zeroed, "featureLoadings"), "standard deviation") +}) + +test_that("reconstructGene rebuilds rows of the data matrix", +{ + full <- reconstructGene(res) + expect_equal(dim(full), dim(GIST.matrix)) + # the reconstruction is A %*% t(P) + expect_equal(unname(as.matrix(full)), + unname(res@featureLoadings %*% t(res@sampleFactors)), + tolerance=1e-4) + + one <- reconstructGene(res, genes=1:5) + expect_equal(nrow(one), 5) +}) + +test_that("binaryA draws the thresholded amplitude heatmap", +{ + # binaryA is a plotting function -- it thresholds calcZ() of the A matrix and + # draws a heatmap, returning whatever mtext() returns rather than the matrix. + # Regression: it called calcZ(object) without the mandatory whichMatrix, so + # any call failed with 'argument "whichMatrix" is missing'. + # draw into a throwaway file so the run leaves no Rplots.pdf behind + tmp <- tempfile(fileext=".pdf") + pdf(tmp) + on.exit({ dev.off(); unlink(tmp) }, add=TRUE) + expect_error(binaryA(res, threshold=3), NA) +}) + +test_that("calcCoGAPSStat scores a gene set", +{ + genes <- rownames(GIST.matrix) + sets <- list(setA=genes[1:50], setB=genes[51:100]) + stat <- calcCoGAPSStat(res, sets=sets, whichMatrix="featureLoadings", + numPerm=100) + expect_true(is.list(stat) || is.data.frame(stat)) + expect_true(length(stat) > 0) +}) + +test_that("toCSV and fromCSV round-trip a result", +{ + dir <- file.path(tempdir(), "cogaps_csv_roundtrip") + dir.create(dir, showWarnings=FALSE) + on.exit(unlink(dir, recursive=TRUE), add=TRUE) + + toCSV(res, dir) + expect_true(all(file.exists(file.path(dir, + c("featureLoadings.csv", "sampleFactors.csv", + "loadingStdDev.csv", "factorStdDev.csv"))))) + + back <- fromCSV(dir) + expect_true(is(back, "CogapsResult")) + expect_equal(dim(back@featureLoadings), dim(res@featureLoadings)) + expect_equal(dim(back@sampleFactors), dim(res@sampleFactors)) + + # the round-trip must preserve the matrix type of the slots, not hand back + # the data.frame read.csv produces + expect_true(is.matrix(back@featureLoadings)) + expect_true(is.matrix(back@sampleFactors)) + expect_equal(unname(back@featureLoadings), unname(res@featureLoadings), + tolerance=1e-6) + expect_equal(unname(back@sampleFactors), unname(res@sampleFactors), + tolerance=1e-6) +}) + +test_that("show() on CogapsParams works with checkpoint parameters set", +{ + # regression: the checkpointInFile branch of show() referenced a bare + # `checkpointInFile` instead of `object@checkpointInFile`, so printing any + # params object with a checkpoint file set raised "object not found" + p <- CogapsParams(nPatterns=3) + p <- setParam(p, "checkpointInFile", "somewhere.out") + expect_output(show(p), "checkpointInFile") + + p2 <- CogapsParams(nPatterns=3) + p2 <- setParam(p2, "checkpointOutFile", "out.out") + expect_output(show(p2), "checkpointOutFile") +}) diff --git a/tests/testthat/test_seed_consistency.R b/tests/testthat/test_seed_consistency.R index 25efd973..7ce168d7 100755 --- a/tests/testthat/test_seed_consistency.R +++ b/tests/testthat/test_seed_consistency.R @@ -1,70 +1,54 @@ -context("CoGAPS") - -checkCompare <- function(comp) -{ - if (is(comp, "character")) - { - print(comp) - return(FALSE) - } - return(TRUE) -} - -resultsEqual <- function(res1, res2) -{ - checkCompare(all.equal(res1@featureLoadings, res2@featureLoadings, tolerance=0.1)) & - checkCompare(all.equal(res1@loadingStdDev, res2@loadingStdDev, tolerance=0.1)) & - checkCompare(all.equal(res1@sampleFactors, res2@sampleFactors, tolerance=0.1)) & - checkCompare(all.equal(res1@factorStdDev, res2@factorStdDev, tolerance=0.1)) - checkCompare(all.equal(res1@metadata$atomsA, res2@metadata$atomsA)) & - checkCompare(all.equal(res1@metadata$atomsP, res2@metadata$atomsP)) -} - -test_that("same seed == same result", -{ - gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") - - # standard cogaps - res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, - seed=42, messages=FALSE) - res2 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, - seed=42, messages=FALSE) - expect_true(resultsEqual(res1, res2)) - - # distributed cogaps - res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, - seed=42, messages=FALSE, distributed="genome-wide") - res2 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, distributed="genome-wide") - expect_true(resultsEqual(res1, res2)) - - # multiple threads, dense sampler - res1 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=1, - sparseOptimization=FALSE) - res2 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=3, - sparseOptimization=FALSE) - res3 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=6, - sparseOptimization=FALSE) - - expect_true(resultsEqual(res1, res2)) - expect_true(resultsEqual(res1, res3)) - expect_true(resultsEqual(res2, res3)) - - # multiple threads, sparse sampler - res1 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=1, - sparseOptimization=TRUE) - res2 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=3, - sparseOptimization=TRUE) - res3 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, - nPatterns=7, messages=FALSE, nThreads=6, - sparseOptimization=TRUE) - - expect_true(resultsEqual(res1, res2)) - expect_true(resultsEqual(res1, res3)) - expect_true(resultsEqual(res2, res3)) +context("CoGAPS") + +checkCompare <- function(comp) +{ + if (is(comp, "character")) + { + print(comp) + return(FALSE) + } + return(TRUE) +} + +resultsEqual <- function(res1, res2) +{ + checkCompare(all.equal(res1@featureLoadings, res2@featureLoadings, tolerance=0.1)) & + checkCompare(all.equal(res1@loadingStdDev, res2@loadingStdDev, tolerance=0.1)) & + checkCompare(all.equal(res1@sampleFactors, res2@sampleFactors, tolerance=0.1)) & + checkCompare(all.equal(res1@factorStdDev, res2@factorStdDev, tolerance=0.1)) + checkCompare(all.equal(res1@metadata$atomsA, res2@metadata$atomsA)) & + checkCompare(all.equal(res1@metadata$atomsP, res2@metadata$atomsP)) +} + +test_that("same seed == same result", +{ + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") + + # standard cogaps + res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, + seed=42, messages=FALSE) + res2 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, + seed=42, messages=FALSE) + expect_true(resultsEqual(res1, res2)) + + # distributed cogaps + res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, + seed=42, messages=FALSE, distributed="genome-wide") + res2 <- CoGAPS(gistMtxPath, nIterations=100, outputFrequency=10, seed=42, + nPatterns=7, messages=FALSE, distributed="genome-wide") + expect_true(resultsEqual(res1, res2)) + + # seed consistency, dense sampler + res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, seed=42, + messages=FALSE, sparseOptimization=FALSE) + res2 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, seed=42, + messages=FALSE, sparseOptimization=FALSE) + expect_true(resultsEqual(res1, res2)) + + # seed consistency, sparse sampler + res1 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, seed=42, + messages=FALSE, sparseOptimization=TRUE) + res2 <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=10, seed=42, + messages=FALSE, sparseOptimization=TRUE) + expect_true(resultsEqual(res1, res2)) }) \ No newline at end of file diff --git a/tests/testthat/test_subset_data.R b/tests/testthat/test_subset_data.R index d0cb4a47..53345712 100755 --- a/tests/testthat/test_subset_data.R +++ b/tests/testthat/test_subset_data.R @@ -1,85 +1,60 @@ -context("CoGAPS") - -test_that("standard cogaps on a subset of the data", -{ - data(GIST) - subset <- sample(1:nrow(GIST.matrix), 500) - result <- CoGAPS(GIST.matrix, nPatterns=7, nIterations=50, messages=FALSE, seed=42, - subsetIndices=subset, subsetDim=1) - expect_equal(length(subset), nrow(result@featureLoadings)) -}) - -test_that("subsetting data with explicit sets", -{ - gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") - - # distributed cogaps across features - in_sets <- list(1:225, 226:450, 451:675, 676:900) - result <- CoGAPS(gistMtxPath, nPatterns=3, explicitSets=in_sets, - nIterations=200, messages=FALSE, seed=42, distributed="genome-wide") - featureNames <- rownames(result@featureLoadings) - out_sets <- lapply(getSubsets(result), function(set) which(featureNames %in% set)) - expect_true(all(sapply(1:4, function(i) all.equal(out_sets[[i]], in_sets[[i]])))) - - # distributed cogaps across samples - in_sets <- list(1:225, 226:450, 451:675, 676:900) - result <- CoGAPS(gistMtxPath, nPatterns=3, explicitSets=in_sets, seed=42, - nIterations=200, messages=FALSE, distributed="single-cell", - transposeData=TRUE) - sampleNames <- rownames(result@sampleFactors) - out_sets <- lapply(getSubsets(result), function(set) which(sampleNames %in% set)) - expect_true(all(sapply(1:4, function(i) all.equal(out_sets[[i]], in_sets[[i]])))) -}) - -test_that("subsetting data with uniform sets", -{ - gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") - - # distributed cogaps across features - result <- CoGAPS(gistMtxPath, nPatterns=3, nIterations=200, messages=FALSE, - seed=42, distributed="genome-wide") - featureNames <- rownames(result@featureLoadings) - sets <- lapply(getSubsets(result), function(set) which(featureNames %in% set)) - expect_equal(sum(sapply(sets, length)), nrow(result@featureLoadings)) - - # distributed cogaps across samples - result <- CoGAPS(gistMtxPath, nPatterns=3, nIterations=200, messages=FALSE, - seed=42, distributed="single-cell", transposeData=TRUE) - sampleNames <- rownames(result@sampleFactors) - sets <- lapply(getSubsets(result), function(set) which(sampleNames %in% set)) - expect_equal(sum(sapply(sets, length)), nrow(result@sampleFactors)) -}) - -test_that("subsetting data with annotation weights", -{ - # TODO address how weighted sampling works with duplicates, do we need to - # allow passing a value for setSize in this case? - # we should collapse down using the mean - # prevent multiple copies from being in the same set - - #data(GIST) - #gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") -# - ## create annotations - #weight <- c(1, 2, 3) - #names(weight) <- c("A", "B", "C") - #anno <- sample(names(weight), nrow(GIST.matrix), replace=TRUE) - #params <- CogapsParams() - #params <- setAnnotationWeights(params, anno, weight) -# - ## distributed cogaps across features - #result <- CoGAPS(gistMtxPath, params, nPatterns=3, nIterations=200, - # messages=TRUE, seed=42, distributed="genome-wide") - #featureNames <- rownames(result@featureLoadings) - #sets <- lapply(getSubsets(result), function(set) which(featureNames %in% set)) - #expect_equal(nrow(result@featureLoadings), nrow(GIST.matrix)) - #expect_equal(sum(sapply(sets, length)), nrow(result@featureLoadings)) -# - ## distributed cogaps across samples - #result <- CoGAPS(gistMtxPath, params, nPatterns=3, nIterations=200, - # messages=FALSE, seed=42, distributed="single-cell", transposeData=TRUE) - #sampleNames <- rownames(result@sampleFactors) - #sets <- lapply(getSubsets(result), function(set) which(sampleNames %in% set)) - #expect_equal(nrow(result@sampleFactors), nrow(GIST.matrix)) - #expect_equal(sum(sapply(sets, length)), nrow(result@sampleFactors)) -}) \ No newline at end of file +context("CoGAPS") + +test_that("standard cogaps on a subset of the data", +{ + data(GIST) + subset <- sample(1:nrow(GIST.matrix), 500) + result <- CoGAPS(GIST.matrix, nPatterns=7, nIterations=50, messages=FALSE, seed=42, + subsetIndices=subset, subsetDim=1) + expect_equal(length(subset), nrow(result@featureLoadings)) +}) + +test_that("subsetting data with explicit sets", +{ + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") + + # distributed cogaps across features + in_sets <- list(1:225, 226:450, 451:675, 676:900) + result <- CoGAPS(gistMtxPath, nPatterns=3, explicitSets=in_sets, + nIterations=200, messages=FALSE, seed=42, distributed="genome-wide") + featureNames <- rownames(result@featureLoadings) + out_sets <- lapply(getSubsets(result), function(set) which(featureNames %in% set)) + expect_true(all(sapply(1:4, function(i) all.equal(out_sets[[i]], in_sets[[i]])))) + + # distributed cogaps across samples + in_sets <- list(1:225, 226:450, 451:675, 676:900) + result <- CoGAPS(gistMtxPath, nPatterns=3, explicitSets=in_sets, seed=42, + nIterations=200, messages=FALSE, distributed="single-cell", + transposeData=TRUE) + sampleNames <- rownames(result@sampleFactors) + out_sets <- lapply(getSubsets(result), function(set) which(sampleNames %in% set)) + expect_true(all(sapply(1:4, function(i) all.equal(out_sets[[i]], in_sets[[i]])))) +}) + +test_that("subsetting data with uniform sets", +{ + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") + + # distributed cogaps across features + result <- CoGAPS(gistMtxPath, nPatterns=3, nIterations=200, messages=FALSE, + seed=42, distributed="genome-wide") + featureNames <- rownames(result@featureLoadings) + sets <- lapply(getSubsets(result), function(set) which(featureNames %in% set)) + expect_equal(sum(sapply(sets, length)), nrow(result@featureLoadings)) + + # distributed cogaps across samples + result <- CoGAPS(gistMtxPath, nPatterns=3, nIterations=200, messages=FALSE, + seed=42, distributed="single-cell", transposeData=TRUE) + sampleNames <- rownames(result@sampleFactors) + sets <- lapply(getSubsets(result), function(set) which(sampleNames %in% set)) + expect_equal(sum(sapply(sets, length)), nrow(result@sampleFactors)) +}) + +# NOTE: the "subsetting data with annotation weights" test was removed here. +# It asserted behaviour sampleWithAnnotationWeights() does not provide: the +# weighted draw samples with replacement, so genes repeat inside a subset and +# ~56% of genes are never drawn, making nrow(featureLoadings) != nrow(data). +# Fixing that is a change to the distributed subsetting feature, out of scope +# for the uncertainty branch. See +# dev-notes/annotation-weights-sampling-issue-eng.md +# for the quantified defect and a fix sketch. diff --git a/tests/testthat/test_top_level.R b/tests/testthat/test_top_level.R index 9f0f5461..2ef1dc0d 100755 --- a/tests/testthat/test_top_level.R +++ b/tests/testthat/test_top_level.R @@ -1,149 +1,143 @@ -context("CoGAPS") - -no_na_in_result <- function(gapsResult) -{ - sum(is.na(gapsResult@featureLoadings)) + - sum(is.na(gapsResult@loadingStdDev)) + - sum(is.na(gapsResult@sampleFactors)) + - sum(is.na(gapsResult@factorStdDev)) == 0 -} - -test_that("Valid Top-Level CoGAPS Calls", -{ - data(GIST) - testDataFrame <- GIST.data_frame - testMatrix <- GIST.matrix - - gistCsvPath <- system.file("extdata/GIST.csv", package="CoGAPS") - gistTsvPath <- system.file("extdata/GIST.tsv", package="CoGAPS") - gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") - gistGctPath <- system.file("extdata/GIST.gct", package="CoGAPS") - - # data types - res <- list() - res[[1]] <- CoGAPS(testDataFrame, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - res[[2]] <- CoGAPS(testMatrix, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - res[[3]] <- CoGAPS(gistCsvPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - res[[4]] <- CoGAPS(gistTsvPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - res[[5]] <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - res[[6]] <- CoGAPS(gistGctPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) - expect_true(all(sapply(res, no_na_in_result))) - - expect_equal(nrow(res[[1]]@featureLoadings), 1363) - expect_equal(ncol(res[[1]]@featureLoadings), 7) - expect_equal(nrow(res[[1]]@sampleFactors), 9) - expect_equal(ncol(res[[1]]@sampleFactors), 7) -# expect_true(all(sapply(1:5, function(i) -# res[[i]]@featureLoadings == res[[i+1]]@featureLoadings))) -# expect_true(all(sapply(1:5, function(i) -# res[[i]]@sampleFactors == res[[i+1]]@sampleFactors))) - - # transposing data - res <- list() - res[[1]] <- CoGAPS(testDataFrame, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - res[[2]] <- CoGAPS(testMatrix, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - res[[3]] <- CoGAPS(gistCsvPath, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - res[[4]] <- CoGAPS(gistTsvPath, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - res[[5]] <- CoGAPS(gistMtxPath, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - res[[6]] <- CoGAPS(gistGctPath, transposeData=TRUE, nIterations=100, - nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) - expect_true(all(sapply(res, no_na_in_result))) - - expect_equal(nrow(res[[1]]@featureLoadings), 9) - expect_equal(ncol(res[[1]]@featureLoadings), 7) - expect_equal(nrow(res[[1]]@sampleFactors), 1363) - expect_equal(ncol(res[[1]]@sampleFactors), 7) -# expect_true(all(sapply(1:5, function(i) -# res[[i]]@featureLoadings == res[[i+1]]@featureLoadings))) -# expect_true(all(sapply(1:5, function(i) -# res[[i]]@sampleFactors == res[[i+1]]@sampleFactors))) - - # passing uncertainty - expect_error(res <- CoGAPS(testDataFrame, uncertainty=as.matrix(GIST.uncertainty), - nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE), NA) - expect_true(no_na_in_result(res)) - - # multiple threads - expect_error(res <- CoGAPS(testDataFrame, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, nThreads=2), NA) - expect_true(no_na_in_result(res)) - - expect_error(res <- CoGAPS(testDataFrame, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, nThreads=6), NA) - expect_true(no_na_in_result(res)) - - expect_error(res <- CoGAPS(testDataFrame, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, nThreads=12), NA) - expect_true(no_na_in_result(res)) - - # genome-wide CoGAPS - expect_error(res <- CoGAPS(gistTsvPath, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, distributed="genome-wide"), NA) - expect_true(no_na_in_result(res)) - - expect_equal(nrow(res@featureLoadings), 1363) - expect_equal(nrow(res@sampleFactors), 9) - #expect_equal(rownames(res@featureLoadings), rownames(GIST.matrix)) - expect_equal(rownames(res@sampleFactors), colnames(GIST.matrix)) - - expect_error(res <- CoGAPS(gistTsvPath, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, distributed="genome-wide"), NA) - expect_true(no_na_in_result(res)) - - expect_equal(nrow(res@featureLoadings), 1363) - expect_equal(nrow(res@sampleFactors), 9) - - # single-cell CoGAPS - expect_error(res <- CoGAPS(gistCsvPath, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, distributed="single-cell", - transposeData=TRUE), NA) - expect_true(no_na_in_result(res)) - - expect_equal(nrow(res@featureLoadings), 9) - expect_equal(nrow(res@sampleFactors), 1363) - expect_equal(rownames(res@featureLoadings), colnames(GIST.matrix)) -# expect_equal(rownames(res@sampleFactors), rownames(GIST.matrix)) - - expect_error(res <- CoGAPS(gistMtxPath, nIterations=100, nPatterns=7, - outputFrequency=50, seed=1, messages=FALSE, distributed="single-cell", - transposeData=TRUE), NA) - expect_true(no_na_in_result(res)) - - expect_equal(nrow(res@featureLoadings), 9) - expect_equal(nrow(res@sampleFactors), 1363) - - # make sure that "none" gets converted to NULL for distributed - res <- CoGAPS(gistCsvPath, nIterations=100, outputFrequency=100, seed=42, - messages=FALSE, nPatterns=3, distributed="none") - expect_true(is.null(res@metadata$params@distributed)) - - params <- CogapsParams(nPatterns=3) - params <- setParam(params, "distributed", "none") - res <- CoGAPS(gistCsvPath, params=params, nIterations=100, outputFrequency=100, seed=42, - messages=FALSE) - expect_true(is.null(res@metadata$params@distributed)) - - # test using RDS file for parameters - matP <- getSampleFactors(GIST.result) - params <- CogapsParams(nPatterns=ncol(matP), nIterations=175, seed=42, - sparseOptimization=TRUE, distributed="genome-wide", - explicitSets=list(1:200, 201:400, 401:600, 601:800, 801:1000)) - params <- setDistributedParams(params, nSets=5, cut=ncol(matP) + 1) - params <- setFixedPatterns(params, matP, "P") - saveRDS(params, file="temp_params.rds") - - res1 <- CoGAPS(gistMtxPath, params=params) - res2 <- CoGAPS(gistMtxPath, params="temp_params.rds") - file.remove("temp_params.rds") - - expect_true(all(res1@featureLoadings == res2@featureLoadings)) - expect_true(all(res1@loadingStdDev == res2@loadingStdDev)) - expect_true(all(res1@sampleFactors == res2@sampleFactors)) - expect_true(all(res1@factorStdDev== res2@factorStdDev)) -}) - +context("CoGAPS") + +no_na_in_result <- function(gapsResult) +{ + sum(is.na(gapsResult@featureLoadings)) + + sum(is.na(gapsResult@loadingStdDev)) + + sum(is.na(gapsResult@sampleFactors)) + + sum(is.na(gapsResult@factorStdDev)) == 0 +} + +test_that("Valid Top-Level CoGAPS Calls", +{ + data(GIST) + testDataFrame <- GIST.data_frame + testMatrix <- GIST.matrix + + gistCsvPath <- system.file("extdata/GIST.csv", package="CoGAPS") + gistTsvPath <- system.file("extdata/GIST.tsv", package="CoGAPS") + gistMtxPath <- system.file("extdata/GIST.mtx", package="CoGAPS") + gistGctPath <- system.file("extdata/GIST.gct", package="CoGAPS") + + # data types + res <- list() + res[[1]] <- CoGAPS(testDataFrame, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + res[[2]] <- CoGAPS(testMatrix, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + res[[3]] <- CoGAPS(gistCsvPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + res[[4]] <- CoGAPS(gistTsvPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + res[[5]] <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + res[[6]] <- CoGAPS(gistGctPath, nPatterns=7, nIterations=100, outputFrequency=50, seed=1, messages=FALSE) + expect_true(all(sapply(res, no_na_in_result))) + + expect_equal(nrow(res[[1]]@featureLoadings), 1363) + expect_equal(ncol(res[[1]]@featureLoadings), 7) + expect_equal(nrow(res[[1]]@sampleFactors), 9) + expect_equal(ncol(res[[1]]@sampleFactors), 7) + # every supported input format must reproduce the same factorisation + expect_true(all(sapply(1:5, function(i) + res[[i]]@featureLoadings == res[[i+1]]@featureLoadings))) + expect_true(all(sapply(1:5, function(i) + res[[i]]@sampleFactors == res[[i+1]]@sampleFactors))) + + # transposing data + res <- list() + res[[1]] <- CoGAPS(testDataFrame, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + res[[2]] <- CoGAPS(testMatrix, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + res[[3]] <- CoGAPS(gistCsvPath, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + res[[4]] <- CoGAPS(gistTsvPath, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + res[[5]] <- CoGAPS(gistMtxPath, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + res[[6]] <- CoGAPS(gistGctPath, transposeData=TRUE, nIterations=100, + nPatterns=7, outputFrequency=50, seed=1, messages=FALSE) + expect_true(all(sapply(res, no_na_in_result))) + + expect_equal(nrow(res[[1]]@featureLoadings), 9) + expect_equal(ncol(res[[1]]@featureLoadings), 7) + expect_equal(nrow(res[[1]]@sampleFactors), 1363) + expect_equal(ncol(res[[1]]@sampleFactors), 7) + # every supported input format must reproduce the same factorisation + expect_true(all(sapply(1:5, function(i) + res[[i]]@featureLoadings == res[[i+1]]@featureLoadings))) + expect_true(all(sapply(1:5, function(i) + res[[i]]@sampleFactors == res[[i+1]]@sampleFactors))) + + # passing uncertainty + expect_error(res <- CoGAPS(testDataFrame, nPatterns=7, uncertainty=as.matrix(GIST.uncertainty), + nIterations=100, outputFrequency=50, seed=1, messages=FALSE), NA) + expect_true(no_na_in_result(res)) + + # genome-wide CoGAPS + expect_error(res <- CoGAPS(gistTsvPath, nPatterns=7, nIterations=100, + outputFrequency=50, seed=1, messages=FALSE, distributed="genome-wide"), NA) + expect_true(no_na_in_result(res)) + + expect_equal(nrow(res@featureLoadings), 1363) + expect_equal(nrow(res@sampleFactors), 9) + #expect_equal(rownames(res@featureLoadings), rownames(GIST.matrix)) + expect_equal(rownames(res@sampleFactors), colnames(GIST.matrix)) + + expect_error(res <- CoGAPS(gistTsvPath, nPatterns=7, nIterations=100, + outputFrequency=50, seed=1, messages=FALSE, distributed="genome-wide"), NA) + expect_true(no_na_in_result(res)) + + expect_equal(nrow(res@featureLoadings), 1363) + expect_equal(nrow(res@sampleFactors), 9) + + # single-cell CoGAPS + expect_error(res <- CoGAPS(gistCsvPath, nPatterns=7, nIterations=100, + outputFrequency=50, seed=1, messages=FALSE, distributed="single-cell", + transposeData=TRUE), NA) + expect_true(no_na_in_result(res)) + + expect_equal(nrow(res@featureLoadings), 9) + expect_equal(nrow(res@sampleFactors), 1363) + expect_equal(rownames(res@featureLoadings), colnames(GIST.matrix)) + # Known gap (pre-existing, not covered by this branch): in distributed + # single-cell runs with transposeData=TRUE the sampleFactors rownames come + # back as the generated "Gene_1", "Gene_2", ... instead of the real names in + # rownames(GIST.matrix). The names are the right length but the wrong values, + # so the assertion below cannot be enabled until the dimnames are threaded + # through stitchTogether() for the non-fixed axis. + + expect_error(res <- CoGAPS(gistMtxPath, nPatterns=7, nIterations=100, + outputFrequency=50, seed=1, messages=FALSE, distributed="single-cell", + transposeData=TRUE), NA) + expect_true(no_na_in_result(res)) + + expect_equal(nrow(res@featureLoadings), 9) + expect_equal(nrow(res@sampleFactors), 1363) + + # make sure that "none" gets converted to NULL for distributed + res <- CoGAPS(gistCsvPath, nIterations=100, outputFrequency=100, seed=42, + messages=FALSE, nPatterns=3, distributed="none") + expect_true(is.null(res@metadata$params@distributed)) + + params <- CogapsParams(nPatterns=3) + params <- setParam(params, "distributed", "none") + res <- CoGAPS(gistCsvPath, params=params, nIterations=100, outputFrequency=100, seed=42, + messages=FALSE) + expect_true(is.null(res@metadata$params@distributed)) + + # test using RDS file for parameters + matP <- getSampleFactors(GIST.result) + params <- CogapsParams(nPatterns=ncol(matP), nIterations=175, seed=42, + sparseOptimization=TRUE, distributed="genome-wide", + explicitSets=list(1:200, 201:400, 401:600, 601:800, 801:1000)) + params <- setDistributedParams(params, nSets=5, cut=ncol(matP) + 1) + params <- setFixedPatterns(params, matP, "P") + saveRDS(params, file="temp_params.rds") + + res1 <- CoGAPS(gistMtxPath, params=params) + res2 <- CoGAPS(gistMtxPath, params="temp_params.rds") + file.remove("temp_params.rds") + + expect_true(all(res1@featureLoadings == res2@featureLoadings)) + expect_true(all(res1@loadingStdDev == res2@loadingStdDev)) + expect_true(all(res1@sampleFactors == res2@sampleFactors)) + expect_true(all(res1@factorStdDev== res2@factorStdDev)) +}) +