Add the ability to decode the chunk layout

.gitignore

@@ -1 +1,2 @@
 /.cache/
+/subprojects/.wraplock

bin/mzp-inspect.cpp

@@ -146,6 +146,24 @@ int print_fmd_kv(MzPeak::Index& index,
   return 0;
 }
 
+/******************************************************************************/
+int dump_spectra(MzPeak::Index& index)
+{
+  auto spectra = index.spectra();
+
+  for (std::size_t spectrum_index : std::views::iota(0ul, spectra.size())) {
+    const auto& spectrum = spectra[spectrum_index];
+    const auto& mz = spectrum.mz();
+    const auto& intensity = spectrum.intensity();
+
+    for (std::size_t row : std::views::iota(0ul, mz.size())) {
+      std::println("{},{:.5f},{:.5f}", spectrum_index, mz[row], intensity[row]);
+    }
+  }
+
+  return 0;
+}
+
 /******************************************************************************/
 int main(int argc, char* argv[])
 {
@@ -169,6 +187,8 @@ int main(int argc, char* argv[])
     desc.add_options()("fmd-key", po::value<std::string>(),
                        "Used with --fmdkv to print the value of the given key");
 
+    desc.add_options()("spectra", "Print all m/z and intensity values");
+
     po::positional_options_description pops;
     pops.add("file", 1);
 
@@ -204,6 +224,8 @@ int main(int argc, char* argv[])
       }
 
       return print_fmd_kv(index, vmap["fmdkv"].as<std::string>(), key);
+    } else if (vmap.count("spectra")) {
+      dump_spectra(index);
     } else {
       std::println("WARN: no command given");
       return 1;

include/mzpeak/data/array_index.h

@@ -30,6 +30,23 @@ using namespace MzPeak::Schema;
  */
 class ArrayIndex final {
 public:
+  /**
+   * How dimensions in the Parquet file are encoded.
+   */
+  enum class Layout {
+    /// Point layout uses one column per dimension.
+    Point,
+
+    /// Chunked layout differentiates between the main axis and
+    /// secondary axes.  Finding the correct column requires the
+    /// `buffer_format` member of the `Entry`.
+    Chunked,
+
+    /// The Parquet file uses an unknown layout and must be decoded
+    /// manually.
+    Unknown,
+  };
+
   /**
    * A type to describe each entry in the index.
    */
@@ -88,6 +105,14 @@ class ArrayIndex final {
     /// denoted as a CURIE from the PSI-MS controlled vocabulary. Some
     /// values are only usable with the chunked layout.
     std::optional<Schema::PSI::Transform> transform = {};
+
+    /// Return `true` if this entry needs to be projected in a query
+    /// in order to properly decode the dimension it represents.
+    bool needed_for_decoding() const;
+
+    /// Return `true` if this entry stores values for the associated
+    /// dimension.
+    bool is_value_entry() const;
   };
 
   /**
@@ -109,11 +134,23 @@ class ArrayIndex final {
     /// The index entries that make up this dimension.
     std::vector<Entry> entries;
 
+    /// Is this dimension on the main axis?
+    bool is_main_axis() const;
+
     /// Does this dimension need a delta model for decoding?
     bool needs_delta_model() const;
 
     /// Return an associated Util::Type or throw an exception.
     Util::Type type_or_throw() const;
+
+    /// Return the entry that holds the (possibly encoded) values for
+    /// this dimension.  Throws an exception of the dimension is
+    /// malformed and thus doesn't include any of the expected
+    /// entries.
+    const Entry& values_entry() const;
+
+    /// Find the first entry with the given buffer format.
+    std::optional<Entry> entry_with(BufferFormat) const;
   };
 
   /// Default constructor.
@@ -135,6 +172,11 @@ class ArrayIndex final {
    */
   const std::string& prefix() const;
 
+  /**
+   * Return the file layout.
+   */
+  Layout layout() const;
+
   /**
    * Get a list of entry definitions.
    */
@@ -162,11 +204,14 @@ class ArrayIndex final {
 
 private:
   // The entity type for the entire Parquet file.
-  EntityType entity_type_;
+  EntityType entity_type_ = EntityType::Other;
 
   // Root node.
   std::string prefix_ = "point";
 
+  // Layout.
+  Layout layout_ = Layout::Unknown;
+
   // Entries;
   std::vector<Entry> entries_;
 

include/mzpeak/data/encoding.h

@@ -15,8 +15,9 @@ top-level directory of this repository.
 #include "mzpeak/data/array_index.h"
 #include "mzpeak/data/null_marking.h"
 #include "mzpeak/data/signals.h"
+#include "mzpeak/data/transformer/primary.h"
+#include "mzpeak/data/transformer/secondary.h"
 #include "mzpeak/exception.h"
-#include "mzpeak/schema/group.h"
 #include "mzpeak/schema/psi/data_type.h"
 #include "mzpeak/util/slice.h"
 #include "mzpeak/util/types.h"
@@ -57,7 +58,9 @@ template <typename T> class Decoder {
   void decode(const ArrayIndex::Dimension&, std::vector<V>&) const;
 
   template <typename N, typename V>
-  void point(const Schema::Column&, const N& null_decoder, std::vector<V>&) const;
+  void decode_with_nulls(const ArrayIndex::Dimension&,
+                         const N& null_decoder,
+                         std::vector<V>&) const;
 
   template <Util::Type From, typename V>
   void remap(const ArrayIndex::Dimension& dim, std::vector<V>& v) const;
@@ -129,42 +132,57 @@ template <typename T>
 template <typename V>
 void Decoder<T>::decode(const ArrayIndex::Dimension& dim, std::vector<V>& v) const
 {
-  const auto& entries = dim.entries;
-
-  if (entries.empty()) {
-    std::string msg("unable to decode dimension, wrong encoding: ");
-    throw ParquetError(msg + dim.name);
-  } else if (entries.size() == 1 &&
-             entries[0].buffer_format == Schema::BufferFormat::Point) {
-
-    auto field =
-        signals_->array_index()->entry_column(*signals_->groups(), entries[0]);
-
-    if (!field.has_value()) {
-      throw ParquetError("unable to decode dimension, not in schema: " + dim.name);
-    }
-
+  switch (signals_->array_index()->layout()) {
+  case ArrayIndex::Layout::Point:
+  case ArrayIndex::Layout::Chunked:
     if (dim.needs_delta_model()) {
       using N = NullMarking::Decoder<V, T>;
-      point<N, V>(field.value(), N{delta_estimator_}, v);
+      decode_with_nulls<N, V>(dim, N{delta_estimator_}, v);
     } else {
       using N = Util::Decoders::NullToZero<V>;
-      point<N, V>(field.value(), N{}, v);
+      decode_with_nulls<N, V>(dim, N{}, v);
     }
-  } else {
-    throw("not implemented");
-    // return decode_chunked(arrays);
+    break;
+
+  case ArrayIndex::Layout::Unknown:
+    throw UnknownLayoutError("cannot decode dimension: " + dim.name);
   }
 }
 
 /******************************************************************************/
 template <typename T>
 template <typename N, typename V>
-void Decoder<T>::point(const Schema::Column& col,
-                       const N& null_decoder,
-                       std::vector<V>& v) const
+void Decoder<T>::decode_with_nulls(const ArrayIndex::Dimension& dim,
+                                   const N& null_decoder,
+                                   std::vector<V>& v) const
 {
-  slice_->array(col, v, Util::Decoders::Scalar<V, std::vector<V>, N>(null_decoder));
+  const auto& primary_entry = dim.values_entry();
+  auto col = signals_->column(primary_entry);
+
+  if (!col.has_value()) {
+    throw ParquetError("unable to decode dimension, not in schema: " + dim.name);
+  }
+
+  auto go = [&](auto&& decoder) -> void { slice_->array(col.value(), v, decoder); };
+
+  if (primary_entry.buffer_format == Schema::BufferFormat::Point) {
+    auto decoder = Util::Decoders::Scalar<V, std::vector<V>, N>(null_decoder);
+    go(decoder);
+  } else {
+    if (dim.is_main_axis()) {
+      using Transformer = Transformer::Primary::Decoder<V>;
+      Transformer transformer(signals_, slice_, dim);
+      auto decoder = Util::Decoders::Flattened<V, std::vector<V>, N, Transformer>(
+          null_decoder, std::move(transformer));
+      go(decoder);
+    } else {
+      using Transformer = Transformer::Secondary::Decoder<V>;
+      Transformer transformer(dim);
+      auto decoder = Util::Decoders::Flattened<V, std::vector<V>, N, Transformer>(
+          null_decoder, std::move(transformer));
+      go(decoder);
+    }
+  }
 }
 
 } // namespace MzPeak::Data::Encoding

include/mzpeak/data/signals.h

@@ -52,11 +52,25 @@ class Signals {
                                       const Util::Query&);
 
   /**
-   * Low-level interface for accessing a group field given its name.
+   * Low-level interface for accessing a column given its name.
    *
    * Useful if you need to manually construct queries.
    */
-  std::optional<Schema::Column> field(const std::string_view&) const;
+  std::optional<Schema::Column> column(const std::string_view&) const;
+
+  /**
+   * Low-level interface for accessing a column given an array index entry.
+   */
+  std::optional<Schema::Column> column(const ArrayIndex::Entry&) const;
+
+  /**
+   * Low-level interface for accessing a column given a dimension and
+   * a buffer format.
+   *
+   * Returns the first matching column.
+   */
+  std::optional<Schema::Column> column(const ArrayIndex::Dimension&,
+                                       Schema::BufferFormat) const;
 
   /**
    * Low-level interface for accessing the schema encoded as a map of