More agressive memory managment

include/mzpeak/data/encoding.h

@@ -122,7 +122,6 @@ void Decoder<T>::remap(const ArrayIndex::Dimension& dim, std::vector<V>& v) cons
   } else {
     std::vector<F> tmp;
     decode<F>(dim, tmp);
-    v.reserve(tmp.size());
     v.insert(v.end(), tmp.begin(), tmp.end());
   }
 }
@@ -161,6 +160,8 @@ void Decoder<T>::decode_with_nulls(const ArrayIndex::Dimension& dim,
 
   if (!col.has_value()) {
     throw ParquetError("unable to decode dimension, not in schema: " + dim.name);
+  } else if (!slice_->has_column(col.value())) {
+    return; // No data to decode so we can exit early.
   }
 
   auto go = [&](auto&& decoder) -> void { slice_->array(col.value(), v, decoder); };

include/mzpeak/data/null_marking.h

@@ -171,7 +171,8 @@ std::optional<T> Decoder<T, U>::operator()(int64_t index)
     prior_.index = index;
 
     if (range.size() == 1) {
-      prior_.value = array_->Value(range.begin);
+      prior_.value =
+          Decoders::unsafe_array_value<enum_type_v<T>>(array_, range.begin);
       prior_.delta = estimator_.predict(prior_.value);
     } else {
       auto slice = array_->Slice(range.begin, range.size());
@@ -182,7 +183,8 @@ std::optional<T> Decoder<T, U>::operator()(int64_t index)
       Decoders::Scalar<T> decoder;
       decoder.decode(slice, values);
 
-      prior_.value = array_->Value(range.anchor(index));
+      prior_.value =
+          Decoders::unsafe_array_value<enum_type_v<T>>(array_, range.anchor(index));
       prior_.delta = Algorithm::median_delta(values, zero_);
     }
 

include/mzpeak/data/signals.h

@@ -56,7 +56,7 @@ class Signals {
    *
    * Useful if you need to manually construct queries.
    */
-  std::optional<Schema::Column> column(const std::string_view&) const;
+  std::optional<Schema::Column> column(std::string_view) const;
 
   /**
    * Low-level interface for accessing a column given an array index entry.

include/mzpeak/data/transformer/primary.h

@@ -71,11 +71,11 @@ Decoder<T>::Decoder(std::shared_ptr<Signals> signals,
       throw InvalidFormatError(msg);
     }
 
-    slice->array(*column, dest, Util::Decoders::Scalar<U>());
+    slice->array(column.value(), dest, Util::Decoders::Scalar<U>());
   };
 
   // Decode the `chunk_encoding` column.
-  std::vector<std::string_view> encodings;
+  std::vector<std::string> encodings;
   decode(Schema::BufferFormat::ChunkEncoding, encodings);
   chunk_encoding_.reserve(encodings.size());
 
@@ -86,7 +86,7 @@ Decoder<T>::Decoder(std::shared_ptr<Signals> signals,
       throw InvalidFormatError("invalid chunk encoding CV: " + std::string(s));
     }
 
-    chunk_encoding_.emplace_back(*cv);
+    chunk_encoding_.emplace_back(cv.value());
   }
 
   // Decode the `chunk_start` column.

include/mzpeak/index.h

@@ -9,6 +9,7 @@ directory of this repository.
 #pragma once
 
 #include <memory>
+#include <string_view>
 #include <vector>
 
 namespace MzPeak {
@@ -43,7 +44,7 @@ class Index {
   /**
    * Find a file in the mzPeak archive with the given name.
    */
-  std::vector<Schema::File>::const_iterator find(const std::string_view&) const;
+  std::vector<Schema::File>::const_iterator find(std::string_view) const;
 
   /**
    * Access the spectra in the file.

include/mzpeak/metadata/table.h

@@ -28,10 +28,9 @@ class Table final {
   ~Table();
 
   /**
-   * Return a group with the given name.  If the group does not
-   * exist in the schema return `nullptr`.
+   * Return a group with the given name.
    */
-  std::shared_ptr<Schema::Group> group(const std::string_view&) const;
+  std::shared_ptr<Schema::Group> group(std::string_view) const;
 
   /**
    * Read all rows from the given group where the index column

include/mzpeak/schema/buffer_format.h

@@ -61,6 +61,6 @@ std::string buffer_format_to_string(BufferFormat);
 /**
  * Parse an BufferFormat from a string.
  */
-BufferFormat buffer_format_from_string(const std::string_view&);
+BufferFormat buffer_format_from_string(std::string_view);
 
 } // namespace MzPeak::Schema

include/mzpeak/schema/cv.h

@@ -19,17 +19,17 @@ namespace MzPeak::Schema {
 class CV {
 public:
   /// Constructor.
-  CV(const std::string_view& code, const std::string_view& accession)
-      : code_(std::move(code))
-      , accession_(std::move(accession))
+  CV(std::string_view code, std::string_view accession)
+      : code_(code)
+      , accession_(accession)
   {
   }
 
   /// Destructor.
   ~CV() = default;
 
   /// Parse a string like "MS:1000511"
-  static std::optional<CV> from_string(const std::string_view&);
+  static std::optional<CV> from_string(std::string_view);
 
   /// Convert this CV term to a string like "MS:1000511"
   std::string to_string() const;

include/mzpeak/schema/data_kind.h

@@ -46,6 +46,6 @@ std::string data_kind_to_string(DataKind);
 /**
  * Parse a DataKind from a string view.
  */
-DataKind data_kind_from_string(const std::string_view&);
+DataKind data_kind_from_string(std::string_view);
 
 } // namespace MzPeak::Schema

include/mzpeak/schema/entity_type.h

@@ -38,6 +38,6 @@ std::string entity_type_to_string(EntityType);
 /**
  * Parse an EntityType from a string view.
  */
-EntityType entity_type_from_string(const std::string_view&);
+EntityType entity_type_from_string(std::string_view);
 
 } // namespace MzPeak::Schema

include/mzpeak/schema/group.h

@@ -73,7 +73,7 @@ class Group final {
     };
 
     /// Constructor from an encoded column name.
-    explicit Field(const std::string_view& column_name,
+    explicit Field(std::string_view column_name,
                    index_type rel_index,
                    index_type abs_index);
 
@@ -171,7 +171,7 @@ class Group final {
    * NOTE: For metadata groups this is the cleaned name, not the raw
    * schema node name.
    */
-  std::optional<std::shared_ptr<const Field>> field(const std::string_view&&) const;
+  std::optional<std::shared_ptr<const Field>> field(std::string_view) const;
 
   /**
    * Find a field given its CV type.

include/mzpeak/schema/psi/array_type.h

@@ -150,6 +150,6 @@ std::string array_type_to_string(ArrayType);
 /**
  * Parse an ArrayType from a string.
  */
-ArrayType array_type_from_string(const std::string_view&);
+ArrayType array_type_from_string(std::string_view);
 
 } // namespace MzPeak::Schema::PSI

include/mzpeak/util/algorithm.h

@@ -16,6 +16,7 @@ top-level directory of this repository.
 #include <ranges>
 #include <vector>
 
+#include "mzpeak/util/decoders.h"
 #include "mzpeak/util/types.h"
 
 namespace MzPeak::Util::Algorithm {
@@ -152,7 +153,7 @@ null_delta_decode(typename type_traits<T>::value_type start,
 
   for (int64_t index : std::views::iota(0, length)) {
     if (casted->IsValid(index)) {
-      ValueType delta = casted->Value(index);
+      ValueType delta = Decoders::unsafe_array_value<T>(casted, index);
       last = last.value_or(zero) + delta;
       append(last);
     } else {

include/mzpeak/util/decoders.h

@@ -43,6 +43,29 @@ concept from_arrow_array =
       { t.decode(a, r) } -> std::same_as<void>;
     };
 
+/******************************************************************************/
+/**
+ * Read a value from an array without checking bounds or if it is NULL.
+ *
+ * This is needed because Parquet/Arrow uses std::string_view for byte
+ * arrays, but that means that the original arrow array needs to
+ * remain resident in memory.  Therefore we need to copy the memory
+ * referenced by a std::string_view into a std::string.
+ */
+template <Type T>
+type_traits<T>::value_type
+unsafe_array_value(const std::shared_ptr<typename type_traits<T>::array_type>& ary,
+                   int64_t index)
+{
+  using A = type_traits<T>::array_type;
+
+  if constexpr (std::is_same_v<A, arrow::StringArray>) {
+    return ary->GetString(index);
+  } else {
+    return ary->Value(index);
+  }
+}
+
 /******************************************************************************/
 /**
  * If the given array is a "list of lists" then visit each element of
@@ -177,7 +200,8 @@ class Scalar final : Helper<Scalar<V, C, N>> {
         std::optional<V> value = std::invoke(null_decoder_, i);
         if (value.has_value()) this->push(dst, std::move(*value));
       } else {
-        this->push(dst, std::move(casted->Value(i)));
+        V value = unsafe_array_value<enum_type_v<V>>(casted, i);
+        this->push(dst, std::move(value));
       }
     }
   }

include/mzpeak/util/manager.h

@@ -9,6 +9,8 @@ directory of this repository.
 #pragma once
 
 #include <memory>
+#include <string_view>
+#include <vector>
 
 #include "mzpeak/io/archive.h"
 #include "mzpeak/schema/file.h"
@@ -17,7 +19,7 @@ directory of this repository.
 namespace MzPeak::Util {
 
 /**
- * FIXME: Write documentation!
+ * File manager for parquet files.
  */
 class Manager final {
 public:
@@ -32,8 +34,7 @@ class Manager final {
   /**
    * Find a file given its name.
    */
-  std::vector<Schema::File>::const_iterator
-  find_file(const std::string_view& name) const;
+  std::vector<Schema::File>::const_iterator find_file(std::string_view name) const;
 
   /**
    * Open a Parquet file from the mzPeak archive.

include/mzpeak/util/parquet.h

@@ -45,8 +45,7 @@ class Parquet final {
   /**
    * Return a Group and Field matching the given names.
    */
-  std::optional<Schema::Column> field(const std::string_view&,
-                                      const std::string_view&) const;
+  std::optional<Schema::Column> field(std::string_view, std::string_view) const;
 
   /**
    * Access the file metadata.
@@ -57,13 +56,13 @@ class Parquet final {
    * Fetch a string value from the metadata key-value store.
    */
   std::optional<std::string> kv_string(const file_metadata_t&,
-                                       const std::string_view&) const;
+                                       std::string_view) const;
 
   /**
    * Fetch a std::size_t value from the metadata key-value store.
    */
   std::optional<std::size_t> kv_size_t(const file_metadata_t&,
-                                       const std::string_view&) const;
+                                       std::string_view) const;
 
   /**
    * Directly access the FileReader.  This reference is only valid

include/mzpeak/util/projection.h

@@ -43,7 +43,7 @@ class Projection final {
   /**
    * Project a column using a field name.
    */
-  Result project(const std::shared_ptr<Schema::Group>&, const std::string_view&&);
+  Result project(const std::shared_ptr<Schema::Group>&, std::string_view);
 
   /**
    * Look up a CV type and project that.

include/mzpeak/util/slice.h

@@ -48,13 +48,18 @@ class Slice final {
   /**
    * Return the raw array for the given field.
    *
+   * The returned raw array is removed from the internal storage
+   * therefore calling this method again with the same column will
+   * fail.
+   *
    * NOTE: If you request a field that does not exist in the slice
    * this function will return a nullptr.
    */
-  std::shared_ptr<Raw> raw(const Column&) const;
+  std::shared_ptr<Raw> raw(const Column&);
 
   /**
-   * Decode the first non-null value.
+   * Decode the first non-null value from the given column.  The
+   * column is then removed from internal storage.
    *
    * Template Parameters:
    *
@@ -63,10 +68,11 @@ class Slice final {
    *   - R: The destination object to update with the decoded value
    */
   template <typename T, typename R = std::optional<typename T::value_type>>
-  void singleton(const Column&, R&, T&& = {}) const;
+  void singleton(const Column&, R&, T&& = {});
 
   /**
-   * Exact and decode an array.
+   * Extract and decode an array.  The array is then removed from the
+   * internal storage.
    *
    * Use one of the decoders defined in `decoders.h`, or write your own.
    *
@@ -78,12 +84,12 @@ class Slice final {
    */
   template <typename T, typename V = std::vector<typename T::value_type>>
     requires Decoders::from_arrow_array<T, V>
-  void array(const Column&, V&, T&& = {}) const;
+  void array(const Column&, V&, T&& = {});
 
   /****************************************************************************/
   template <typename T, typename V = std::vector<typename T::value_type>>
     requires Decoders::from_arrow_array<T, V>
-  void array(const Column&, V&, T&) const;
+  void array(const Column&, V&, T&);
 
 private:
   friend class MzPeak::Util::Executor;
@@ -100,7 +106,7 @@ class Slice final {
 
 /******************************************************************************/
 template <typename T, typename R>
-void Slice::singleton(const Column& field, R& dst, T&& t) const
+void Slice::singleton(const Column& field, R& dst, T&& t)
 {
   std::shared_ptr<Raw> chunks = raw(field);
 
@@ -130,15 +136,15 @@ void Slice::singleton(const Column& field, R& dst, T&& t) const
 /******************************************************************************/
 template <typename T, typename V>
   requires Decoders::from_arrow_array<T, V>
-void Slice::array(const Column& field, V& v, T&& t) const
+void Slice::array(const Column& field, V& v, T&& t)
 {
   array(field, v, t);
 }
 
 /******************************************************************************/
 template <typename T, typename V>
   requires Decoders::from_arrow_array<T, V>
-void Slice::array(const Column& field, V& v, T& t) const
+void Slice::array(const Column& field, V& v, T& t)
 {
   std::shared_ptr<Raw> chunks = raw(field);
   if (chunks == nullptr) return;