From 952525b1f67be3cec5309120f525e8aac9c7540e Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:08:17 +0200 Subject: [PATCH 1/9] std::string_view -> std::string Arrow/parquet uses std::string_view to represent byte arrays. However, since they are non-owning objects they need the original Arrow array to remain resident in memory. That's unhelpful if we want to delete arrow arrays after decoding them. This change copies std::string_view values into std::string objects so there is no dependency on the original Arrow array memory. --- include/mzpeak/data/null_marking.h | 6 ++-- include/mzpeak/data/transformer/primary.h | 2 +- include/mzpeak/util/algorithm.h | 3 +- include/mzpeak/util/decoders.h | 26 +++++++++++++++- include/mzpeak/util/types.h | 36 +++++++++++++++++++---- src/util/executor.cpp | 3 +- src/util/planner.cpp | 10 ++++--- 7 files changed, 71 insertions(+), 15 deletions(-) diff --git a/include/mzpeak/data/null_marking.h b/include/mzpeak/data/null_marking.h index c21acca..4cbee07 100644 --- a/include/mzpeak/data/null_marking.h +++ b/include/mzpeak/data/null_marking.h @@ -171,7 +171,8 @@ std::optional Decoder::operator()(int64_t index) prior_.index = index; if (range.size() == 1) { - prior_.value = array_->Value(range.begin); + prior_.value = + Decoders::unsafe_array_value>(array_, range.begin); prior_.delta = estimator_.predict(prior_.value); } else { auto slice = array_->Slice(range.begin, range.size()); @@ -182,7 +183,8 @@ std::optional Decoder::operator()(int64_t index) Decoders::Scalar decoder; decoder.decode(slice, values); - prior_.value = array_->Value(range.anchor(index)); + prior_.value = + Decoders::unsafe_array_value>(array_, range.anchor(index)); prior_.delta = Algorithm::median_delta(values, zero_); } diff --git a/include/mzpeak/data/transformer/primary.h b/include/mzpeak/data/transformer/primary.h index 3061d40..dc95f0a 100644 --- a/include/mzpeak/data/transformer/primary.h +++ b/include/mzpeak/data/transformer/primary.h @@ -75,7 +75,7 @@ Decoder::Decoder(std::shared_ptr signals, }; // Decode the `chunk_encoding` column. - std::vector encodings; + std::vector encodings; decode(Schema::BufferFormat::ChunkEncoding, encodings); chunk_encoding_.reserve(encodings.size()); diff --git a/include/mzpeak/util/algorithm.h b/include/mzpeak/util/algorithm.h index e0d1f35..4f6950b 100644 --- a/include/mzpeak/util/algorithm.h +++ b/include/mzpeak/util/algorithm.h @@ -16,6 +16,7 @@ top-level directory of this repository. #include #include +#include "mzpeak/util/decoders.h" #include "mzpeak/util/types.h" namespace MzPeak::Util::Algorithm { @@ -152,7 +153,7 @@ null_delta_decode(typename type_traits::value_type start, for (int64_t index : std::views::iota(0, length)) { if (casted->IsValid(index)) { - ValueType delta = casted->Value(index); + ValueType delta = Decoders::unsafe_array_value(casted, index); last = last.value_or(zero) + delta; append(last); } else { diff --git a/include/mzpeak/util/decoders.h b/include/mzpeak/util/decoders.h index 34f4153..90761b3 100644 --- a/include/mzpeak/util/decoders.h +++ b/include/mzpeak/util/decoders.h @@ -43,6 +43,29 @@ concept from_arrow_array = { t.decode(a, r) } -> std::same_as; }; +/******************************************************************************/ +/** + * Read a value from an array without checking bounds or if it is NULL. + * + * This is needed because Parquet/Arrow uses std::string_view for byte + * arrays, but that means that the original arrow array needs to + * remain resident in memory. Therefore we need to copy the memory + * referenced by a std::string_view into a std::string. + */ +template +type_traits::value_type +unsafe_array_value(const std::shared_ptr::array_type>& ary, + int64_t index) +{ + using A = type_traits::array_type; + + if constexpr (std::is_same_v) { + return ary->GetString(index); + } else { + return ary->Value(index); + } +} + /******************************************************************************/ /** * If the given array is a "list of lists" then visit each element of @@ -177,7 +200,8 @@ class Scalar final : Helper> { std::optional value = std::invoke(null_decoder_, i); if (value.has_value()) this->push(dst, std::move(*value)); } else { - this->push(dst, std::move(casted->Value(i))); + V value = unsafe_array_value>(casted, i); + this->push(dst, std::move(value)); } } } diff --git a/include/mzpeak/util/types.h b/include/mzpeak/util/types.h index a3703be..08eb52b 100644 --- a/include/mzpeak/util/types.h +++ b/include/mzpeak/util/types.h @@ -14,9 +14,11 @@ directory of this repository. #include #include #include +#include #include #include "mzpeak/exception.h" +#include "mzpeak/util/compat.h" // IWYU pragma: keep namespace parquet::schema { class PrimitiveNode; @@ -36,7 +38,7 @@ concept supported_type = std::same_as, int8_t> || std::same_as, uint64_t> || std::same_as, float> || std::same_as, double> || - std::same_as, std::string_view>; + std::same_as, std::string>; /// A variant that can hold any supported type. using any_value_type = std::variant; + std::string>; /// A variant that can hold pairs of any supported type. using any_pair_type = std::variant, @@ -58,7 +60,7 @@ using any_pair_type = std::variant, std::pair, std::pair, std::pair, - std::pair>; + std::pair>; /// Enum of supported data types for tracking at run time. enum class Type { @@ -165,7 +167,7 @@ template <> struct type_traits { template <> struct type_traits { static constexpr const char* name = "bytes"; - using value_type = std::string_view; + using value_type = std::string; using parquet_type = parquet::ByteArrayType; using array_type = arrow::StringArray; using builder_type = arrow::StringBuilder; @@ -209,7 +211,7 @@ template <> struct type_from_value_type { static constexpr Type enum_type = Type::Float64; }; -template <> struct type_from_value_type { +template <> struct type_from_value_type { static constexpr Type enum_type = Type::ByteArray; }; @@ -217,6 +219,30 @@ template <> struct type_from_value_type { template inline constexpr Type enum_type_v = type_from_value_type::enum_type; +/** + * Cast or convert a value from one type (usually Type::parquet_type) + * to another (usually Type::value_type). + * + * This is needed because Parquet/Arrow uses std::string_view for byte + * arrays, but that means that the original arrow array needs to + * remain resident in memory. Therefore we need to copy the memory + * referenced by a std::string_view into a std::string. + */ +template To safe_cast_or_copy(From value) +{ + using from_t = std::remove_cvref_t; + using to_t = std::remove_cvref_t; + + if constexpr (std::is_convertible_v) { + return value; + } else if constexpr (std::is_same_v && + std::is_same_v) { + return parquet::ByteArrayToString(value); + } else { + static_assert(false_type, "no conversion available"); + } +} + /** * Return a type for the given parquet node. */ diff --git a/src/util/executor.cpp b/src/util/executor.cpp index 7c3298b..9c1eba7 100644 --- a/src/util/executor.cpp +++ b/src/util/executor.cpp @@ -77,7 +77,8 @@ Query::Result Executor::Impl::ArrayValueHelper::operator()() } auto casted = std::static_pointer_cast::array_type>(a); - return Query::Result(casted->Value(row_index_)); + auto value = Decoders::unsafe_array_value(casted, row_index_); + return Query::Result(value); } /******************************************************************************/ diff --git a/src/util/planner.cpp b/src/util/planner.cpp index dc07f47..fc8cfbe 100644 --- a/src/util/planner.cpp +++ b/src/util/planner.cpp @@ -275,14 +275,16 @@ struct ColMinMax final { if (index_ != nullptr) { using Index = parquet::TypedColumnIndex

; std::shared_ptr index = std::static_pointer_cast(index_); - return R(std::make_pair(static_cast(index->min_values()[page_index_]), - static_cast(index->max_values()[page_index_]))); + V min = safe_cast_or_copy(index->min_values()[page_index_]); + V max = safe_cast_or_copy(index->max_values()[page_index_]); + return R(std::make_pair(min, max)); } else if (stats_ != nullptr) { using Stats = parquet::TypedStatistics

; if (!stats_->HasMinMax()) return R::skip(); std::shared_ptr stats = std::static_pointer_cast(stats_); - return R(std::make_pair(static_cast(stats->min()), - static_cast(stats->max()))); + V min = safe_cast_or_copy(stats->min()); + V max = safe_cast_or_copy(stats->max()); + return R(std::make_pair(min, max)); } else { return R::skip(); } From 680277f416c463b60df3c75f187fc0312e9aeb93 Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:34:12 +0200 Subject: [PATCH 2/9] Delete arrow arrays after decoding them No need to keep this memory around longer than necessary. --- include/mzpeak/data/encoding.h | 2 ++ include/mzpeak/util/slice.h | 24 +++++++++++++++--------- src/util/slice.cpp | 6 ++++-- test/executor_test.cpp | 9 +++------ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/include/mzpeak/data/encoding.h b/include/mzpeak/data/encoding.h index 88024b1..b8ee50e 100644 --- a/include/mzpeak/data/encoding.h +++ b/include/mzpeak/data/encoding.h @@ -161,6 +161,8 @@ void Decoder::decode_with_nulls(const ArrayIndex::Dimension& dim, if (!col.has_value()) { throw ParquetError("unable to decode dimension, not in schema: " + dim.name); + } else if (!slice_->has_column(col.value())) { + return; // No data to decode so we can exit early. } auto go = [&](auto&& decoder) -> void { slice_->array(col.value(), v, decoder); }; diff --git a/include/mzpeak/util/slice.h b/include/mzpeak/util/slice.h index d29c1e9..cb0372d 100644 --- a/include/mzpeak/util/slice.h +++ b/include/mzpeak/util/slice.h @@ -48,13 +48,18 @@ class Slice final { /** * Return the raw array for the given field. * + * The returned raw array is removed from the internal storage + * therefore calling this method again with the same column will + * fail. + * * NOTE: If you request a field that does not exist in the slice * this function will return a nullptr. */ - std::shared_ptr raw(const Column&) const; + std::shared_ptr raw(const Column&); /** - * Decode the first non-null value. + * Decode the first non-null value from the given column. The + * column is then removed from internal storage. * * Template Parameters: * @@ -63,10 +68,11 @@ class Slice final { * - R: The destination object to update with the decoded value */ template > - void singleton(const Column&, R&, T&& = {}) const; + void singleton(const Column&, R&, T&& = {}); /** - * Exact and decode an array. + * Extract and decode an array. The array is then removed from the + * internal storage. * * Use one of the decoders defined in `decoders.h`, or write your own. * @@ -78,12 +84,12 @@ class Slice final { */ template > requires Decoders::from_arrow_array - void array(const Column&, V&, T&& = {}) const; + void array(const Column&, V&, T&& = {}); /****************************************************************************/ template > requires Decoders::from_arrow_array - void array(const Column&, V&, T&) const; + void array(const Column&, V&, T&); private: friend class MzPeak::Util::Executor; @@ -100,7 +106,7 @@ class Slice final { /******************************************************************************/ template -void Slice::singleton(const Column& field, R& dst, T&& t) const +void Slice::singleton(const Column& field, R& dst, T&& t) { std::shared_ptr chunks = raw(field); @@ -130,7 +136,7 @@ void Slice::singleton(const Column& field, R& dst, T&& t) const /******************************************************************************/ template requires Decoders::from_arrow_array -void Slice::array(const Column& field, V& v, T&& t) const +void Slice::array(const Column& field, V& v, T&& t) { array(field, v, t); } @@ -138,7 +144,7 @@ void Slice::array(const Column& field, V& v, T&& t) const /******************************************************************************/ template requires Decoders::from_arrow_array -void Slice::array(const Column& field, V& v, T& t) const +void Slice::array(const Column& field, V& v, T& t) { std::shared_ptr chunks = raw(field); if (chunks == nullptr) return; diff --git a/src/util/slice.cpp b/src/util/slice.cpp index a79ba45..21ad787 100644 --- a/src/util/slice.cpp +++ b/src/util/slice.cpp @@ -52,14 +52,16 @@ bool Slice::has_column(const Column& column) const } /******************************************************************************/ -std::shared_ptr Slice::raw(const Column& field) const +std::shared_ptr Slice::raw(const Column& field) { auto it = impl_->arrays_.find(impl_->key(field)); if (it == impl_->arrays_.end()) { return nullptr; } else { - return it->second; + auto sp = it->second; + impl_->arrays_.erase(it); + return sp; } } diff --git a/test/executor_test.cpp b/test/executor_test.cpp index b32d80b..2921edb 100644 --- a/test/executor_test.cpp +++ b/test/executor_test.cpp @@ -47,9 +47,7 @@ BOOST_AUTO_TEST_CASE(can_find_spectrum) Util::Executor executor = parquet->executor(projection); const auto& slice = executor.execute(plan); BOOST_TEST((slice->fields() == projection.get())); - - auto raw = slice->raw(*mz_field); - BOOST_TEST((raw != nullptr)); + BOOST_TEST(slice->has_column(mz_field.value())); // Decode, dropping null values. std::vector mz; @@ -88,10 +86,9 @@ BOOST_AUTO_TEST_CASE(can_read_uint8_t) Util::Executor executor = parquet->executor(projection); const auto& slice = executor.execute(plan); - BOOST_TEST((slice->fields() == projection.get())); - auto raw = slice->raw(*ms_level); - BOOST_TEST((raw != nullptr)); + BOOST_TEST((slice->fields() == projection.get())); + BOOST_TEST(slice->has_column(ms_level.value())); std::vector levels; slice->array>(*ms_level, levels); From 1529adf6c6982bfbdb6bbcd2108237fa25fb7368 Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:35:07 +0200 Subject: [PATCH 3/9] Remove more uses of std::optional::operator* --- include/mzpeak/data/transformer/primary.h | 4 ++-- test/executor_test.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/mzpeak/data/transformer/primary.h b/include/mzpeak/data/transformer/primary.h index dc95f0a..a718280 100644 --- a/include/mzpeak/data/transformer/primary.h +++ b/include/mzpeak/data/transformer/primary.h @@ -71,7 +71,7 @@ Decoder::Decoder(std::shared_ptr signals, throw InvalidFormatError(msg); } - slice->array(*column, dest, Util::Decoders::Scalar()); + slice->array(column.value(), dest, Util::Decoders::Scalar()); }; // Decode the `chunk_encoding` column. @@ -86,7 +86,7 @@ Decoder::Decoder(std::shared_ptr signals, throw InvalidFormatError("invalid chunk encoding CV: " + std::string(s)); } - chunk_encoding_.emplace_back(*cv); + chunk_encoding_.emplace_back(cv.value()); } // Decode the `chunk_start` column. diff --git a/test/executor_test.cpp b/test/executor_test.cpp index 2921edb..cbf199f 100644 --- a/test/executor_test.cpp +++ b/test/executor_test.cpp @@ -35,14 +35,14 @@ BOOST_AUTO_TEST_CASE(can_find_spectrum) auto mz_field = parquet->field("point", "mz"); BOOST_TEST(mz_field.has_value()); - auto query = Util::Query::Builder(*index_field).eq(1ul); + auto query = Util::Query::Builder(index_field.value()).eq(1ul); Util::Planner planner = parquet->planner(query); auto plan = planner.plan(); BOOST_TEST(plan.ranges.size() == 1ul); Util::Projection projection; - projection.project(*mz_field); + projection.project(mz_field.value()); Util::Executor executor = parquet->executor(projection); const auto& slice = executor.execute(plan); @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(can_find_spectrum) // Decode, dropping null values. std::vector mz; - slice->array>(*mz_field, mz); + slice->array>(mz_field.value(), mz); BOOST_TEST(mz.size() == 15063); BOOST_TEST(mz[0] == 200.09, boost::test_tools::tolerance(0.001)); BOOST_TEST(mz[mz.size() - 1] == 1999.81, boost::test_tools::tolerance(0.001)); @@ -76,13 +76,13 @@ BOOST_AUTO_TEST_CASE(can_read_uint8_t) auto ms_level = parquet->field("root", "ms_level"); BOOST_TEST(ms_level.has_value()); - auto query = Util::Query::Builder(*index_field).eq(0ul); + auto query = Util::Query::Builder(index_field.value()).eq(0ul); Util::Planner planner = parquet->planner(query); auto plan = planner.plan(); BOOST_TEST(plan.ranges.size() == 1ul); Util::Projection projection; - projection.project(*ms_level); + projection.project(ms_level.value()); Util::Executor executor = parquet->executor(projection); const auto& slice = executor.execute(plan); @@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(can_read_uint8_t) BOOST_TEST(slice->has_column(ms_level.value())); std::vector levels; - slice->array>(*ms_level, levels); + slice->array>(ms_level.value(), levels); BOOST_TEST(levels.size() == 1); BOOST_TEST(levels[0] == 1); } From 6d1f9727e0da3fdcb66a9d84e6dae4d3cebd2a9c Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:48:04 +0200 Subject: [PATCH 4/9] Save a few stack bytes by passing std::string_view by value --- include/mzpeak/data/signals.h | 2 +- include/mzpeak/index.h | 2 +- include/mzpeak/metadata/table.h | 2 +- include/mzpeak/schema/buffer_format.h | 2 +- include/mzpeak/schema/cv.h | 8 ++++---- include/mzpeak/schema/data_kind.h | 2 +- include/mzpeak/schema/entity_type.h | 2 +- include/mzpeak/schema/group.h | 4 ++-- include/mzpeak/schema/psi/array_type.h | 2 +- include/mzpeak/util/manager.h | 3 +-- include/mzpeak/util/parquet.h | 7 +++---- include/mzpeak/util/projection.h | 2 +- src/data/signals.cpp | 2 +- src/index.cpp | 3 +-- src/metadata/table.cpp | 2 +- src/schema/buffer_format.cpp | 2 +- src/schema/cv.cpp | 2 +- src/schema/data_kind.cpp | 2 +- src/schema/entity_type.cpp | 2 +- src/schema/group.cpp | 4 ++-- src/schema/psi/array_type.cpp | 2 +- src/util/executor.cpp | 2 +- src/util/manager.cpp | 2 +- src/util/parquet.cpp | 13 ++++++------- src/util/projection.cpp | 2 +- 25 files changed, 37 insertions(+), 41 deletions(-) diff --git a/include/mzpeak/data/signals.h b/include/mzpeak/data/signals.h index 7cef70c..a2e69eb 100644 --- a/include/mzpeak/data/signals.h +++ b/include/mzpeak/data/signals.h @@ -56,7 +56,7 @@ class Signals { * * Useful if you need to manually construct queries. */ - std::optional column(const std::string_view&) const; + std::optional column(std::string_view) const; /** * Low-level interface for accessing a column given an array index entry. diff --git a/include/mzpeak/index.h b/include/mzpeak/index.h index 93dfdb8..1fe7acf 100644 --- a/include/mzpeak/index.h +++ b/include/mzpeak/index.h @@ -43,7 +43,7 @@ class Index { /** * Find a file in the mzPeak archive with the given name. */ - std::vector::const_iterator find(const std::string_view&) const; + std::vector::const_iterator find(std::string_view) const; /** * Access the spectra in the file. diff --git a/include/mzpeak/metadata/table.h b/include/mzpeak/metadata/table.h index 8b1cb22..a3ecc21 100644 --- a/include/mzpeak/metadata/table.h +++ b/include/mzpeak/metadata/table.h @@ -31,7 +31,7 @@ class Table final { * Return a group with the given name. If the group does not * exist in the schema return `nullptr`. */ - std::shared_ptr group(const std::string_view&) const; + std::shared_ptr group(std::string_view) const; /** * Read all rows from the given group where the index column diff --git a/include/mzpeak/schema/buffer_format.h b/include/mzpeak/schema/buffer_format.h index d0bcfc4..de4c8a9 100644 --- a/include/mzpeak/schema/buffer_format.h +++ b/include/mzpeak/schema/buffer_format.h @@ -61,6 +61,6 @@ std::string buffer_format_to_string(BufferFormat); /** * Parse an BufferFormat from a string. */ -BufferFormat buffer_format_from_string(const std::string_view&); +BufferFormat buffer_format_from_string(std::string_view); } // namespace MzPeak::Schema diff --git a/include/mzpeak/schema/cv.h b/include/mzpeak/schema/cv.h index 6b4f9ec..0b75ce2 100644 --- a/include/mzpeak/schema/cv.h +++ b/include/mzpeak/schema/cv.h @@ -19,9 +19,9 @@ namespace MzPeak::Schema { class CV { public: /// Constructor. - CV(const std::string_view& code, const std::string_view& accession) - : code_(std::move(code)) - , accession_(std::move(accession)) + CV(std::string_view code, std::string_view accession) + : code_(code) + , accession_(accession) { } @@ -29,7 +29,7 @@ class CV { ~CV() = default; /// Parse a string like "MS:1000511" - static std::optional from_string(const std::string_view&); + static std::optional from_string(std::string_view); /// Convert this CV term to a string like "MS:1000511" std::string to_string() const; diff --git a/include/mzpeak/schema/data_kind.h b/include/mzpeak/schema/data_kind.h index 7bfb16b..5adb5d2 100644 --- a/include/mzpeak/schema/data_kind.h +++ b/include/mzpeak/schema/data_kind.h @@ -46,6 +46,6 @@ std::string data_kind_to_string(DataKind); /** * Parse a DataKind from a string view. */ -DataKind data_kind_from_string(const std::string_view&); +DataKind data_kind_from_string(std::string_view); } // namespace MzPeak::Schema diff --git a/include/mzpeak/schema/entity_type.h b/include/mzpeak/schema/entity_type.h index d041830..e0a72bd 100644 --- a/include/mzpeak/schema/entity_type.h +++ b/include/mzpeak/schema/entity_type.h @@ -38,6 +38,6 @@ std::string entity_type_to_string(EntityType); /** * Parse an EntityType from a string view. */ -EntityType entity_type_from_string(const std::string_view&); +EntityType entity_type_from_string(std::string_view); } // namespace MzPeak::Schema diff --git a/include/mzpeak/schema/group.h b/include/mzpeak/schema/group.h index 4e041d7..db44a07 100644 --- a/include/mzpeak/schema/group.h +++ b/include/mzpeak/schema/group.h @@ -73,7 +73,7 @@ class Group final { }; /// Constructor from an encoded column name. - explicit Field(const std::string_view& column_name, + explicit Field(std::string_view column_name, index_type rel_index, index_type abs_index); @@ -171,7 +171,7 @@ class Group final { * NOTE: For metadata groups this is the cleaned name, not the raw * schema node name. */ - std::optional> field(const std::string_view&&) const; + std::optional> field(std::string_view) const; /** * Find a field given its CV type. diff --git a/include/mzpeak/schema/psi/array_type.h b/include/mzpeak/schema/psi/array_type.h index 88a2a27..898932c 100644 --- a/include/mzpeak/schema/psi/array_type.h +++ b/include/mzpeak/schema/psi/array_type.h @@ -150,6 +150,6 @@ std::string array_type_to_string(ArrayType); /** * Parse an ArrayType from a string. */ -ArrayType array_type_from_string(const std::string_view&); +ArrayType array_type_from_string(std::string_view); } // namespace MzPeak::Schema::PSI diff --git a/include/mzpeak/util/manager.h b/include/mzpeak/util/manager.h index ca88079..5d4f467 100644 --- a/include/mzpeak/util/manager.h +++ b/include/mzpeak/util/manager.h @@ -32,8 +32,7 @@ class Manager final { /** * Find a file given its name. */ - std::vector::const_iterator - find_file(const std::string_view& name) const; + std::vector::const_iterator find_file(std::string_view name) const; /** * Open a Parquet file from the mzPeak archive. diff --git a/include/mzpeak/util/parquet.h b/include/mzpeak/util/parquet.h index 3c7cd5f..9ae1b49 100644 --- a/include/mzpeak/util/parquet.h +++ b/include/mzpeak/util/parquet.h @@ -45,8 +45,7 @@ class Parquet final { /** * Return a Group and Field matching the given names. */ - std::optional field(const std::string_view&, - const std::string_view&) const; + std::optional field(std::string_view, std::string_view) const; /** * Access the file metadata. @@ -57,13 +56,13 @@ class Parquet final { * Fetch a string value from the metadata key-value store. */ std::optional kv_string(const file_metadata_t&, - const std::string_view&) const; + std::string_view) const; /** * Fetch a std::size_t value from the metadata key-value store. */ std::optional kv_size_t(const file_metadata_t&, - const std::string_view&) const; + std::string_view) const; /** * Directly access the FileReader. This reference is only valid diff --git a/include/mzpeak/util/projection.h b/include/mzpeak/util/projection.h index 5b4a53f..6bf909c 100644 --- a/include/mzpeak/util/projection.h +++ b/include/mzpeak/util/projection.h @@ -43,7 +43,7 @@ class Projection final { /** * Project a column using a field name. */ - Result project(const std::shared_ptr&, const std::string_view&&); + Result project(const std::shared_ptr&, std::string_view); /** * Look up a CV type and project that. diff --git a/src/data/signals.cpp b/src/data/signals.cpp index 9e5b660..1b743bb 100644 --- a/src/data/signals.cpp +++ b/src/data/signals.cpp @@ -102,7 +102,7 @@ std::size_t Signals::record_count() const } /******************************************************************************/ -std::optional Signals::column(const std::string_view& name) const +std::optional Signals::column(std::string_view name) const { return impl_->parquet_->field(impl_->array_index_->prefix(), name); } diff --git a/src/index.cpp b/src/index.cpp index 9c41888..41330b0 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -27,8 +27,7 @@ Index::Index(std::unique_ptr archive) const std::vector& Index::files() const { return manager_->files(); } /******************************************************************************/ -std::vector::const_iterator -Index::find(const std::string_view& name) const +std::vector::const_iterator Index::find(std::string_view name) const { return manager_->find_file(name); } diff --git a/src/metadata/table.cpp b/src/metadata/table.cpp index 46197c6..0fbbb00 100644 --- a/src/metadata/table.cpp +++ b/src/metadata/table.cpp @@ -50,7 +50,7 @@ Table::Table(std::unique_ptr parquet) Table::~Table() = default; /******************************************************************************/ -std::shared_ptr Table::group(const std::string_view& name) const +std::shared_ptr Table::group(std::string_view name) const { const std::shared_ptr& map = impl_->parquet_->groups(); auto it = map->find(std::string(name)); diff --git a/src/schema/buffer_format.cpp b/src/schema/buffer_format.cpp index ddf56db..20f34b6 100644 --- a/src/schema/buffer_format.cpp +++ b/src/schema/buffer_format.cpp @@ -38,7 +38,7 @@ std::string buffer_format_to_string(BufferFormat v) } /******************************************************************************/ -BufferFormat buffer_format_from_string(const std::string_view& s) +BufferFormat buffer_format_from_string(std::string_view s) { using enum BufferFormat; diff --git a/src/schema/cv.cpp b/src/schema/cv.cpp index 6ff4ddf..69c1bdf 100644 --- a/src/schema/cv.cpp +++ b/src/schema/cv.cpp @@ -11,7 +11,7 @@ top-level directory of this repository. namespace MzPeak::Schema { /******************************************************************************/ -std::optional CV::from_string(const std::string_view& s) +std::optional CV::from_string(std::string_view s) { std::string_view::size_type sep_pos = s.find(':'); diff --git a/src/schema/data_kind.cpp b/src/schema/data_kind.cpp index 58905e3..200ce4a 100644 --- a/src/schema/data_kind.cpp +++ b/src/schema/data_kind.cpp @@ -32,7 +32,7 @@ std::string data_kind_to_string(DataKind dk) } /******************************************************************************/ -DataKind data_kind_from_string(const std::string_view& s) +DataKind data_kind_from_string(std::string_view s) { using enum DataKind; diff --git a/src/schema/entity_type.cpp b/src/schema/entity_type.cpp index 537b1fe..ec7b047 100644 --- a/src/schema/entity_type.cpp +++ b/src/schema/entity_type.cpp @@ -29,7 +29,7 @@ std::string entity_type_to_string(EntityType et) return "other"; } -EntityType entity_type_from_string(const std::string_view& s) +EntityType entity_type_from_string(std::string_view s) { using enum EntityType; diff --git a/src/schema/group.cpp b/src/schema/group.cpp index 63e1642..fdedadb 100644 --- a/src/schema/group.cpp +++ b/src/schema/group.cpp @@ -80,7 +80,7 @@ field_type_from_parquet(const std::shared_ptr& node) } /******************************************************************************/ -Group::Field::Field(const std::string_view& column_name, +Group::Field::Field(std::string_view column_name, index_type rel_index, index_type abs_index) : rel_index_(rel_index) @@ -208,7 +208,7 @@ Group::index_type Group::index() const { return index_; } /******************************************************************************/ std::optional> -Group::field(const std::string_view&& name) const +Group::field(std::string_view name) const { auto it = fields_.find(std::string{name}); diff --git a/src/schema/psi/array_type.cpp b/src/schema/psi/array_type.cpp index 629f387..412cf12 100644 --- a/src/schema/psi/array_type.cpp +++ b/src/schema/psi/array_type.cpp @@ -64,7 +64,7 @@ std::string array_type_to_string(ArrayType v) } /******************************************************************************/ -ArrayType array_type_from_string(const std::string_view& s) +ArrayType array_type_from_string(std::string_view s) { using enum ArrayType; diff --git a/src/util/executor.cpp b/src/util/executor.cpp index 9c1eba7..4832ca4 100644 --- a/src/util/executor.cpp +++ b/src/util/executor.cpp @@ -30,7 +30,7 @@ struct Executor::Impl { } /// Helper to check a result and throw an error if necessary. - template T check(const std::string_view& msg, arrow::Result r) + template T check(std::string_view msg, arrow::Result r) { if (!r.ok()) { std::string error("while executing a query: " + std::string(msg) + ": "); diff --git a/src/util/manager.cpp b/src/util/manager.cpp index 77c00b7..39af105 100644 --- a/src/util/manager.cpp +++ b/src/util/manager.cpp @@ -66,7 +66,7 @@ const std::vector& Manager::files() const { return files_; } /******************************************************************************/ std::vector::const_iterator -Manager::find_file(const std::string_view& name) const +Manager::find_file(std::string_view name) const { return std::ranges::find(files_, name, &Schema::File::file_name); } diff --git a/src/util/parquet.cpp b/src/util/parquet.cpp index fab820d..a7b0a7a 100644 --- a/src/util/parquet.cpp +++ b/src/util/parquet.cpp @@ -24,7 +24,7 @@ namespace MzPeak::Util { /******************************************************************************/ std::optional get_kv_string(const Parquet::file_metadata_t& fmd, - const std::string_view& key) + std::string_view key) { auto result(fmd->key_value_metadata()->Get(key)); @@ -37,7 +37,7 @@ std::optional get_kv_string(const Parquet::file_metadata_t& fmd, /******************************************************************************/ std::optional get_kv_uint(const Parquet::file_metadata_t& fmd, - const std::string_view& key) + std::string_view key) { return get_kv_string(fmd, key).and_then( [](const std::string& s) -> std::optional { @@ -147,9 +147,8 @@ const std::shared_ptr& Parquet::groups() const } /******************************************************************************/ -std::optional -Parquet::field(const std::string_view& group_name, - const std::string_view& field_name) const +std::optional Parquet::field(std::string_view group_name, + std::string_view field_name) const { auto group_ptr = impl_->groups_->find(std::string{group_name}); if (group_ptr == impl_->groups_->end()) return {}; @@ -168,14 +167,14 @@ Parquet::file_metadata_t Parquet::file_metadata() const /******************************************************************************/ std::optional Parquet::kv_string(const file_metadata_t& fmd, - const std::string_view& key) const + std::string_view key) const { return get_kv_string(fmd, key); } /******************************************************************************/ std::optional Parquet::kv_size_t(const file_metadata_t& fmd, - const std::string_view& key) const + std::string_view key) const { return get_kv_uint(fmd, key); } diff --git a/src/util/projection.cpp b/src/util/projection.cpp index dee9964..c2588c0 100644 --- a/src/util/projection.cpp +++ b/src/util/projection.cpp @@ -41,7 +41,7 @@ Projection::Result Projection::project( /******************************************************************************/ Projection::Result Projection::project(const std::shared_ptr& group, - const std::string_view&& name) + std::string_view name) { return project(group, group->field(std::move(name))); } From 529dd8f45d3f7db28877cd8ebe21316e78951d74 Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:48:25 +0200 Subject: [PATCH 5/9] Correctly prevent copying --- src/io/zip.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io/zip.cpp b/src/io/zip.cpp index 9ecf5ad..41ff62c 100644 --- a/src/io/zip.cpp +++ b/src/io/zip.cpp @@ -84,8 +84,8 @@ class ZipFile_ final : public MzPeak::IO::File { fs::path path_; private: - Impl& operator=(const Impl&) = default; - Impl(const Impl&) = default; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&) = delete; }; std::shared_ptr impl_; From b981b03eb65d802f417b4e01356c20383beadddd Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:48:37 +0200 Subject: [PATCH 6/9] Unneeded call to reserve --- include/mzpeak/data/encoding.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/mzpeak/data/encoding.h b/include/mzpeak/data/encoding.h index b8ee50e..2c86801 100644 --- a/include/mzpeak/data/encoding.h +++ b/include/mzpeak/data/encoding.h @@ -122,7 +122,6 @@ void Decoder::remap(const ArrayIndex::Dimension& dim, std::vector& v) cons } else { std::vector tmp; decode(dim, tmp); - v.reserve(tmp.size()); v.insert(v.end(), tmp.begin(), tmp.end()); } } From f67853e94d74f956486c0ecb892610697a1dadfc Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:54:25 +0200 Subject: [PATCH 7/9] Add missing header files --- include/mzpeak/index.h | 1 + include/mzpeak/util/manager.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/include/mzpeak/index.h b/include/mzpeak/index.h index 1fe7acf..8730603 100644 --- a/include/mzpeak/index.h +++ b/include/mzpeak/index.h @@ -9,6 +9,7 @@ directory of this repository. #pragma once #include +#include #include namespace MzPeak { diff --git a/include/mzpeak/util/manager.h b/include/mzpeak/util/manager.h index 5d4f467..533e589 100644 --- a/include/mzpeak/util/manager.h +++ b/include/mzpeak/util/manager.h @@ -9,6 +9,8 @@ directory of this repository. #pragma once #include +#include +#include #include "mzpeak/io/archive.h" #include "mzpeak/schema/file.h" From 8579280130b1381789b81630090bb8ccd24a56fb Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:56:43 +0200 Subject: [PATCH 8/9] Use correct exception --- include/mzpeak/metadata/table.h | 3 +-- src/metadata/table.cpp | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/mzpeak/metadata/table.h b/include/mzpeak/metadata/table.h index a3ecc21..1b79b4b 100644 --- a/include/mzpeak/metadata/table.h +++ b/include/mzpeak/metadata/table.h @@ -28,8 +28,7 @@ class Table final { ~Table(); /** - * Return a group with the given name. If the group does not - * exist in the schema return `nullptr`. + * Return a group with the given name. */ std::shared_ptr group(std::string_view) const; diff --git a/src/metadata/table.cpp b/src/metadata/table.cpp index 0fbbb00..06ca5bc 100644 --- a/src/metadata/table.cpp +++ b/src/metadata/table.cpp @@ -33,7 +33,7 @@ Table::Impl::Impl(std::unique_ptr parquet) if (file.data_kind() != Schema::DataKind::Metadata) { std::string msg("file is not a metadata file: " + file.file_name()); - throw ParquetError(msg); + throw InvalidFormatError(msg); } } @@ -56,8 +56,8 @@ std::shared_ptr Table::group(std::string_view name) const auto it = map->find(std::string(name)); if (it == map->end()) { - // FIXME: Replace with the InvalidFormat exception. - throw ParquetError("schema is missing the " + std::string(name) + " group"); + throw InvalidFormatError("schema is missing the " + std::string(name) + + " group"); } else { return it->second; } From 8cbd93b712ceb1ebc5fd4e4f4930732b5204d52d Mon Sep 17 00:00:00 2001 From: "Peter J. Jones" Date: Tue, 18 Aug 2026 14:56:53 +0200 Subject: [PATCH 9/9] Add missing documentation --- include/mzpeak/util/manager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mzpeak/util/manager.h b/include/mzpeak/util/manager.h index 533e589..a3dd001 100644 --- a/include/mzpeak/util/manager.h +++ b/include/mzpeak/util/manager.h @@ -19,7 +19,7 @@ directory of this repository. namespace MzPeak::Util { /** - * FIXME: Write documentation! + * File manager for parquet files. */ class Manager final { public: