diff --git a/include/mzpeak/metadata/scans.h b/include/mzpeak/metadata/scans.h new file mode 100644 index 0000000..7c85dd2 --- /dev/null +++ b/include/mzpeak/metadata/scans.h @@ -0,0 +1,45 @@ +/* + +This file is part of the mzpeak project. It is subject to the license +specified in the LICENSE file which can be found in the top-level +directory of this repository. + +*/ + +#pragma once + +#include +#include + +namespace MzPeak::Util { +class Parquet; +} + +namespace MzPeak::Metadata { + +/** + * The method of precursor-ion selection and activation. + */ +class Scans final { +public: + /// Data for all scans. + struct Raw { + std::vector scan_start_time; + }; + + /// Default constructor; + Scans(); + + /// Constructor. + Scans(std::unique_ptr, uint64_t); + + /** + * Return the raw table data for all scans. + */ + const Raw& raw() const { return raw_; } + +private: + Raw raw_; +}; + +} // namespace MzPeak::Metadata diff --git a/include/mzpeak/metadata/spectrum.h b/include/mzpeak/metadata/spectrum.h index 7dcb41b..60a0830 100644 --- a/include/mzpeak/metadata/spectrum.h +++ b/include/mzpeak/metadata/spectrum.h @@ -40,8 +40,14 @@ class Spectrum final { */ const std::vector& delta_model() const { return delta_model_; } + /** + * The scan time in minutes. + */ + std::optional scan_time() const { return scan_time_; }; + private: std::optional ms_level_; + std::optional scan_time_; std::vector delta_model_; }; diff --git a/include/mzpeak/schema/data_kind.h b/include/mzpeak/schema/data_kind.h index 5adb5d2..e7b4b4e 100644 --- a/include/mzpeak/schema/data_kind.h +++ b/include/mzpeak/schema/data_kind.h @@ -8,44 +8,79 @@ directory of this repository. #pragma once +#include #include +#include namespace MzPeak::Schema { /** * Indicates how data is encoded in a parquet file. */ -enum class DataKind { - /// Files that contain signal data. - DataArray, - - /// Processed data (e.g. centroided). It is expected that the - /// unprocessed version of the data is present in the same MzPeak - /// file as a DataArray. - Peaks, - - /// Metadata relating to one of the other files. - Metadata, - - /// Non-standard file that can't be decoded by this library. - /// However, users of this library can access the raw bytes of - /// this file. - Proprietary, - - /// Non-standard file that can't be decoded by this library. - /// However, users of this library can access the raw bytes of - /// this file. - Other -}; +class DataKind final { +public: + enum Type { + /// Files that contain signal data. + DataArray, -/** - * Convert a DataKind to a string. - */ -std::string data_kind_to_string(DataKind); + /// Processed data (e.g. centroided). It is expected that the + /// unprocessed version of the data is present in the same MzPeak + /// file as a DataArray. + Peaks, -/** - * Parse a DataKind from a string view. - */ -DataKind data_kind_from_string(std::string_view); + /// Metadata relating to one of the other files. + Metadata, + + /// A scan or acquisition from the original raw file used to + /// create a spectrum. + Scans, + + /// The method of precursor-ion selection and activation. + Precursors, + + /// An ion isolated for dissociation. + SelectedIons, + + /// When describing single reaction monitoring (SRM) or multiple + /// reaction monitoring (MRM) experiments, each product ion is + /// isolated separately with a different isolation window. This + /// table is usually empty or absent + Products, + + /// Non-standard file that can't be decoded by this library. + /// However, users of this library can access the raw bytes of + /// this file. + Proprietary, + }; + + // Internal storage type. + using value_type = std::variant; + + /// Constructor. + DataKind(std::string_view); + + /// Constructor. + DataKind(Type); + + /** + * Return the string representation of the data kind. + */ + std::string to_string() const; + + /** + * Return the enumerated type of the data kind if it is known. + */ + std::optional type() const; + + /** + * Return `true` if this data kind is a known holder of metadata. + * For example, the `Scans` and `Precursors` type are both + * considered to be metadata. + */ + bool is_metadata() const; + +private: + value_type val_; +}; } // namespace MzPeak::Schema diff --git a/include/mzpeak/schema/file.h b/include/mzpeak/schema/file.h index 527ce40..264e155 100644 --- a/include/mzpeak/schema/file.h +++ b/include/mzpeak/schema/file.h @@ -35,11 +35,6 @@ class File final { /// Conversion from JSON. explicit File(const json::object&); - /// Return `true` if this file is associated with the given file. - /// For example, if this file is a DataArray and the other file is a - /// Metadata file with a similar name. - bool is_associated_with(const File&) const; - /// Equality operator. bool operator==(const File&) const; @@ -57,7 +52,7 @@ class File final { private: std::string file_name_; - DataKind data_kind_ = DataKind::Other; + DataKind data_kind_ = DataKind("other"); EntityType entity_type_ = EntityType::Other; std::vector columns_; }; diff --git a/include/mzpeak/spectrum.h b/include/mzpeak/spectrum.h index 0df37ae..269355b 100644 --- a/include/mzpeak/spectrum.h +++ b/include/mzpeak/spectrum.h @@ -12,6 +12,7 @@ top-level directory of this repository. #include #include "mzpeak/data/array_index.h" +#include "mzpeak/metadata/scans.h" namespace MzPeak { @@ -50,6 +51,22 @@ class Spectrum final { */ uint8_t ms_level() const; + /** + * Scan time in minutes. + * + * If the scan time was not stored in the spectra metadata then it + * will be fetched from the scans array. + * + * If the scan metadata is also empty then this method will return + * 0.0. + */ + double scan_time(); + + /** + * Return scan details. + */ + const Metadata::Scans& scans(); + protected: friend class Spectra; @@ -66,6 +83,8 @@ class Spectrum final { std::vector mz_; std::vector intensity_; uint8_t ms_level_; + std::optional scan_time_; + std::optional scans_; }; } // namespace MzPeak diff --git a/include/mzpeak/util/manager.h b/include/mzpeak/util/manager.h index a3dd001..2b2bde6 100644 --- a/include/mzpeak/util/manager.h +++ b/include/mzpeak/util/manager.h @@ -34,7 +34,13 @@ class Manager final { /** * Find a file given its name. */ - std::vector::const_iterator find_file(std::string_view name) const; + std::vector::const_iterator find_file(std::string_view) const; + + /** + * Find a file given its `EntityType` and `DataKind`. + */ + std::vector::const_iterator find_file(Schema::EntityType, + Schema::DataKind::Type) const; /** * Open a Parquet file from the mzPeak archive. diff --git a/include/mzpeak/util/slice.h b/include/mzpeak/util/slice.h index cb0372d..99eafce 100644 --- a/include/mzpeak/util/slice.h +++ b/include/mzpeak/util/slice.h @@ -70,6 +70,24 @@ class Slice final { template > void singleton(const Column&, R&, T&& = {}); + /** + * A helper function that calls the `singleton` method with the + * scalar decoder. + */ + template void scalar(const Schema::Column&, T&); + + /// Overload that resets the destination type if column is nullopt. + template void scalar(const std::optional&, T&); + + /** + * A helper function that calls the `singleton` method with the list + * decoder. + */ + template void list(const Schema::Column&, T&); + + /// Overload that resets the destination type if column is nullopt. + template void list(const std::optional&, T&); + /** * Extract and decode an array. The array is then removed from the * internal storage. @@ -86,11 +104,25 @@ class Slice final { requires Decoders::from_arrow_array void array(const Column&, V&, T&& = {}); - /****************************************************************************/ + /// Override that takes an optional column. + template > + requires Decoders::from_arrow_array + void array(const std::optional&, V&, T&& = {}); + + /// Override that takes a reference to a decoder. template > requires Decoders::from_arrow_array void array(const Column&, V&, T&); + /** + * A helper function that calls `array` with a scalar decoder that + * skips null values. + */ + template void non_null(const Column&, std::vector&); + + /// Override that takes an optional column. + template void non_null(const std::optional&, std::vector&); + private: friend class MzPeak::Util::Executor; @@ -106,7 +138,7 @@ class Slice final { /******************************************************************************/ template -void Slice::singleton(const Column& field, R& dst, T&& t) +void Slice::singleton(const Column& field, R& dst, T&& decoder) { std::shared_ptr chunks = raw(field); @@ -117,11 +149,11 @@ void Slice::singleton(const Column& field, R& dst, T&& t) std::shared_ptr sliced = chunk->Slice(i, 1); if constexpr (std::is_same_v) { - t.decode(sliced, dst); + decoder.decode(sliced, dst); return; } else { typename T::value_type val{}; - t.decode(sliced, val); + decoder.decode(sliced, val); dst = {std::move(val)}; return; } @@ -133,6 +165,47 @@ void Slice::singleton(const Column& field, R& dst, T&& t) dst = {}; } +/******************************************************************************/ +template void Slice::scalar(const Schema::Column& field, T& dst) +{ + if constexpr (requires { typename T::value_type; }) { + using V = typename T::value_type; + singleton, T>(field, dst); + } else { + singleton, T>(field, dst); + } +} + +/******************************************************************************/ +template +void Slice::scalar(const std::optional& field, T& dst) +{ + if (field.has_value()) { + scalar(field.value(), dst); + } else { + dst = {}; + } +} + +/******************************************************************************/ +template void Slice::list(const Schema::Column& field, T& dst) +{ + using V = T::value_type; + using D = Decoders::List; + singleton(field, dst); +} + +/******************************************************************************/ +template +void Slice::list(const std::optional& field, T& dst) +{ + if (field.has_value()) { + list(field.value(), dst); + } else { + dst = {}; + } +} + /******************************************************************************/ template requires Decoders::from_arrow_array @@ -141,6 +214,16 @@ void Slice::array(const Column& field, V& v, T&& t) array(field, v, t); } +/******************************************************************************/ +template + requires Decoders::from_arrow_array +void Slice::array(const std::optional& field, V& v, T&& t) +{ + if (field.has_value()) { + array(field.value(), v, t); + } +} + /******************************************************************************/ template requires Decoders::from_arrow_array @@ -164,4 +247,20 @@ void Slice::array(const Column& field, V& v, T& t) } } +/******************************************************************************/ +template void Slice::non_null(const Column& field, std::vector& dst) +{ + using D = Decoders::Scalar, Decoders::NullSkip>; + array(field, dst); +} + +/******************************************************************************/ +template +void Slice::non_null(const std::optional& field, std::vector& dst) +{ + if (field.has_value()) { + non_null(field.value(), dst); + } +} + } // namespace MzPeak::Util diff --git a/meson.build b/meson.build index 13d6d8b..83c35d8 100644 --- a/meson.build +++ b/meson.build @@ -27,6 +27,7 @@ lib_sources = [ 'src/io/directory.cpp', 'src/io/file.cpp', 'src/io/zip.cpp', + 'src/metadata/scans.cpp', 'src/metadata/spectrum.cpp', 'src/metadata/table.cpp', 'src/open.cpp', @@ -70,6 +71,7 @@ install_headers([ 'include/mzpeak/io/directory.h', 'include/mzpeak/io/file.h', 'include/mzpeak/io/zip.h', + 'include/mzpeak/metadata/scans.h', 'include/mzpeak/metadata/spectrum.h', 'include/mzpeak/metadata/table.h', 'include/mzpeak/open.h', diff --git a/src/metadata/scans.cpp b/src/metadata/scans.cpp new file mode 100644 index 0000000..84bac01 --- /dev/null +++ b/src/metadata/scans.cpp @@ -0,0 +1,40 @@ +/* + +This file is part of the mzpeak project. It is subject to the license +specified in the LICENSE file which can be found in the top-level +directory of this repository. + +*/ + +#include "mzpeak/metadata/scans.h" +#include "mzpeak/metadata/table.h" +#include "mzpeak/schema/group.h" +#include "mzpeak/util/projection.h" +#include "mzpeak/util/slice.h" + +namespace MzPeak::Metadata { + +using namespace MzPeak::Schema; +using namespace MzPeak::Util; + +/******************************************************************************/ +Scans::Scans() + : raw_() +{ +} + +/******************************************************************************/ +Scans::Scans(std::unique_ptr parquet, uint64_t index) + : raw_() +{ + Table table(std::move(parquet)); + std::shared_ptr group = table.group("root"); + + Projection projection; + auto scan_time_field = projection.project(group, Group::CVType("MS", "1000016")); + + std::unique_ptr slice = table.indexed(index, group, projection); + slice->non_null(scan_time_field, raw_.scan_start_time); +} + +} // namespace MzPeak::Metadata diff --git a/src/metadata/spectrum.cpp b/src/metadata/spectrum.cpp index f7f1e4a..8728341 100644 --- a/src/metadata/spectrum.cpp +++ b/src/metadata/spectrum.cpp @@ -8,7 +8,6 @@ top-level directory of this repository. #include -#include "mzpeak/exception.h" #include "mzpeak/metadata/spectrum.h" #include "mzpeak/metadata/table.h" #include "mzpeak/schema/group.h" @@ -23,6 +22,7 @@ using namespace MzPeak::Util; /******************************************************************************/ Spectrum::Spectrum(std::unique_ptr parquet, uint64_t index) : ms_level_() + , scan_time_() , delta_model_() { Table table(std::move(parquet)); @@ -30,21 +30,14 @@ Spectrum::Spectrum(std::unique_ptr parquet, uint64_t index) Projection projection; auto level_field = projection.project(group, Group::CVType("MS", "1000511")); + auto scan_time_field = projection.project(group, "time"); auto delta_field = projection.project(group, "mz_delta_model"); std::unique_ptr slice = table.indexed(index, group, projection); - if (level_field.has_value()) { - using ms_level_t = decltype(ms_level_)::value_type; - using decoder = Decoders::Scalar; - slice->singleton(*level_field, ms_level_); - } - - if (delta_field.has_value()) { - using delta_type = decltype(delta_model_)::value_type; - using decoder = Decoders::List; - slice->singleton(*delta_field, delta_model_); - } + slice->scalar(level_field, ms_level_); + slice->scalar(scan_time_field, scan_time_); + slice->list(delta_field, delta_model_); } } // namespace MzPeak::Metadata diff --git a/src/metadata/table.cpp b/src/metadata/table.cpp index 06ca5bc..174bf57 100644 --- a/src/metadata/table.cpp +++ b/src/metadata/table.cpp @@ -21,20 +21,39 @@ struct Table::Impl { ~Impl(); std::unique_ptr parquet_; - std::optional n_entries; + std::string index_field_name_; }; /******************************************************************************/ Table::Impl::Impl(std::unique_ptr parquet) : parquet_(std::move(parquet)) - , n_entries() + , index_field_name_("index") { auto file = parquet_->index_file(); - if (file.data_kind() != Schema::DataKind::Metadata) { + if (!file.data_kind().is_metadata()) { std::string msg("file is not a metadata file: " + file.file_name()); throw InvalidFormatError(msg); } + + auto type = file.data_kind().type(); + if (type.has_value()) { + using enum Schema::DataKind::Type; + + switch (type.value()) { + case DataArray: + case Peaks: + case Metadata: + case Proprietary: + break; + case Scans: + case Precursors: + case SelectedIons: + case Products: + index_field_name_ = "source_index"; + break; + } + } } /******************************************************************************/ @@ -69,8 +88,11 @@ Table::indexed(uint64_t index, const std::shared_ptr& group, const Util::Projection& projection) const { - auto index_field = group->field("index"); - if (!index_field.has_value()) return nullptr; + auto index_field = group->field(impl_->index_field_name_); + + if (!index_field.has_value()) { + throw InvalidFormatError("metadata column missing: " + impl_->index_field_name_); + } auto column = std::make_pair(group, index_field.value()); Util::Query q = Util::Query::Builder(column).eq(index); diff --git a/src/schema/data_kind.cpp b/src/schema/data_kind.cpp index 200ce4a..758272d 100644 --- a/src/schema/data_kind.cpp +++ b/src/schema/data_kind.cpp @@ -6,39 +6,20 @@ directory of this repository. */ +#include + #include "mzpeak/schema/data_kind.h" namespace MzPeak::Schema { /******************************************************************************/ -std::string data_kind_to_string(DataKind dk) +DataKind::value_type data_kind_from_string(std::string_view s) { - using enum DataKind; - - switch (dk) { - case DataArray: - return "data_arrays"; - case Peaks: - return "peaks"; - case Metadata: - return "metadata"; - case Proprietary: - return "proprietary"; - case Other: - return "other"; - default: - return "other"; - } -} - -/******************************************************************************/ -DataKind data_kind_from_string(std::string_view s) -{ - using enum DataKind; - // The two string variants of `DataArray` are for backwards // compatibility. See https://github.com/HUPO-PSI/mzPeak/issues/26 + using enum DataKind::Type; + if (s == "data_arrays") { return DataArray; } else if (s == "data arrays") { @@ -47,10 +28,106 @@ DataKind data_kind_from_string(std::string_view s) return Peaks; } else if (s == "metadata") { return Metadata; + } else if (s == "scans") { + return Scans; + } else if (s == "precursors") { + return Precursors; + } else if (s == "selected_ions") { + return SelectedIons; + } else if (s == "products") { + return Products; } else if (s == "proprietary") { return Proprietary; } else { - return Other; + return std::string(s); + } +} + +/******************************************************************************/ +DataKind::DataKind(std::string_view s) + : val_(data_kind_from_string(s)) +{ +} + +/******************************************************************************/ +DataKind::DataKind(Type t) + : val_(t) +{ +} + +/******************************************************************************/ +std::string DataKind::to_string() const +{ + return std::visit( + [](auto&& v) -> std::string { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + return v; + } else { + switch (v) { + case DataArray: + return "data_arrays"; + case Peaks: + return "peaks"; + case Metadata: + return "metadata"; + case Scans: + return "scans"; + case Precursors: + return "precursors"; + case SelectedIons: + return "selected_ions"; + case Products: + return "products"; + case Proprietary: + return "proprietary"; + } + } + + std::unreachable(); + }, + val_); +} + +/******************************************************************************/ +std::optional DataKind::type() const +{ + if (std::holds_alternative(val_)) { + return std::get(val_); + } else { + return {}; + } +} + +/******************************************************************************/ +bool DataKind::is_metadata() const +{ + auto enum_type = type(); + + if (enum_type.has_value()) { + switch (enum_type.value()) { + case DataArray: + return false; + case Peaks: + return false; + case Metadata: + return true; + case Scans: + return true; + case Precursors: + return true; + case SelectedIons: + return true; + case Products: + return true; + case Proprietary: + return false; + } + + std::unreachable(); + } else { + return false; } } diff --git a/src/schema/file.cpp b/src/schema/file.cpp index 37f3cac..a377c41 100644 --- a/src/schema/file.cpp +++ b/src/schema/file.cpp @@ -56,7 +56,7 @@ File::File(const std::string& name) /******************************************************************************/ File::File(const json::object& o) : file_name_(o.at("name").as_string()) - , data_kind_(data_kind_from_string(o.at("data_kind").as_string())) + , data_kind_(o.at("data_kind").as_string()) , entity_type_(entity_type_from_string(o.at("entity_type").as_string())) , columns_() { @@ -67,23 +67,6 @@ File::File(const json::object& o) } } -/******************************************************************************/ -bool File::is_associated_with(const File& other) const -{ - std::string::size_type underscore(file_name_.find("_")); - if (underscore == std::string::npos) return false; - if (other.file_name_.size() < underscore) return false; - - if (file_name_.compare(0, underscore, other.file_name_, 0, underscore) != 0) { - return false; - } - - return (data_kind_ == DataKind::DataArray && - other.data_kind_ == DataKind::Metadata) || - (data_kind_ == DataKind::Metadata && - other.data_kind_ == DataKind::DataArray); -} - /******************************************************************************/ bool File::operator==(const File& other) const { diff --git a/src/spectrum.cpp b/src/spectrum.cpp index 1176ffc..7f7de6f 100644 --- a/src/spectrum.cpp +++ b/src/spectrum.cpp @@ -30,6 +30,8 @@ Spectrum::Spectrum(uint64_t index, , mz_() , intensity_() , ms_level_(0) + , scan_time_() + , scans_() { auto meta_it = manager_->find_file(PRIMARY_METADATA_FILE); @@ -40,6 +42,7 @@ Spectrum::Spectrum(uint64_t index, Metadata::Spectrum meta(manager_->parquet(*meta_it), index_); ms_level_ = meta.ms_level().value_or(0); + scan_time_ = meta.scan_time(); Data::Encoding::Decoder decoder( std::move(data), std::move(slice), @@ -66,4 +69,42 @@ const std::vector& Spectrum::intensity() const { return intensity_; } /******************************************************************************/ uint8_t Spectrum::ms_level() const { return ms_level_; } +/******************************************************************************/ +double Spectrum::scan_time() +{ + if (scan_time_.has_value()) { + return scan_time_.value(); + } else { + auto raw = scans().raw(); + + if (!raw.scan_start_time.empty()) { + scan_time_ = raw.scan_start_time.front(); + return scan_time_.value(); + } else { + return {}; + } + } +} + +/******************************************************************************/ +const Metadata::Scans& Spectrum::scans() +{ + using namespace Schema; + + if (scans_.has_value()) { + return scans_.value(); + } else { + auto file = manager_->find_file(EntityType::Spectrum, DataKind::Type::Scans); + + if (file == manager_->files().end()) { + // No scans. + scans_ = Metadata::Scans(); + return scans_.value(); + } + + scans_ = Metadata::Scans(manager_->parquet(*file), index_); + return scans_.value(); + } +} + } // namespace MzPeak diff --git a/src/util/manager.cpp b/src/util/manager.cpp index 39af105..46156cf 100644 --- a/src/util/manager.cpp +++ b/src/util/manager.cpp @@ -71,6 +71,16 @@ Manager::find_file(std::string_view name) const return std::ranges::find(files_, name, &Schema::File::file_name); } +/******************************************************************************/ +std::vector::const_iterator +Manager::find_file(Schema::EntityType et, Schema::DataKind::Type dkt) const +{ + return std::ranges::find_if(files_, [&et, &dkt](const auto& file) -> bool { + auto type = file.data_kind().type(); + return file.entity_type() == et && type.has_value() && type.value() == dkt; + }); +} + /******************************************************************************/ std::unique_ptr Manager::parquet(const Schema::File& file) const { diff --git a/test/index_test.cpp b/test/index_test.cpp index 3d55823..846ce3b 100644 --- a/test/index_test.cpp +++ b/test/index_test.cpp @@ -9,38 +9,13 @@ directory of this repository. #define BOOST_TEST_MODULE Index #include -#include - #include "mzpeak/index.h" #include "mzpeak/open.h" -#include "mzpeak/schema/file.h" /******************************************************************************/ BOOST_AUTO_TEST_CASE(can_parse_json) { auto index = MzPeak::open("../test/files/small.mzpeak"); const auto& files = index.files(); - BOOST_TEST(!files.empty(), "files should not be empty but is"); -} - -/******************************************************************************/ -BOOST_AUTO_TEST_CASE(is_associated_with) -{ - auto index = MzPeak::open("../test/files/small.mzpeak"); - - const auto& files = index.files(); - const auto& spectra = index.find("spectra_data.parquet"); - BOOST_TEST((spectra != files.end()), "missing spectra_data.parquet"); - - auto matches = [&](const auto& other) -> bool { - return other != *spectra && spectra->is_associated_with(other); - }; - - for (const auto& other : files | std::views::filter(matches)) { - BOOST_TEST_CONTEXT(spectra->file_name() - << " should not be associated with " << other.file_name()) - { - BOOST_TEST((other.file_name() == "spectra_metadata.parquet")); - } - } + BOOST_TEST(!files.empty(), "files should not be empty"); } diff --git a/test/spectra_test.cpp b/test/spectra_test.cpp index 410af51..26574c4 100644 --- a/test/spectra_test.cpp +++ b/test/spectra_test.cpp @@ -64,6 +64,7 @@ BOOST_AUTO_TEST_CASE(can_read_spectra) BOOST_TEST_REQUIRE(intensity[intensity.size() - 1] == 0.0, tolerance); BOOST_TEST_REQUIRE(spectrum.ms_level() == 1u); + BOOST_TEST_REQUIRE(spectrum.scan_time() == 0.004935, tolerance); } };