diff --git a/bin/mzp-inspect.cpp b/bin/mzp-inspect.cpp index f2ab8c4..5b008c7 100644 --- a/bin/mzp-inspect.cpp +++ b/bin/mzp-inspect.cpp @@ -14,6 +14,7 @@ top-level directory of this repository. #include "mzpeak.h" #include "mzpeak/schema/group.h" +#include "mzpeak/util/manager.h" // IWYU pragma: keep /******************************************************************************/ namespace po = boost::program_options; @@ -22,14 +23,14 @@ namespace po = boost::program_options; std::unique_ptr open_parquet_file(MzPeak::Index& index, const std::string& file) { - auto it = std::ranges::find(index.files(), file, &MzPeak::Schema::File::file_name); + auto it = index.find(file); if (it == index.files().end()) { std::println(stderr, "file \"{}\" is not in the mzPeak file index", file); return nullptr; } - return index.parquet(*it); + return index.manager()->parquet(*it); } /******************************************************************************/ @@ -39,7 +40,7 @@ int print_array_index(MzPeak::Index& index, const std::string& file) if (parquet == nullptr) return 1; auto fmd = parquet->file_metadata(); - auto et = parquet->index_file().entity_type; + auto et = parquet->index_file().entity_type(); auto key = MzPeak::Schema::entity_type_to_string(et) + "_array_index"; auto json = parquet->kv_string(fmd, key); diff --git a/include/mzpeak/index.h b/include/mzpeak/index.h index b3dc12a..93dfdb8 100644 --- a/include/mzpeak/index.h +++ b/include/mzpeak/index.h @@ -8,14 +8,24 @@ directory of this repository. #pragma once -#include "mzpeak/io/archive.h" -#include "mzpeak/schema/file.h" -#include "mzpeak/spectra.h" -#include "mzpeak/util/parquet.h" +#include +#include namespace MzPeak { -// Internal implementation. +namespace IO { +class Archive; +} + +namespace Schema { +class File; +} + +namespace Util { +class Manager; +} + +class Spectra; /** * Read-only access to the index inside a MzPeak archive. @@ -25,27 +35,28 @@ class Index { /// Constructor. Index(std::unique_ptr); - /// Destructor. - ~Index(); - /** * Return a list of files found in the index. */ const std::vector& files() const; + /** + * Find a file in the mzPeak archive with the given name. + */ + std::vector::const_iterator find(const std::string_view&) const; + /** * Access the spectra in the file. */ Spectra spectra() const; /** - * Open a Parquet file directly. + * Access the low-level MzPeak Manager object. */ - std::unique_ptr parquet(const Schema::File&) const; + std::shared_ptr manager() const; protected: - struct Impl; - std::unique_ptr impl_; + std::shared_ptr manager_; }; } // namespace MzPeak diff --git a/include/mzpeak/metadata/spectrum.h b/include/mzpeak/metadata/spectrum.h index a686c2a..7dcb41b 100644 --- a/include/mzpeak/metadata/spectrum.h +++ b/include/mzpeak/metadata/spectrum.h @@ -13,27 +13,18 @@ top-level directory of this repository. #include namespace MzPeak::Util { -class Slice; -} - -namespace MzPeak::Schema { -class Group; +class Parquet; } namespace MzPeak::Metadata { -class Table; - /** * Metadata from the spectrum table. */ class Spectrum final { public: /// Constructor. - Spectrum(std::shared_ptr, uint64_t); - - /// Destructor. - ~Spectrum() = default; + Spectrum(std::unique_ptr, uint64_t); /** * Return the spectrum level (MS:1000511). @@ -50,10 +41,6 @@ class Spectrum final { const std::vector& delta_model() const { return delta_model_; } private: - std::shared_ptr
table_; - std::shared_ptr group_; - uint64_t index_; - std::optional ms_level_; std::vector delta_model_; }; diff --git a/include/mzpeak/schema/file.h b/include/mzpeak/schema/file.h index 618ffe3..527ce40 100644 --- a/include/mzpeak/schema/file.h +++ b/include/mzpeak/schema/file.h @@ -17,13 +17,20 @@ directory of this repository. namespace MzPeak::Schema { namespace json = boost::json; -struct File { +/** + * A description of a file in the mzPeak archive. + */ +class File final { +public: + struct Column { + std::string name; + std::string path; + std::optional accession; + std::optional unit; + }; /// Constructor from a file name. - explicit File(const std::string& name) - : file_name(name) - { - } + explicit File(const std::string& name); /// Conversion from JSON. explicit File(const json::object&); @@ -34,16 +41,25 @@ struct File { bool is_associated_with(const File&) const; /// Equality operator. - bool operator==(const File&) const = default; + bool operator==(const File&) const; /// The name of this file. - std::string file_name; + const std::string& file_name() const { return file_name_; } /// This file's data kind. - DataKind data_kind = DataKind::Other; + DataKind data_kind() const { return data_kind_; } /// This file's entity type. - EntityType entity_type = EntityType::Other; + EntityType entity_type() const { return entity_type_; } + + /// Column definitions for this file. + const std::vector& columns() const { return columns_; } + +private: + std::string file_name_; + DataKind data_kind_ = DataKind::Other; + EntityType entity_type_ = EntityType::Other; + std::vector columns_; }; } // namespace MzPeak::Schema diff --git a/include/mzpeak/schema/group.h b/include/mzpeak/schema/group.h index 5617752..9b4550e 100644 --- a/include/mzpeak/schema/group.h +++ b/include/mzpeak/schema/group.h @@ -14,10 +14,12 @@ top-level directory of this repository. #include #include "mzpeak/schema/cv.h" +#include "mzpeak/schema/file.h" #include "mzpeak/util/types.h" // Forward declarations. namespace parquet::schema { +class Node; class GroupNode; } // namespace parquet::schema @@ -87,16 +89,10 @@ class Group final { */ index_type absolute_index() const; - /** - * The name of this field using underscores to replace spaces and - * other special characters. - */ - const std::string& name() const; - /** * The name of this field as recognized by parquet. */ - const std::string& schema_name() const; + const std::string& name() const; /** * The structural type this field represents. @@ -129,7 +125,6 @@ class Group final { index_type rel_index_; index_type abs_index_; std::string schema_name_; - std::string clean_name_; std::optional cv_type_; std::optional cv_unit_; std::optional type_; @@ -140,8 +135,13 @@ class Group final { /// Fields are stored in a map for quick look-up using their name. using field_map_t = std::map>; + /// Constructor for the root group to hold all of the top-level + /// columns that are not in a separate struct/group. + explicit Group(const parquet::schema::GroupNode&, const Schema::File&); + /// Constructor from a parquet schema descriptor. explicit Group(const parquet::schema::GroupNode&, + const Schema::File&, index_type index, index_type offset); @@ -176,8 +176,27 @@ class Group final { */ const field_map_t& fields() const; + /** + * Return true if this is the root group. + * + * There is only one root group and its fields represent the + * top-level columns that are not themselves members of a group or + * struct. + */ + bool is_root() const; + + /** + * Return a schema path to the given field. Mostly useful for error + * messages. + */ + std::string path(const Field&) const; + private: + void + make_fields(const parquet::schema::GroupNode&, const Schema::File&, index_type); + std::string name_; + bool is_root_; index_type index_; field_map_t fields_; }; diff --git a/include/mzpeak/spectra.h b/include/mzpeak/spectra.h index ddf7f8e..231ceba 100644 --- a/include/mzpeak/spectra.h +++ b/include/mzpeak/spectra.h @@ -16,8 +16,8 @@ namespace MzPeak::Data { class Signals; } // namespace MzPeak::Data -namespace MzPeak::Metadata { -class Table; +namespace MzPeak::Util { +class Manager; } namespace MzPeak { @@ -28,12 +28,12 @@ namespace MzPeak { class Spectra final : public Util::EnumerableProxy { public: /// Low-level constructor from a Parquet file. - explicit Spectra(std::unique_ptr, std::unique_ptr); + explicit Spectra(std::unique_ptr, std::shared_ptr); private: // Internal data access. std::shared_ptr data_; - std::shared_ptr meta_; + std::shared_ptr manager_; // Function to fetch a specific spectrum. Spectrum fetch(uint64_t); diff --git a/include/mzpeak/spectrum.h b/include/mzpeak/spectrum.h index e85dc70..0df37ae 100644 --- a/include/mzpeak/spectrum.h +++ b/include/mzpeak/spectrum.h @@ -9,17 +9,21 @@ top-level directory of this repository. #pragma once #include +#include #include "mzpeak/data/array_index.h" -#include "mzpeak/data/encoding.h" -#include "mzpeak/data/signals.h" -#include "mzpeak/metadata/spectrum.h" -#include "mzpeak/metadata/table.h" -#include "mzpeak/util/slice.h" namespace MzPeak { -// Forward declaration. +namespace Data { +class Signals; +} // namespace Data + +namespace Util { +class Slice; +class Manager; +} // namespace Util + class Spectra; /** @@ -27,9 +31,6 @@ class Spectra; */ class Spectrum final { public: - /// The type of decoder used. - using decoder_type = Data::Encoding::Decoder; - /// Destructor. ~Spectrum() = default; @@ -54,18 +55,17 @@ class Spectrum final { /// Internal constructor. Spectrum(uint64_t index, + std::shared_ptr, std::shared_ptr, const std::vector&, - std::unique_ptr, - std::shared_ptr); + std::unique_ptr); private: uint64_t index_; - std::shared_ptr md_table_; - Metadata::Spectrum md_spec_; - decoder_type decoder_; + std::shared_ptr manager_; std::vector mz_; std::vector intensity_; + uint8_t ms_level_; }; } // namespace MzPeak diff --git a/include/mzpeak/util/manager.h b/include/mzpeak/util/manager.h new file mode 100644 index 0000000..ca88079 --- /dev/null +++ b/include/mzpeak/util/manager.h @@ -0,0 +1,48 @@ +/* + +This file is part of the mzpeak project. It is subject to the license +specified in the LICENSE file which can be found in the top-level +directory of this repository. + +*/ + +#pragma once + +#include + +#include "mzpeak/io/archive.h" +#include "mzpeak/schema/file.h" +#include "mzpeak/util/parquet.h" + +namespace MzPeak::Util { + +/** + * FIXME: Write documentation! + */ +class Manager final { +public: + /// Constructor. + Manager(std::unique_ptr); + + /** + * Return a vector of files that are located in the mzPeak archive. + */ + const std::vector& files() const; + + /** + * Find a file given its name. + */ + std::vector::const_iterator + find_file(const std::string_view& name) const; + + /** + * Open a Parquet file from the mzPeak archive. + */ + std::unique_ptr parquet(const Schema::File&) const; + +private: + std::shared_ptr archive_; + std::vector files_; +}; + +} // namespace MzPeak::Util diff --git a/meson.build b/meson.build index 59b0653..62a13f7 100644 --- a/meson.build +++ b/meson.build @@ -43,6 +43,7 @@ lib_sources = [ 'src/spectrum.cpp', 'src/util/arrow.cpp', 'src/util/executor.cpp', + 'src/util/manager.cpp', 'src/util/parquet.cpp', 'src/util/planner.cpp', 'src/util/projection.cpp', @@ -85,6 +86,7 @@ install_headers([ 'include/mzpeak/util/delta_estimator.h', 'include/mzpeak/util/enumerable_proxy.h', 'include/mzpeak/util/executor.h', + 'include/mzpeak/util/manager.h', 'include/mzpeak/util/parquet.h', 'include/mzpeak/util/planner.h', 'include/mzpeak/util/projection.h', diff --git a/src/data/array_index.cpp b/src/data/array_index.cpp index 9d2cff6..6324057 100644 --- a/src/data/array_index.cpp +++ b/src/data/array_index.cpp @@ -44,7 +44,7 @@ ArrayIndex::ArrayIndex(EntityType entity_type, const json::object& obj) if (entries != obj.end() && entries->value().is_array()) { auto entries_ary(entries->value().as_array()); - entries_.reserve(entries_ary.size() + 1); + entries_.reserve(entries_ary.size()); for (const auto& entry_obj : entries_ary) { if (entry_obj.is_object()) { diff --git a/src/data/signals.cpp b/src/data/signals.cpp index 4da918b..6144bc1 100644 --- a/src/data/signals.cpp +++ b/src/data/signals.cpp @@ -36,7 +36,7 @@ struct Signals::Impl { std::shared_ptr Signals::Impl::parse_array_index() const { Util::Parquet::file_metadata_t fmd(parquet_->file_metadata()); - EntityType entity_type = parquet_->index_file().entity_type; + EntityType entity_type = parquet_->index_file().entity_type(); std::string num_key(Schema::entity_type_to_string(entity_type) + "_count"); std::optional num_entities(parquet_->kv_size_t(fmd, num_key)); diff --git a/src/index.cpp b/src/index.cpp index db9b02a..9c41888 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -12,122 +12,43 @@ directory of this repository. #include "mzpeak/exception.h" #include "mzpeak/index.h" #include "mzpeak/io/archive.h" -#include "mzpeak/metadata/table.h" +#include "mzpeak/spectra.h" +#include "mzpeak/util/manager.h" -/* - * Boost JSON: - * https://www.boost.org/doc/libs/latest/libs/json/doc/html/index.html - */ namespace MzPeak { -/******************************************************************************/ -const static char* INDEX_FILE_NAME = "mzpeak_index.json"; - -/******************************************************************************/ -namespace json = boost::json; - -/******************************************************************************/ -struct Index::Impl { - - /// Constructor. - Impl(std::unique_ptr archive) - : archive_(std::move(archive)) - , files_() - { - parse_index(); - } - - /// Parse the JSON that makes up the MzPeak index. - void parse_index(); - - // The archive we are reading files out of. - std::unique_ptr archive_; - - // Parsed file entries. - std::vector files_; - - // Return an iterator to the requested file. - std::vector::const_iterator find_file(const std::string_view& name) - { - return std::ranges::find(files_, name, &Schema::File::file_name); - } -}; - /******************************************************************************/ Index::Index(std::unique_ptr archive) - : impl_(std::make_unique(std::move(archive))) + : manager_(std::make_shared(std::move(archive))) { } /******************************************************************************/ -Index::~Index() = default; +const std::vector& Index::files() const { return manager_->files(); } /******************************************************************************/ -const std::vector& Index::files() const { return impl_->files_; } - -/******************************************************************************/ -void Index::Impl::parse_index() +std::vector::const_iterator +Index::find(const std::string_view& name) const { - auto file = archive_->read_file(INDEX_FILE_NAME); - uint8_t buffer[64 * 1024]; - std::optional bytes; - - json::stream_parser parser; - boost::system::error_code ec; - - do { - bytes = file->read(buffer, sizeof(buffer)); - - if (bytes.has_value() && *bytes > 0) { - parser.write(reinterpret_cast(buffer), *bytes, ec); - } - - } while (bytes.has_value() && !ec); - - if (!ec) parser.finish(ec); - if (ec) throw MzPeak::JsonError(ec.message()); - - json::value v = parser.release(); - json::object o = v.as_object(); - - if (const auto it = o.find("files"); it != o.end() && it->value().is_array()) { - const json::array files(it->value().as_array()); - files_.reserve(files.size()); - - for (const auto& file_obj : files) { - if (file_obj.is_object()) { - files_.push_back(Schema::File(file_obj.as_object())); - } - } - } + return manager_->find_file(name); } /******************************************************************************/ Spectra Index::spectra() const { - auto data_it = impl_->find_file("spectra_data.parquet"); - auto meta_it = impl_->find_file("spectra_metadata.parquet"); + auto data_it = manager_->find_file("spectra_data.parquet"); - if (data_it == impl_->files_.end()) { + if (data_it == manager_->files().end()) { throw ParquetError("missing files: spectra_data.parquet"); } std::unique_ptr data = - std::make_unique(parquet(*data_it)); - std::unique_ptr meta = nullptr; + std::make_unique(manager_->parquet(*data_it)); - if (meta_it != impl_->files_.end()) { - meta = std::make_unique(parquet(*meta_it)); - } - - return Spectra(std::move(data), std::move(meta)); + return Spectra(std::move(data), manager_); } /******************************************************************************/ -std::unique_ptr Index::parquet(const Schema::File& file) const -{ - std::unique_ptr data(impl_->archive_->read_file(file.file_name)); - return std::make_unique(std::move(data), file); -} +std::shared_ptr Index::manager() const { return manager_; } } // namespace MzPeak diff --git a/src/metadata/spectrum.cpp b/src/metadata/spectrum.cpp index 007671a..f7f1e4a 100644 --- a/src/metadata/spectrum.cpp +++ b/src/metadata/spectrum.cpp @@ -21,22 +21,18 @@ using namespace MzPeak::Schema; using namespace MzPeak::Util; /******************************************************************************/ -Spectrum::Spectrum(std::shared_ptr
table, uint64_t index) - : table_(std::move(table)) - , group_(table_ == nullptr ? nullptr : table_->group("spectrum")) - , index_(index) - , ms_level_() +Spectrum::Spectrum(std::unique_ptr parquet, uint64_t index) + : ms_level_() , delta_model_() { - if (table_ == nullptr || group_ == nullptr) { - throw ParquetError("metadata file missing or does not have the spectrum group"); - } + Table table(std::move(parquet)); + std::shared_ptr group = table.group("root"); Projection projection; - auto level_field = projection.project(group_, Group::CVType("MS", "1000511")); - auto delta_field = projection.project(group_, "mz_delta_model"); + auto level_field = projection.project(group, Group::CVType("MS", "1000511")); + auto delta_field = projection.project(group, "mz_delta_model"); - std::unique_ptr slice = table_->indexed(index_, group_, projection); + std::unique_ptr slice = table.indexed(index, group, projection); if (level_field.has_value()) { using ms_level_t = decltype(ms_level_)::value_type; diff --git a/src/metadata/table.cpp b/src/metadata/table.cpp index 14a2dd2..46197c6 100644 --- a/src/metadata/table.cpp +++ b/src/metadata/table.cpp @@ -31,8 +31,8 @@ Table::Impl::Impl(std::unique_ptr parquet) { auto file = parquet_->index_file(); - if (file.data_kind != Schema::DataKind::Metadata) { - std::string msg("file is not a metadata file: " + file.file_name); + if (file.data_kind() != Schema::DataKind::Metadata) { + std::string msg("file is not a metadata file: " + file.file_name()); throw ParquetError(msg); } } @@ -56,7 +56,8 @@ std::shared_ptr Table::group(const std::string_view& name) const auto it = map->find(std::string(name)); if (it == map->end()) { - return nullptr; + // FIXME: Replace with the InvalidFormat exception. + throw ParquetError("schema is missing the " + std::string(name) + " group"); } else { return it->second; } diff --git a/src/schema/file.cpp b/src/schema/file.cpp index 6a59650..37f3cac 100644 --- a/src/schema/file.cpp +++ b/src/schema/file.cpp @@ -10,28 +10,83 @@ directory of this repository. namespace MzPeak::Schema { +/******************************************************************************/ +void parse_columns(const json::array& input, std::vector& output) +{ + auto get_string = [](const json::object& ob, + std::string_view key) -> std::optional { + auto it = ob.find(key); + + if (it != ob.end() && it->value().is_string()) { + return std::string(it->value().as_string()); + } else { + return {}; + } + }; + + output.reserve(input.size()); + + for (const auto& column : input) { + if (!column.is_object()) { + // FIXME: emit a warning + continue; + } + + const auto& colobj(column.as_object()); + + File::Column fc = { + .name = get_string(colobj, "name").value_or(""), + .path = get_string(colobj, "path") + .or_else(std::bind(get_string, colobj, "name")) + .value_or(""), + .accession = get_string(colobj, "accession"), + .unit = get_string(colobj, "unit"), + }; + + output.push_back(fc); + } +} + +/******************************************************************************/ +File::File(const std::string& name) + : file_name_(name) +{ +} + /******************************************************************************/ File::File(const json::object& o) - : file_name(o.at("name").as_string()) - , data_kind(data_kind_from_string(o.at("data_kind").as_string())) - , entity_type(entity_type_from_string(o.at("entity_type").as_string())) + : file_name_(o.at("name").as_string()) + , data_kind_(data_kind_from_string(o.at("data_kind").as_string())) + , entity_type_(entity_type_from_string(o.at("entity_type").as_string())) + , columns_() { + auto cs = o.find("column_mapping"); + + if (cs != o.end() && cs->value().is_array()) { + parse_columns(cs->value().as_array(), columns_); + } } /******************************************************************************/ bool File::is_associated_with(const File& other) const { - std::string::size_type underscore(file_name.find("_")); + std::string::size_type underscore(file_name_.find("_")); if (underscore == std::string::npos) return false; - if (other.file_name.size() < underscore) return false; + if (other.file_name_.size() < underscore) return false; - if (file_name.compare(0, underscore, other.file_name, 0, underscore) != 0) { + if (file_name_.compare(0, underscore, other.file_name_, 0, underscore) != 0) { return false; } - return (data_kind == DataKind::DataArray && - other.data_kind == DataKind::Metadata) || - (data_kind == DataKind::Metadata && other.data_kind == DataKind::DataArray); + return (data_kind_ == DataKind::DataArray && + other.data_kind_ == DataKind::Metadata) || + (data_kind_ == DataKind::Metadata && + other.data_kind_ == DataKind::DataArray); } +/******************************************************************************/ +bool File::operator==(const File& other) const +{ + return file_name_ == other.file_name_; +}; } // namespace MzPeak::Schema diff --git a/src/schema/group.cpp b/src/schema/group.cpp index f14b2ff..78e1702 100644 --- a/src/schema/group.cpp +++ b/src/schema/group.cpp @@ -16,6 +16,14 @@ top-level directory of this repository. namespace MzPeak::Schema { +/******************************************************************************/ +template std::optional string_to_cv_child(std::string_view s) +{ + return CV::from_string(s).and_then([](const auto& cv) -> std::optional { + return T(cv.code(), cv.accession()); + }); +} + /******************************************************************************/ // clang doesn't support views::join yet :( std::string join_(auto begin, auto end) @@ -78,37 +86,10 @@ Group::Field::Field(const std::string_view& column_name, : rel_index_(rel_index) , abs_index_(abs_index) , schema_name_(column_name) - , clean_name_(column_name) , cv_type_() , cv_unit_() , type_() { - using std::operator""sv; - - auto tokens = column_name | std::views::split("_"sv) | - std::ranges::to>(); - - if (tokens.size() < 3) { - return; - } - - auto name_begin = tokens.begin(); - auto name_end = tokens.end(); - - if (*name_begin == "MS" || *name_begin == "UO") { - cv_type_ = CVType(*name_begin, *(name_begin + 1)); - name_begin += 2; - } - - auto unit = std::ranges::find_last(name_begin, name_end, "unit"sv); - - if (unit.begin() != name_begin && unit.begin() != name_end && - std::ranges::distance(unit.begin(), name_end) == 3) { - cv_unit_ = CVUnit(*(unit.begin() + 1), *(unit.begin() + 2)); - name_end = unit.begin(); - } - - clean_name_ = join_(name_begin, name_end); } /******************************************************************************/ @@ -118,10 +99,7 @@ Group::index_type Group::Field::relative_index() const { return rel_index_; } Group::index_type Group::Field::absolute_index() const { return abs_index_; } /******************************************************************************/ -const std::string& Group::Field::name() const { return clean_name_; } - -/******************************************************************************/ -const std::string& Group::Field::schema_name() const { return schema_name_; } +const std::string& Group::Field::name() const { return schema_name_; } /******************************************************************************/ Group::Field::Kind Group::Field::kind() const { return kind_; } @@ -144,37 +122,75 @@ const std::optional& Group::Field::type() const { return type_; } /******************************************************************************/ void Group::Field::type(Util::Type type) { type_ = type; } +/******************************************************************************/ +Group::Group(const parquet::schema::GroupNode& node, const Schema::File& file) + : name_("root") + , is_root_(true) + , index_(0) + , fields_() +{ + // This is the root group so only collect non-group top-level columns. + make_fields(node, file, 0); +} + /******************************************************************************/ Group::Group(const parquet::schema::GroupNode& node, + const Schema::File& file, index_type index, index_type offset) : name_(node.name()) + , is_root_(false) , index_(index) , fields_() { + make_fields(node, file, offset); +} + +/******************************************************************************/ +void Group::make_fields(const parquet::schema::GroupNode& node, + const Schema::File& file, + index_type offset) +{ + auto link = [&](std::shared_ptr& field) -> void { + fields_[field->name()] = field; + + std::string col_path = path(*field); + const auto it = std::ranges::find(file.columns(), col_path, &File::Column::path); + + if (it != file.columns().end()) { + field->cv_type_ = it->accession.and_then(&string_to_cv_child); + field->cv_unit_ = it->unit.and_then(&string_to_cv_child); + } + }; + for (index_type i : std::views::iota(0, node.field_count())) { auto child = node.field(i); - std::shared_ptr field = - std::make_shared(child->name(), i, offset + i); - if (child->is_primitive()) { + std::shared_ptr field = + std::make_shared(child->name(), i, offset + i); + link(field); + auto prim = std::static_pointer_cast(child); field->kind_ = Field::Kind::Scalar; field->type_ = Util::type_from_parquet(*prim); - } else { + } else if (child->is_group()) { auto grp = std::static_pointer_cast(child); - if (field->name() == "parameters" && grp->logical_type()->is_list()) { - field->kind_ = Group::Field::Kind::Params; - } else { + if (grp->field_count() == 1) { + std::shared_ptr field = + std::make_shared(grp->name(), i, offset + i); + link(field); + auto grp_type = field_type_from_parquet(grp); field->kind_ = grp_type.first; field->type_ = grp_type.second; + + if (field->name() == "parameters") { + field->kind_ = Group::Field::Kind::Params; + } } } - - fields_[field->name()] = field; } } @@ -218,4 +234,17 @@ Group::field(const CVType&& cvt) const /******************************************************************************/ const Group::field_map_t& Group::fields() const { return fields_; } +/******************************************************************************/ +bool Group::is_root() const { return is_root_; } + +/******************************************************************************/ +std::string Group::path(const Field& field) const +{ + if (is_root_) { + return field.name(); + } else { + return name() + "." + field.name(); + } +} + } // namespace MzPeak::Schema diff --git a/src/spectra.cpp b/src/spectra.cpp index db8603f..f784038 100644 --- a/src/spectra.cpp +++ b/src/spectra.cpp @@ -9,7 +9,6 @@ top-level directory of this repository. #include #include "mzpeak/data/signals.h" -#include "mzpeak/metadata/table.h" #include "mzpeak/spectra.h" #include "mzpeak/spectrum.h" #include "mzpeak/util/enumerable_proxy.h" @@ -18,11 +17,11 @@ namespace MzPeak { /******************************************************************************/ Spectra::Spectra(std::unique_ptr data, - std::unique_ptr meta) + std::shared_ptr manager) : EnumerableProxy( 0, std::bind(std::mem_fn(&Spectra::fetch), this, std::placeholders::_1)) , data_(std::move(data)) - , meta_(std::move(meta)) + , manager_(std::move(manager)) { // Update the record count. resize(data_->record_count()); @@ -40,7 +39,7 @@ Spectrum Spectra::fetch(uint64_t index) std::ranges::to>(); std::unique_ptr slice = data_->select(dims, data_->index().eq(index)); - return Spectrum(index, data_, dims, std::move(slice), meta_); + return Spectrum(index, manager_, data_, dims, std::move(slice)); } } // namespace MzPeak diff --git a/src/spectrum.cpp b/src/spectrum.cpp index 0d679bb..1176ffc 100644 --- a/src/spectrum.cpp +++ b/src/spectrum.cpp @@ -9,30 +9,47 @@ top-level directory of this repository. #include #include +#include "mzpeak/data/encoding.h" +#include "mzpeak/metadata/spectrum.h" #include "mzpeak/spectrum.h" +#include "mzpeak/util/manager.h" namespace MzPeak { +/******************************************************************************/ +const static char* PRIMARY_METADATA_FILE = "spectra_metadata.parquet"; + /******************************************************************************/ Spectrum::Spectrum(uint64_t index, + std::shared_ptr manager, std::shared_ptr data, const std::vector& dims, - std::unique_ptr slice, - std::shared_ptr metadata) + std::unique_ptr slice) : index_(index) - , md_table_(std::move(metadata)) - , md_spec_(md_table_, index_) - , decoder_(std::move(data), - std::move(slice), - Util::DeltaEstimator(md_spec_.delta_model())) + , manager_(std::move(manager)) , mz_() , intensity_() + , ms_level_(0) { + auto meta_it = manager_->find_file(PRIMARY_METADATA_FILE); + + if (meta_it == manager_->files().end()) { + throw ParquetError("missing necessary mzpeak file: " + + std::string(PRIMARY_METADATA_FILE)); + } + + Metadata::Spectrum meta(manager_->parquet(*meta_it), index_); + ms_level_ = meta.ms_level().value_or(0); + + Data::Encoding::Decoder decoder( + std::move(data), std::move(slice), + Util::DeltaEstimator(meta.delta_model())); + for (auto& dim : dims) { if (dim.array_type == Schema::PSI::ArrayType::Mz) { - decoder_.decimal(dim, mz_); + decoder.decimal(dim, mz_); } else if (dim.array_type == Schema::PSI::ArrayType::Intensity) { - decoder_.decimal(dim, intensity_); + decoder.decimal(dim, intensity_); } else { // FIXME: should we throw an exception here? continue; @@ -47,6 +64,6 @@ const std::vector& Spectrum::mz() const { return mz_; } const std::vector& Spectrum::intensity() const { return intensity_; } /******************************************************************************/ -uint8_t Spectrum::ms_level() const { return md_spec_.ms_level().value_or(0u); } +uint8_t Spectrum::ms_level() const { return ms_level_; } } // namespace MzPeak diff --git a/src/util/executor.cpp b/src/util/executor.cpp index d4344f5..7c3298b 100644 --- a/src/util/executor.cpp +++ b/src/util/executor.cpp @@ -119,14 +119,17 @@ std::shared_ptr Executor::Impl::array(std::shared_ptr& batch, const Schema::Column& field) { - std::shared_ptr col(batch->column(field.first->index())); - - if (col && col->type_id() == arrow::Type::STRUCT) { - auto sa = std::static_pointer_cast(col); - return sa->field(field.second->relative_index()); + if (field.first->is_root()) { + return batch->column(field.second->absolute_index()); } else { - throw ParquetError("column not in batch: " + field.first->name() + "." + - field.second->name()); + std::shared_ptr col(batch->column(field.first->index())); + + if (col && col->type_id() == arrow::Type::STRUCT) { + auto sa = std::static_pointer_cast(col); + return sa->field(field.second->relative_index()); + } else { + throw ParquetError("column not in batch: " + field.first->path(*field.second)); + } } } diff --git a/src/util/manager.cpp b/src/util/manager.cpp new file mode 100644 index 0000000..77c00b7 --- /dev/null +++ b/src/util/manager.cpp @@ -0,0 +1,81 @@ +/* + +This file is part of the mzpeak project. It is subject to the license +specified in the LICENSE file which can be found in the top-level +directory of this repository. + +*/ + +#include "mzpeak/util/manager.h" + +namespace MzPeak::Util { + +/******************************************************************************/ +const static char* INDEX_FILE_NAME = "mzpeak_index.json"; + +/******************************************************************************/ +namespace json = boost::json; + +/******************************************************************************/ +void parse_index(std::shared_ptr& archive, + std::vector& files) +{ + auto file = archive->read_file(INDEX_FILE_NAME); + uint8_t buffer[64 * 1024]; + std::optional bytes; + + json::stream_parser parser; + boost::system::error_code ec; + + do { + bytes = file->read(buffer, sizeof(buffer)); + + if (bytes.has_value() && *bytes > 0) { + parser.write(reinterpret_cast(buffer), *bytes, ec); + } + } while (bytes.has_value() && !ec); + + if (!ec) parser.finish(ec); + if (ec) throw MzPeak::JsonError(ec.message()); + + json::value v = parser.release(); + json::object o = v.as_object(); + + if (const auto it = o.find("files"); it != o.end() && it->value().is_array()) { + const json::array file_list(it->value().as_array()); + files.reserve(file_list.size()); + + for (const auto& file_obj : file_list) { + if (file_obj.is_object()) { + files.push_back(Schema::File(file_obj.as_object())); + } + } + } +} + +/******************************************************************************/ +Manager::Manager(std::unique_ptr archive) + : archive_(std::move(archive)) + , files_() +{ + parse_index(archive_, files_); +} + +/******************************************************************************/ +const std::vector& Manager::files() const { return files_; } + +/******************************************************************************/ +std::vector::const_iterator +Manager::find_file(const std::string_view& name) const +{ + return std::ranges::find(files_, name, &Schema::File::file_name); +} + +/******************************************************************************/ +std::unique_ptr Manager::parquet(const Schema::File& file) const +{ + std::unique_ptr data(archive_->read_file(file.file_name())); + return std::make_unique(std::move(data), file); +} + +} // namespace MzPeak::Util diff --git a/src/util/parquet.cpp b/src/util/parquet.cpp index 8f5c44b..fab820d 100644 --- a/src/util/parquet.cpp +++ b/src/util/parquet.cpp @@ -66,7 +66,7 @@ struct Parquet::Impl { auto status = reader_builder.Open(std::move(raf)); if (!status.ok()) { - std::string msg("while opening file: " + file.file_name + ": "); + std::string msg("while opening file: " + file_.file_name() + ": "); throw ParquetError(msg + status.ToString()); } @@ -74,7 +74,7 @@ struct Parquet::Impl { status = reader_builder.Build(&reader); if (!status.ok()) { - std::string msg("while reading file: " + file.file_name + ": "); + std::string msg("while reading file: " + file_.file_name() + ": "); throw ParquetError(msg + status.ToString()); } @@ -86,7 +86,7 @@ struct Parquet::Impl { void error(const std::string& error) { - std::string msg("file accessing " + file_.file_name + ": " + error); + std::string msg("file accessing " + file_.file_name() + ": " + error); throw ParquetError(msg); } @@ -106,17 +106,22 @@ void Parquet::Impl::parse_schema() auto root = fmd->schema()->group_node(); int32_t offset = 0; + std::shared_ptr root_group = + std::make_shared(*root, file_); + + if (!root_group->fields().empty()) { + (*groups_)[root_group->name()] = root_group; + } + for (int32_t i : std::views::iota(0, root->field_count())) { auto node = root->field(i); - // TODO: Should we emit a warning if there is a top-level - // primitive column? - if (node->is_group()) { + if (node->is_group() && !node->logical_type()->is_list()) { std::shared_ptr group = std::static_pointer_cast(node); std::shared_ptr s = - std::make_shared(*group, i, offset); + std::make_shared(*group, file_, i, offset); (*groups_)[s->name()] = s; offset += group->field_count(); } diff --git a/src/util/query.cpp b/src/util/query.cpp index a1d023d..e3fa539 100644 --- a/src/util/query.cpp +++ b/src/util/query.cpp @@ -62,7 +62,7 @@ Query Query::Builder::validate(Query::Predicate&& p) const { if (!p.dest.second->type().has_value()) { std::string msg("cannot query field with unknown type: "); - msg += p.dest.first->name() + "." + p.dest.second->name(); + msg += p.dest.first->path(*p.dest.second); throw TypeError(msg); } @@ -239,7 +239,7 @@ Trampoline> EvalHelper::eval(const Query& query, if constexpr (std::is_same_v) { if (!tree.dest.second->type().has_value()) { std::string msg("invalid query on field with unknown data type: "); - msg += tree.dest.first->name() + "." + tree.dest.second->name(); + msg += tree.dest.first->path(*tree.dest.second); throw TypeError(msg); } diff --git a/test/array_index_test.cpp b/test/array_index_test.cpp index 3589bf4..1fda385 100644 --- a/test/array_index_test.cpp +++ b/test/array_index_test.cpp @@ -11,6 +11,7 @@ directory of this repository. #include "mzpeak/data/signals.h" #include "mzpeak/open.h" +#include "mzpeak/util/manager.h" /******************************************************************************/ BOOST_AUTO_TEST_CASE(can_get_array_index) @@ -24,31 +25,39 @@ BOOST_AUTO_TEST_CASE(can_get_array_index) BOOST_TEST((entry != mzpeak.files().end())); - auto parquet = mzpeak.parquet(*entry); + auto parquet = mzpeak.manager()->parquet(*entry); Data::Signals data(std::move(parquet)); std::shared_ptr index(data.array_index()); BOOST_TEST(index->prefix() == "point"); BOOST_TEST(index->entries().size() == 2ul); - std::optional transform_cv(Schema::CV::from_string("MS:1003901")); - BOOST_TEST((transform_cv.has_value())); - std::optional transform(transform_cv.value()); - - // NOTE: Due to a sort after parsing the index, the m/z array gets - // moved to the end of the index. - BOOST_TEST((index->entries()[1].array_name == "m/z array")); - BOOST_TEST((index->entries()[1].buffer_format == Schema::BufferFormat::Point)); - BOOST_TEST((index->entries()[1].context == Schema::EntityType::Spectrum)); - BOOST_TEST((index->entries()[1].path == "point.mz")); - BOOST_TEST((index->entries()[1].data_type == - Schema::PSI::DataType(Schema::PSI::DataType::Float64))); - BOOST_TEST((index->entries()[1].array_type == Schema::PSI::ArrayType::Mz)); - BOOST_TEST((index->entries()[1].unit == "MS:1000040")); - BOOST_TEST((index->entries()[1].buffer_priority)); - BOOST_TEST((index->entries()[1].sorting_rank == std::optional{0})); - BOOST_TEST((index->entries()[1].data_processing_id == std::nullopt)); - BOOST_TEST((index->entries()[1].transform == transform)); + const auto mz_it = std::ranges::find(index->entries(), "m/z array", + &Data::ArrayIndex::Entry::array_name); + BOOST_TEST((mz_it != index->entries().end())); + + const std::optional interpolation( + Schema::PSI::Transform::ZeroIntensityInterpolation); + const Schema::PSI::DataType f64(Schema::PSI::DataType::Float64); + + BOOST_TEST((mz_it->buffer_format == Schema::BufferFormat::Point)); + BOOST_TEST((mz_it->context == Schema::EntityType::Spectrum)); + BOOST_TEST((mz_it->path == "point.mz")); + BOOST_TEST((mz_it->data_type == f64)); + BOOST_TEST((mz_it->array_type == Schema::PSI::ArrayType::Mz)); + BOOST_TEST((mz_it->unit == "MS:1000040")); + BOOST_TEST((mz_it->buffer_priority)); + BOOST_TEST((mz_it->sorting_rank == std::optional{0})); + BOOST_TEST((mz_it->data_processing_id == std::nullopt)); + BOOST_TEST((mz_it->transform == interpolation)); + + const auto intensity_it = std::ranges::find(index->entries(), "intensity array", + &Data::ArrayIndex::Entry::array_name); + BOOST_TEST((intensity_it != index->entries().end())); + + const std::optional trim( + Schema::PSI::Transform::ZeroIntensityTrim); + BOOST_TEST((intensity_it->transform == trim)); std::size_t count = index->num_entities().value_or(0); BOOST_TEST(count == 48ul); @@ -65,7 +74,7 @@ BOOST_AUTO_TEST_CASE(can_read_mz_array) BOOST_TEST((entry != mzpeak.files().end())); - auto parquet = mzpeak.parquet(*entry); + auto parquet = mzpeak.manager()->parquet(*entry); Data::Signals data(std::move(parquet)); diff --git a/test/executor_test.cpp b/test/executor_test.cpp index 248c850..b32d80b 100644 --- a/test/executor_test.cpp +++ b/test/executor_test.cpp @@ -11,6 +11,7 @@ in the LICENSE file found in the top-level directory of this project. #include "mzpeak/open.h" #include "mzpeak/util/executor.h" +#include "mzpeak/util/manager.h" // IWYU pragma: keep #include "mzpeak/util/parquet.h" #include "mzpeak/util/planner.h" #include "mzpeak/util/query.h" @@ -26,7 +27,7 @@ BOOST_AUTO_TEST_CASE(can_find_spectrum) BOOST_TEST((entry != index.files().end())); - auto parquet = index.parquet(*entry); + auto parquet = index.manager()->parquet(*entry); auto index_field = parquet->field("point", "spectrum_index"); BOOST_TEST(index_field.has_value()); @@ -69,12 +70,12 @@ BOOST_AUTO_TEST_CASE(can_read_uint8_t) BOOST_TEST((entry != index.files().end())); - auto parquet = index.parquet(*entry); + auto parquet = index.manager()->parquet(*entry); - auto index_field = parquet->field("spectrum", "index"); + auto index_field = parquet->field("root", "index"); BOOST_TEST(index_field.has_value()); - auto ms_level = parquet->field("spectrum", "ms_level"); + auto ms_level = parquet->field("root", "ms_level"); BOOST_TEST(ms_level.has_value()); auto query = Util::Query::Builder(*index_field).eq(0ul); diff --git a/test/files/Example_Processed.img.mzpeak b/test/files/Example_Processed.img.mzpeak index 6033462..3b348f3 100644 Binary files a/test/files/Example_Processed.img.mzpeak and b/test/files/Example_Processed.img.mzpeak differ diff --git a/test/files/has_uv.mzpeak b/test/files/has_uv.mzpeak index eb3e531..7550dec 100644 Binary files a/test/files/has_uv.mzpeak and b/test/files/has_uv.mzpeak differ diff --git a/test/files/small.chunked.mzpeak b/test/files/small.chunked.mzpeak index 4d70e92..53b002e 100644 Binary files a/test/files/small.chunked.mzpeak and b/test/files/small.chunked.mzpeak differ diff --git a/test/files/small.dir/chromatograms_data.parquet b/test/files/small.dir/chromatograms_data.parquet index 1716b16..614135b 100644 Binary files a/test/files/small.dir/chromatograms_data.parquet and b/test/files/small.dir/chromatograms_data.parquet differ diff --git a/test/files/small.dir/chromatograms_metadata.parquet b/test/files/small.dir/chromatograms_metadata.parquet index ef616d4..e6ea0cb 100644 Binary files a/test/files/small.dir/chromatograms_metadata.parquet and b/test/files/small.dir/chromatograms_metadata.parquet differ diff --git a/test/files/small.dir/chromatograms_metadata_precursors.parquet b/test/files/small.dir/chromatograms_metadata_precursors.parquet new file mode 100644 index 0000000..27615fa Binary files /dev/null and b/test/files/small.dir/chromatograms_metadata_precursors.parquet differ diff --git a/test/files/small.dir/chromatograms_metadata_selected_ions.parquet b/test/files/small.dir/chromatograms_metadata_selected_ions.parquet new file mode 100644 index 0000000..f060e43 Binary files /dev/null and b/test/files/small.dir/chromatograms_metadata_selected_ions.parquet differ diff --git a/test/files/small.dir/mzpeak_index.json b/test/files/small.dir/mzpeak_index.json index 6663228..c2959fd 100644 --- a/test/files/small.dir/mzpeak_index.json +++ b/test/files/small.dir/mzpeak_index.json @@ -3,31 +3,400 @@ { "name": "spectra_data.parquet", "entity_type": "spectrum", - "data_kind": "data arrays" + "data_kind": "data_arrays", + "column_mapping": [], + "parameters": [] }, { "name": "spectra_peaks.parquet", "entity_type": "spectrum", - "data_kind": "peaks" + "data_kind": "peaks", + "column_mapping": [], + "parameters": [] }, { "name": "spectra_metadata.parquet", "entity_type": "spectrum", - "data_kind": "metadata" + "data_kind": "metadata", + "column_mapping": [ + { + "name": "ms level", + "path": "ms_level", + "accession": "MS:1000511", + "unit": null + }, + { + "name": "scan polarity", + "path": "scan_polarity", + "accession": "MS:1000465", + "unit": null + }, + { + "name": "spectrum representation", + "path": "spectrum_representation", + "accession": "MS:1000525", + "unit": null + }, + { + "name": "spectrum type", + "path": "spectrum_type", + "accession": "MS:1000559", + "unit": null + }, + { + "name": "lowest observed m/z", + "path": "lowest_observed_mz", + "accession": "MS:1000528", + "unit": "MS:1000040" + }, + { + "name": "highest observed m/z", + "path": "highest_observed_mz", + "accession": "MS:1000527", + "unit": "MS:1000040" + }, + { + "name": "number of data points", + "path": "number_of_data_points", + "accession": "MS:1003060", + "unit": null + }, + { + "name": "number of peaks", + "path": "number_of_peaks", + "accession": "MS:1003059", + "unit": null + }, + { + "name": "base peak m/z", + "path": "base_peak_mz", + "accession": "MS:1000504", + "unit": "MS:1000040" + }, + { + "name": "base peak intensity", + "path": "base_peak_intensity", + "accession": "MS:1000505", + "unit": "MS:1000131" + }, + { + "name": "total ion current", + "path": "total_ion_current", + "accession": "MS:1000285", + "unit": "MS:1000131" + } + ], + "parameters": [] + }, + { + "name": "spectra_metadata_scans.parquet", + "entity_type": "spectrum", + "data_kind": "scans", + "column_mapping": [ + { + "name": "scan start time", + "path": "scan_start_time", + "accession": "MS:1000016", + "unit": "UO:0000031" + }, + { + "name": "preset scan configuration", + "path": "preset_scan_configuration", + "accession": "MS:1000616", + "unit": null + }, + { + "name": "filter string", + "path": "filter_string", + "accession": "MS:1000512", + "unit": null + }, + { + "name": "ion injection time", + "path": "ion_injection_time", + "accession": "MS:1000927", + "unit": "UO:0000028" + }, + { + "name": "scan window lower limit", + "path": "scan_windows.scan_window_lower_limit", + "accession": "MS:1000501", + "unit": "MS:1000040" + }, + { + "name": "scan window upper limit", + "path": "scan_windows.scan_window_upper_limit", + "accession": "MS:1000500", + "unit": "MS:1000040" + } + ], + "parameters": [] + }, + { + "name": "spectra_metadata_precursors.parquet", + "entity_type": "spectrum", + "data_kind": "precursors", + "column_mapping": [ + { + "name": "isolation window target m/z", + "path": "isolation_window.isolation_window_target", + "accession": "MS:1000827", + "unit": "MS:1000040" + }, + { + "name": "isolation window lower offset", + "path": "isolation_window.isolation_window_lower_offset", + "accession": "MS:1000828", + "unit": "MS:1000040" + }, + { + "name": "isolation window upper offset", + "path": "isolation_window.isolation_window_upper_offset", + "accession": "MS:1000829", + "unit": "MS:1000040" + } + ], + "parameters": [] + }, + { + "name": "spectra_metadata_selected_ions.parquet", + "entity_type": "spectrum", + "data_kind": "selected_ions", + "column_mapping": [ + { + "name": "selected ion m/z", + "path": "selected_ion_mz", + "accession": "MS:1000744", + "unit": "MS:1000040" + }, + { + "name": "charge state", + "path": "charge_state", + "accession": "MS:1000041", + "unit": null + }, + { + "name": "intensity", + "path": "peak_intensity", + "accession": "MS:1000042", + "unit": "MS:1000131" + } + ], + "parameters": [] }, { "name": "chromatograms_metadata.parquet", "entity_type": "chromatogram", - "data_kind": "metadata" + "data_kind": "metadata", + "column_mapping": [ + { + "name": "scan polarity", + "path": "scan_polarity", + "accession": "MS:1000465", + "unit": null + }, + { + "name": "chromatogram type", + "path": "chromatogram_type", + "accession": "MS:1000626", + "unit": null + }, + { + "name": "number of data points", + "path": "number_of_data_points", + "accession": "MS:1003060", + "unit": null + } + ], + "parameters": [] + }, + { + "name": "chromatograms_metadata_precursors.parquet", + "entity_type": "chromatogram", + "data_kind": "precursors", + "column_mapping": [ + { + "name": "isolation window target m/z", + "path": "isolation_window.isolation_window_target", + "accession": "MS:1000827", + "unit": "MS:1000040" + }, + { + "name": "isolation window lower offset", + "path": "isolation_window.isolation_window_lower_offset", + "accession": "MS:1000828", + "unit": "MS:1000040" + }, + { + "name": "isolation window upper offset", + "path": "isolation_window.isolation_window_upper_offset", + "accession": "MS:1000829", + "unit": "MS:1000040" + } + ], + "parameters": [] + }, + { + "name": "chromatograms_metadata_selected_ions.parquet", + "entity_type": "chromatogram", + "data_kind": "selected_ions", + "column_mapping": [ + { + "name": "selected ion m/z", + "path": "selected_ion_mz", + "accession": "MS:1000744", + "unit": "MS:1000040" + }, + { + "name": "charge state", + "path": "charge_state", + "accession": "MS:1000041", + "unit": null + }, + { + "name": "intensity", + "path": "peak_intensity", + "accession": "MS:1000042", + "unit": "MS:1000131" + } + ], + "parameters": [] }, { "name": "chromatograms_data.parquet", "entity_type": "chromatogram", - "data_kind": "data arrays" + "data_kind": "data_arrays", + "column_mapping": [], + "parameters": [] } ], "metadata": { "scan_settings_list": [], + "version": "0.9.0", + "data_processing_method_list": [ + { + "id": "pwiz_Reader_Thermo_conversion", + "methods": [ + { + "order": 0, + "parameters": [ + { + "accession": "MS:1000544", + "name": "Conversion to mzML", + "unit": null, + "value": null + } + ], + "software_reference": "pwiz" + } + ] + }, + { + "id": "mzpeak_conversion1", + "methods": [ + { + "order": 1, + "parameters": [ + { + "accession": null, + "name": "conversion options", + "unit": null, + "value": "-y -z -u small.mzML -o small.mzpeak" + } + ], + "software_reference": "mzpeak_prototyping_convert1" + } + ] + } + ], + "file_description": { + "contents": [ + { + "accession": "MS:1000579", + "name": "MS1 spectrum", + "unit": null, + "value": null + }, + { + "accession": "MS:1000580", + "name": "MSn spectrum", + "unit": null, + "value": null + } + ], + "source_files": [ + { + "id": "RAW1", + "location": "file:///C:\\Users\\Joshua\\Dev\\learn-rust\\ffirawfilereader\\tests\\data", + "name": "small.RAW", + "parameters": [ + { + "accession": "MS:1000569", + "name": "SHA-1", + "unit": null, + "value": "b43e9286b40e8b5dbc0dfa2e428495769ca96a96" + }, + { + "accession": "MS:1000563", + "name": "Thermo RAW format", + "unit": null, + "value": null + }, + { + "accession": "MS:1000768", + "name": "Thermo nativeID format", + "unit": null, + "value": null + } + ] + } + ] + }, + "software_list": [ + { + "id": "Xcalibur", + "parameters": [ + { + "accession": "MS:1000532", + "name": "Xcalibur", + "unit": null, + "value": null + } + ], + "version": "1.1 Beta 7" + }, + { + "id": "pwiz", + "parameters": [ + { + "accession": "MS:1000615", + "name": "ProteoWizard software", + "unit": null, + "value": null + } + ], + "version": "3.0.23307" + }, + { + "id": "mzpeak_prototyping_convert1", + "parameters": [ + { + "accession": "MS:1000799", + "name": "custom unreleased software tool", + "unit": null, + "value": "mzpeak_prototyping_convert" + } + ], + "version": "0.1.0" + } + ], + "run": { + "default_data_processing_id": "pwiz_Reader_Thermo_conversion", + "default_instrument_id": 0, + "default_source_file_id": "RAW1", + "id": "small", + "start_time": "2005-07-20T19:44:22Z" + }, "instrument_configuration_list": [ { "components": [ @@ -54,8 +423,8 @@ "order": 2, "parameters": [ { - "accession": "MS:1000079", - "name": "fourier transform ion cyclotron resonance mass spectrometer", + "accession": "MS:1000083", + "name": "radial ejection linear ion trap", "unit": null, "value": null } @@ -66,15 +435,15 @@ "order": 3, "parameters": [ { - "accession": "MS:1000624", - "name": "inductive detector", + "accession": "MS:1000253", + "name": "electron multiplier", "unit": null, "value": null } ] } ], - "id": 0, + "id": 1, "parameters": [ { "accession": "MS:1000448", @@ -116,8 +485,8 @@ "order": 2, "parameters": [ { - "accession": "MS:1000083", - "name": "radial ejection linear ion trap", + "accession": "MS:1000079", + "name": "fourier transform ion cyclotron resonance mass spectrometer", "unit": null, "value": null } @@ -128,15 +497,15 @@ "order": 3, "parameters": [ { - "accession": "MS:1000253", - "name": "electron multiplier", + "accession": "MS:1000624", + "name": "inductive detector", "unit": null, "value": null } ] } ], - "id": 1, + "id": 0, "parameters": [ { "accession": "MS:1000448", @@ -154,92 +523,20 @@ "software_reference": "Xcalibur" } ], - "run": { - "default_data_processing_id": "pwiz_Reader_Thermo_conversion", - "default_instrument_id": 0, - "default_source_file_id": "RAW1", - "id": "small", - "start_time": "2005-07-20T19:44:22Z" - }, - "data_processing_method_list": [ + "cv_list": [ { - "id": "pwiz_Reader_Thermo_conversion", - "methods": [ - { - "order": 0, - "parameters": [ - { - "accession": "MS:1000544", - "name": "Conversion to mzML", - "unit": null, - "value": null - } - ], - "software_reference": "pwiz" - } - ] + "full_name": "Proteomics Standards Initiative Mass Spectrometry Ontology", + "id": "MS", + "uri": "http://purl.obolibrary.org/obo/ms/4.1.249/psi-ms.obo", + "version": "4.1.249" }, { - "id": "mzpeak_conversion1", - "methods": [ - { - "order": 1, - "parameters": [ - { - "accession": null, - "name": "conversion options", - "unit": null, - "value": "-y -z -u small.mzML -o small.mzpeak" - } - ], - "software_reference": "mzpeak_prototyping_convert1" - } - ] + "full_name": "Units of measurement ontology", + "id": "UO", + "uri": "http://purl.obolibrary.org/obo/uo/releases/2026-01-16/uo.obo", + "version": "2026-01-16" } ], - "file_description": { - "contents": [ - { - "accession": "MS:1000579", - "name": "MS1 spectrum", - "unit": null, - "value": null - }, - { - "accession": "MS:1000580", - "name": "MSn spectrum", - "unit": null, - "value": null - } - ], - "source_files": [ - { - "id": "RAW1", - "location": "file:///C:\\Users\\Joshua\\Dev\\learn-rust\\ffirawfilereader\\tests\\data", - "name": "small.RAW", - "parameters": [ - { - "accession": "MS:1000569", - "name": "SHA-1", - "unit": null, - "value": "b43e9286b40e8b5dbc0dfa2e428495769ca96a96" - }, - { - "accession": "MS:1000563", - "name": "Thermo RAW format", - "unit": null, - "value": null - }, - { - "accession": "MS:1000768", - "name": "Thermo nativeID format", - "unit": null, - "value": null - } - ] - } - ] - }, "sample_list": [ { "id": "_x0031_", @@ -253,44 +550,6 @@ } ] } - ], - "software_list": [ - { - "id": "Xcalibur", - "parameters": [ - { - "accession": "MS:1000532", - "name": "Xcalibur", - "unit": null, - "value": null - } - ], - "version": "1.1 Beta 7" - }, - { - "id": "pwiz", - "parameters": [ - { - "accession": "MS:1000615", - "name": "ProteoWizard software", - "unit": null, - "value": null - } - ], - "version": "3.0.23307" - }, - { - "id": "mzpeak_prototyping_convert1", - "parameters": [ - { - "accession": "MS:1000799", - "name": "custom unreleased software tool", - "unit": null, - "value": "mzpeak_prototyping_convert" - } - ], - "version": "0.1.0" - } ] } } \ No newline at end of file diff --git a/test/files/small.dir/spectra_data.parquet b/test/files/small.dir/spectra_data.parquet index e297aa5..8d0f750 100644 Binary files a/test/files/small.dir/spectra_data.parquet and b/test/files/small.dir/spectra_data.parquet differ diff --git a/test/files/small.dir/spectra_metadata.parquet b/test/files/small.dir/spectra_metadata.parquet index 8ce8000..c4a0289 100644 Binary files a/test/files/small.dir/spectra_metadata.parquet and b/test/files/small.dir/spectra_metadata.parquet differ diff --git a/test/files/small.dir/spectra_metadata_precursors.parquet b/test/files/small.dir/spectra_metadata_precursors.parquet new file mode 100644 index 0000000..87e6461 Binary files /dev/null and b/test/files/small.dir/spectra_metadata_precursors.parquet differ diff --git a/test/files/small.dir/spectra_metadata_scans.parquet b/test/files/small.dir/spectra_metadata_scans.parquet new file mode 100644 index 0000000..7a410f5 Binary files /dev/null and b/test/files/small.dir/spectra_metadata_scans.parquet differ diff --git a/test/files/small.dir/spectra_metadata_selected_ions.parquet b/test/files/small.dir/spectra_metadata_selected_ions.parquet new file mode 100644 index 0000000..1c6d8a0 Binary files /dev/null and b/test/files/small.dir/spectra_metadata_selected_ions.parquet differ diff --git a/test/files/small.mzpeak b/test/files/small.mzpeak index 859d5b8..23363b7 100644 Binary files a/test/files/small.mzpeak and b/test/files/small.mzpeak differ diff --git a/test/files/small.numpress.mzpeak b/test/files/small.numpress.mzpeak index 6cdfcdd..68119c4 100644 Binary files a/test/files/small.numpress.mzpeak and b/test/files/small.numpress.mzpeak differ diff --git a/test/group_test.cpp b/test/group_test.cpp index bc7752b..efeaf12 100644 --- a/test/group_test.cpp +++ b/test/group_test.cpp @@ -10,60 +10,24 @@ top-level directory of this repository. #include #include "mzpeak/open.h" -#include "mzpeak/schema/group.h" +#include "mzpeak/util/manager.h" // IWYU pragma: keep #include "mzpeak/util/parquet.h" -/******************************************************************************/ -BOOST_AUTO_TEST_CASE(can_parse_column_names) -{ - using namespace MzPeak::Schema; - - // All components. - Group::Field a("MS_1000528_lowest_observed_mz_unit_MS_1000040", 0, 2); - BOOST_TEST(a.relative_index() == 0); - BOOST_TEST(a.absolute_index() == 2); - BOOST_TEST(a.name() == "lowest_observed_mz"); - - BOOST_TEST(a.cv_type().has_value()); - BOOST_TEST((a.cv_type()->code() == "MS")); - BOOST_TEST((a.cv_type()->accession() == "1000528")); - BOOST_TEST((a.cv_type()->to_string() == "MS:1000528")); - - BOOST_TEST((a.cv_unit().has_value())); - BOOST_TEST((a.cv_unit().has_value() && a.cv_unit()->code() == "MS")); - BOOST_TEST((a.cv_unit()->accession() == "1000040")); - - // No unit. - Group::Field b("MS_1000016_scan_start_time", 0, 0); - BOOST_TEST(b.name() == "scan_start_time"); - BOOST_TEST((b.cv_type().has_value() && b.cv_type()->code() == "MS")); - BOOST_TEST((b.cv_type().has_value() && b.cv_type()->accession() == "1000016")); - BOOST_TEST((!b.cv_unit().has_value())); - - // Nmae only. - Group::Field c("mz", 0, 0); - BOOST_TEST(c.name() == "mz"); - BOOST_TEST(!c.cv_type().has_value()); - BOOST_TEST(!c.cv_unit().has_value()); -} - /******************************************************************************/ BOOST_AUTO_TEST_CASE(can_load_all_groups) { using namespace MzPeak::Util; auto mzpeak = MzPeak::open("../test/files/small.mzpeak"); - auto entry = std::ranges::find(mzpeak.files(), "spectra_metadata.parquet", - &MzPeak::Schema::File::file_name); - + auto entry = mzpeak.find("spectra_metadata.parquet"); BOOST_TEST((entry != mzpeak.files().end())); - auto parquet = mzpeak.parquet(*entry); + auto parquet = mzpeak.manager()->parquet(*entry); auto groups = parquet->groups(); - BOOST_TEST((groups->size() == 4)); + BOOST_TEST((groups->size() == 1)); - auto spectrum_index = parquet->field("spectrum", "index"); + auto spectrum_index = parquet->field("root", "index"); BOOST_TEST(spectrum_index.has_value()); BOOST_TEST((spectrum_index->second->type().has_value())); BOOST_TEST((spectrum_index->second->type().value() == MzPeak::Util::Type::UInt64)); diff --git a/test/index_test.cpp b/test/index_test.cpp index eaade54..3d55823 100644 --- a/test/index_test.cpp +++ b/test/index_test.cpp @@ -13,6 +13,7 @@ directory of this repository. #include "mzpeak/index.h" #include "mzpeak/open.h" +#include "mzpeak/schema/file.h" /******************************************************************************/ BOOST_AUTO_TEST_CASE(can_parse_json) @@ -26,11 +27,9 @@ BOOST_AUTO_TEST_CASE(can_parse_json) BOOST_AUTO_TEST_CASE(is_associated_with) { auto index = MzPeak::open("../test/files/small.mzpeak"); - const auto& files = index.files(); - - const auto& spectra = std::ranges::find(files, "spectra_data.parquet", - &MzPeak::Schema::File::file_name); + const auto& files = index.files(); + const auto& spectra = index.find("spectra_data.parquet"); BOOST_TEST((spectra != files.end()), "missing spectra_data.parquet"); auto matches = [&](const auto& other) -> bool { @@ -38,10 +37,10 @@ BOOST_AUTO_TEST_CASE(is_associated_with) }; for (const auto& other : files | std::views::filter(matches)) { - BOOST_TEST_CONTEXT(spectra->file_name << " should not be associated with " - << other.file_name) + BOOST_TEST_CONTEXT(spectra->file_name() + << " should not be associated with " << other.file_name()) { - BOOST_TEST((other.file_name == "spectra_metadata.parquet")); + BOOST_TEST((other.file_name() == "spectra_metadata.parquet")); } } } diff --git a/test/parquet_test.cpp b/test/parquet_test.cpp index 0d86171..6ad40c3 100644 --- a/test/parquet_test.cpp +++ b/test/parquet_test.cpp @@ -10,6 +10,7 @@ directory of this repository. #include #include "mzpeak/open.h" +#include "mzpeak/util/manager.h" // IWYU pragma: keep #include "mzpeak/util/parquet.h" /******************************************************************************/ @@ -24,9 +25,9 @@ BOOST_AUTO_TEST_CASE(can_get_kv_string) BOOST_TEST((entry != mzpeak.files().end())); - auto parquet = mzpeak.parquet(*entry); + auto parquet = mzpeak.manager()->parquet(*entry); auto fmd = parquet->file_metadata(); - auto et = parquet->index_file().entity_type; + auto et = parquet->index_file().entity_type(); auto key = MzPeak::Schema::entity_type_to_string(et) + "_array_index"; auto json = parquet->kv_string(fmd, key); diff --git a/test/planner_test.cpp b/test/planner_test.cpp index 0d1a412..477b70c 100644 --- a/test/planner_test.cpp +++ b/test/planner_test.cpp @@ -9,6 +9,7 @@ in the LICENSE file found in the top-level directory of this project. #include #include "mzpeak/open.h" +#include "mzpeak/util/manager.h" // IWYU pragma: keep #include "mzpeak/util/parquet.h" #include "mzpeak/util/planner.h" @@ -23,7 +24,7 @@ BOOST_AUTO_TEST_CASE(can_locate_correct_rows) BOOST_TEST((entry != index.files().end())); - auto parquet = index.parquet(*entry); + auto parquet = index.manager()->parquet(*entry); auto index_field = parquet->field("point", "spectrum_index"); BOOST_TEST(index_field.has_value()); @@ -56,57 +57,22 @@ BOOST_AUTO_TEST_CASE(can_use_two_columns) using namespace MzPeak; auto index = MzPeak::open("../test/files/small.mzpeak"); - auto entry = std::ranges::find(index.files(), "spectra_metadata.parquet", + auto entry = std::ranges::find(index.files(), "spectra_metadata_scans.parquet", &Schema::File::file_name); BOOST_TEST((entry != index.files().end())); - auto parquet = index.parquet(*entry); + auto parquet = index.manager()->parquet(*entry); - auto index_field = parquet->field("scan", "source_index"); + auto index_field = parquet->field("root", "source_index"); BOOST_TEST(index_field.has_value()); - auto start_time_field = parquet->field("scan", "scan_start_time"); + auto start_time_field = parquet->field("root", "scan_start_time"); BOOST_TEST(start_time_field.has_value()); auto query = Util::Query::Builder(*index_field) .eq(3) - .and_then(Util::Query::Builder(*start_time_field).gt(0)); - - Util::Planner planner = parquet->planner(query); - - auto plan = planner.plan(); - BOOST_TEST(plan.ranges.size() == 1ul); - - // Ug, this file is too small to exercise the planner. - auto first = plan.ranges[0]; - BOOST_TEST(first.row_group == 0); - BOOST_TEST(first.offset == 0); - BOOST_TEST(first.length == 48); -} - -/******************************************************************************/ -BOOST_AUTO_TEST_CASE(can_access_multiple_groups) -{ - using namespace MzPeak; - auto index = MzPeak::open("../test/files/small.mzpeak"); - - auto entry = std::ranges::find(index.files(), "spectra_metadata.parquet", - &Schema::File::file_name); - - BOOST_TEST((entry != index.files().end())); - - auto parquet = index.parquet(*entry); - - auto index_field = parquet->field("scan", "source_index"); - BOOST_TEST(index_field.has_value()); - - auto level_field = parquet->field("spectrum", "ms_level"); - BOOST_TEST(level_field.has_value()); - - auto query = Util::Query::Builder(*index_field) - .eq(30) - .and_then(Util::Query::Builder(*level_field).ge(1)); + .and_then(Util::Query::Builder(*start_time_field).gt(0.0)); Util::Planner planner = parquet->planner(query); diff --git a/test/spectra_test.cpp b/test/spectra_test.cpp index d8047d5..bd3993c 100644 --- a/test/spectra_test.cpp +++ b/test/spectra_test.cpp @@ -9,6 +9,8 @@ directory of this repository. #define BOOST_TEST_MODULE Spectra #include +#include + #include "mzpeak/open.h" #include "mzpeak/spectra.h" #include "mzpeak/spectrum.h"