diff --git a/tree/ntuple/inc/ROOT/RNTupleMerger.hxx b/tree/ntuple/inc/ROOT/RNTupleMerger.hxx index 7dee28137a618..4f02bc66ecc95 100644 --- a/tree/ntuple/inc/ROOT/RNTupleMerger.hxx +++ b/tree/ntuple/inc/ROOT/RNTupleMerger.hxx @@ -140,6 +140,12 @@ public: return ENTupleMergeActionResult::kEmit; } + // Invoked at the end of each source's merging process. + virtual ENTupleMergeActionResult OnSourceEnd(ROOT::REntry &, const RNTupleMergeActionContext &) + { + return ENTupleMergeActionResult::kDrop; + } + /// Invoked at the end of the merge process, right before committing the dataset. /// This allows appending more entries at the end of the merge. virtual ENTupleMergeActionResult BeforeCommit(ROOT::REntry &, const RNTupleMergeActionContext &) diff --git a/tree/ntuple/src/RNTupleAttrWriting.cxx b/tree/ntuple/src/RNTupleAttrWriting.cxx index 4a1711f66d13b..544c435cac638 100644 --- a/tree/ntuple/src/RNTupleAttrWriting.cxx +++ b/tree/ntuple/src/RNTupleAttrWriting.cxx @@ -17,12 +17,16 @@ static ROOT::RResult ValidateAttributeModel(const ROOT::RNTupleModel &mode { const auto &projFields = ROOT::Internal::GetProjectedFieldsOfModel(model); if (!projFields.IsEmpty()) - return R__FAIL("The Model used to create an AttributeSet cannot contain projected fields."); + return R__FAIL("The Model used to create an attribute set cannot contain projected fields."); for (const auto &field : model.GetConstFieldZero()) { if (field.GetStructure() == ROOT::ENTupleStructure::kStreamer) - return R__FAIL(std::string("The Model used to create an AttributeSet cannot contain Streamer field '") + + return R__FAIL(std::string("The Model used to create an attribute set cannot contain Streamer field '") + field.GetQualifiedFieldName() + "'"); + if (field.GetTypeName().empty()) + return R__FAIL( + std::string("Untyped records or collections are currently unsupported as attribute types (field '") + + field.GetQualifiedFieldName() + "')"); } return ROOT::RResult::Success(); } diff --git a/tree/ntuple/src/RNTupleMerger.cxx b/tree/ntuple/src/RNTupleMerger.cxx index d588bac578abf..9b235cc9f71e0 100644 --- a/tree/ntuple/src/RNTupleMerger.cxx +++ b/tree/ntuple/src/RNTupleMerger.cxx @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -414,6 +415,9 @@ struct RDescriptorsComparison { std::vector fExtraDstFields; std::vector fExtraSrcFields; std::vector fCommonFields; + // { parentId => subfields } + // This is currently only used for the _userData attribute field. + std::unordered_map> fExtraSrcSubfields; // Empty if and only if the source can be fast merged. std::string fFastMergeErrors; @@ -530,11 +534,13 @@ static void MatchColumnRepresentations(const ROOT::RNTupleDescriptor &srcDesc, c } } -static RDescriptorsComparison -FillCommonAndExtraFields(const ROOT::RNTupleDescriptor &dst, const ROOT::RNTupleDescriptor &src) +static ROOT::RResult FillCommonAndExtraFields(const ROOT::RNTupleDescriptor &dst, + const ROOT::RNTupleDescriptor &src, + bool forAttributeMerging) { RDescriptorsComparison res; + // collect common and extra dst fields for (const auto &dstField : dst.GetTopLevelFields()) { const auto srcFieldId = src.FindFieldId(dstField.GetFieldName()); if (srcFieldId != ROOT::kInvalidDescriptorId) { @@ -544,6 +550,8 @@ FillCommonAndExtraFields(const ROOT::RNTupleDescriptor &dst, const ROOT::RNTuple res.fExtraDstFields.emplace_back(&dstField); } } + + // collect extra src fields for (const auto &srcField : src.GetTopLevelFields()) { const auto dstFieldId = dst.FindFieldId(srcField.GetFieldName()); if (dstFieldId == ROOT::kInvalidDescriptorId) { @@ -551,6 +559,23 @@ FillCommonAndExtraFields(const ROOT::RNTupleDescriptor &dst, const ROOT::RNTuple } } + if (forAttributeMerging) { + using namespace ROOT::Experimental::Internal::RNTupleAttributes; + const auto userDataFieldId = src.FindFieldId(kMetaFieldNames[kMetaFieldIndex_UserData]); + if (userDataFieldId == ROOT::kInvalidDescriptorId) { + return R__FAIL("attribute user data field not found. The RNTuple is corrupted and cannot be merged."); + } + const auto &userDataField = src.GetFieldDescriptor(userDataFieldId); + for (const auto subfieldId : userDataField.GetLinkIds()) { + const auto subfieldName = src.GetQualifiedFieldName(subfieldId); + const auto dstFieldId = dst.FindFieldId(subfieldName); + if (dstFieldId == ROOT::kInvalidDescriptorId) { + const auto &subfield = src.GetFieldDescriptor(subfieldId); + res.fExtraSrcSubfields[userDataFieldId].push_back(&subfield); + } + } + } + return res; } @@ -581,8 +606,9 @@ static void VerifyProjectedFieldsCompatibility(const ROOT::RNTupleDescriptor &ds /// Compares the top level fields of `dst` and `src` and determines whether they can be merged or not. /// In addition, returns the differences between `dst` and `src`'s structures -static ROOT::RResult -CompareDescriptorStructure(const ROOT::RNTupleDescriptor &dst, const ROOT::RNTupleDescriptor &src) +static ROOT::RResult CompareDescriptorStructure(const ROOT::RNTupleDescriptor &dst, + const ROOT::RNTupleDescriptor &src, + bool forAttributeMerging = false) { // Cases: // 1. dst == src @@ -590,7 +616,11 @@ CompareDescriptorStructure(const ROOT::RNTupleDescriptor &dst, const ROOT::RNTup // 3. src has fields that dst hasn't // 4. dst and src have fields that differ (compatible or incompatible) - RDescriptorsComparison res = FillCommonAndExtraFields(dst, src); + auto descCmpRes = FillCommonAndExtraFields(dst, src, forAttributeMerging); + if (!descCmpRes) + return R__FORWARD_RESULT(descCmpRes); + + auto res = descCmpRes.Unwrap(); std::vector slowMergeErrors; std::vector errors; @@ -775,7 +805,6 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM ROOT::NTupleSize_t nEntries, bool verbose, bool forSlowMerge) { const auto &newFields = descCmp.fExtraSrcFields; - auto &commonFields = descCmp.fCommonFields; dstModel.Unfreeze(); ROOT::Internal::RNTupleModelChangeset changeset{dstModel}; @@ -789,17 +818,18 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM return acc + (acc.length() ? ", " : "") + '`' + field->GetFieldName() + '`'; }); R__LOG_INFO(NTupleMergeLog()) << msg << ": adding " << (newFields.size() > 1 ? "them" : "it") - << " to the destination model (entry #" << destination.GetNEntries() << ")."; + << " to the destination model (entry #" << nEntries << ")."; } + ROOT::RCreateFieldOptions cfOpts; + cfOpts.SetEmulateUnknownTypes(true); + changeset.fAddedFields.reserve(newFields.size()); // First add all non-projected fields... for (const auto *fieldDesc : newFields) { if (fieldDesc->IsProjectedField()) continue; - ROOT::RCreateFieldOptions cfOpts; - cfOpts.SetEmulateUnknownTypes(true); auto field = fieldDesc->CreateField(srcDesc, cfOpts); if (!forSlowMerge) { // Explicitly set the field representatives. This prevents UpdateSchema() from changing our column @@ -818,14 +848,12 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM } changeset.AddField(std::move(field)); } - // ...then add all projected fields. + // ...then add all projected fields... for (const auto *fieldDesc : newFields) { if (!fieldDesc->IsProjectedField()) continue; ROOT::Internal::RProjectedFields::FieldMap_t fieldMap; - ROOT::RCreateFieldOptions cfOpts; - cfOpts.SetEmulateUnknownTypes(true); auto field = fieldDesc->CreateField(srcDesc, cfOpts); const auto sourceId = fieldDesc->GetProjectionSourceId(); const auto &sourceField = dstModel.GetConstField(srcDesc.GetQualifiedFieldName(sourceId)); @@ -840,6 +868,17 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM changeset.fAddedProjectedFields.emplace_back(field.get()); ROOT::Internal::GetProjectedFieldsOfModel(dstModel).Add(std::move(field), fieldMap); } + // ...finally add all subfields. + if (forSlowMerge) { // TODO(gparolini): figure out if this makes sense also in default merging + for (const auto &[parentId, subfields] : descCmp.fExtraSrcSubfields) { + const auto parentFQName = srcDesc.GetQualifiedFieldName(parentId); + // TODO(gparolini): should we RegisterSubfield(parentFQName) here if the parent is not a top-level field? + for (const auto *subfield : subfields) { + auto field = subfield->CreateField(srcDesc, cfOpts); + changeset.AddField(std::move(field), parentFQName); + } + } + } dstModel.Freeze(); try { // FIXME: here we are connecting the new fields/columns to the sink! @@ -852,6 +891,7 @@ ExtendDestinationModel(RDescriptorsComparison &descCmp, ROOT::RNTupleModel &dstM return R__FAIL(ex.what()); } + auto &commonFields = descCmp.fCommonFields; commonFields.reserve(commonFields.size() + newFields.size()); // NOTE(gparolini): Insert the new fields at the beginning of `commonFields`. // We need to make sure the extended fields appear before all other common fields for the following reason: @@ -963,7 +1003,7 @@ namespace { class RNTupleMergeStrategy { protected: // Fixed during the whole merging. Note that these references are owned by the RNTupleMerger and share its lifetime. - std::unique_ptr &fDstModel; + std::unique_ptr fDstModel; ROOT::Internal::RPagePersistentSink &fDestination; const ROOT::RNTupleDescriptor &fDstDescriptor; RNTupleMergeOptions fMergeOpts; @@ -972,14 +1012,18 @@ class RNTupleMergeStrategy { const ROOT::RNTupleDescriptor *fSrcDescriptor = nullptr; public: - RNTupleMergeStrategy(std::unique_ptr &dstModel, ROOT::Internal::RPagePersistentSink &dst, + RNTupleMergeStrategy(std::unique_ptr dstModel, ROOT::Internal::RPagePersistentSink &dst, const RNTupleMergeOptions &mergeOpts) - : fDstModel(dstModel), fDestination(dst), fDstDescriptor(dst.GetDescriptor()), fMergeOpts(mergeOpts) + : fDstModel(std::move(dstModel)), fDestination(dst), fDstDescriptor(dst.GetDescriptor()), fMergeOpts(mergeOpts) { } virtual ~RNTupleMergeStrategy() = default; + // Returns the number of entries currently written to the destination. + // This is only accurate before or after a call to MergeSource(), not during it. + virtual ROOT::NTupleSize_t GetNDstEntries() const = 0; + // Invoked as the first thing when we start processing a new source virtual void InitSource(RPageSource &source) = 0; @@ -993,6 +1037,41 @@ class RNTupleMergeStrategy { virtual void BeforeCommit() = 0; }; +/// Merge Actions used internally to merge attributes. +class RNTupleAttrMergeActions final : public RNTupleMergeActions { + /// How much do we need to shift the _rangeStart field by. This changes on each new source (and is equal to the + /// total number of entries committed at the end of the previous source). + ROOT::NTupleSize_t fAttrEntryStartOffset = 0; + const RNTupleMergeStrategy &fParentStrategy; + std::shared_ptr fRangeStartPtr; + +public: + RNTupleAttrMergeActions(const RNTupleMergeStrategy &parentStrategy, ROOT::NTupleSize_t initialStartOffset) + : fAttrEntryStartOffset(initialStartOffset), fParentStrategy(parentStrategy) + { + } + + ENTupleMergeActionResult OnFill(ROOT::REntry &entry, const RNTupleMergeActionContext &) final + { + using namespace ROOT::Experimental::Internal::RNTupleAttributes; + + // Cache pointer per source + if (!fRangeStartPtr) + fRangeStartPtr = entry.GetPtr(kMetaFieldNames[kMetaFieldIndex_RangeStart]); + + // Shift _rangeStart field by the number of main entries we already written in the destination + *fRangeStartPtr += fAttrEntryStartOffset; + return ENTupleMergeActionResult::kEmit; + } + + ENTupleMergeActionResult OnSourceEnd(ROOT::REntry &, const RNTupleMergeActionContext &) final + { + fAttrEntryStartOffset = fParentStrategy.GetNDstEntries(); + fRangeStartPtr = nullptr; // invalidate cached ptr every time we switch source + return ENTupleMergeActionResult::kDrop; + } +}; + // ========================================================================================= // // MergeStrategyDefault @@ -1028,16 +1107,16 @@ class RNTupleMergeStrategyDefault final : public RNTupleMergeStrategy { std::span extraDstColumns); public: - RNTupleMergeStrategyDefault(std::unique_ptr &dstModel, ROOT::Internal::RPagePersistentSink &dst, + RNTupleMergeStrategyDefault(std::unique_ptr dstModel, ROOT::Internal::RPagePersistentSink &dst, const RNTupleMergeOptions &mergeOpts) - : RNTupleMergeStrategy(dstModel, dst, mergeOpts) + : RNTupleMergeStrategy(std::move(dstModel), dst, mergeOpts) { #ifdef R__USE_IMT if (ROOT::IsImplicitMTEnabled()) fTaskGroup = ROOT::Experimental::TTaskGroup(); #endif - if (dstModel) { + if (fDstModel) { // If this is an incremental merging, pre-fill the column id map with the existing destination ids. // Otherwise we would generate new output ids that may not match the ones in the destination! for (const auto &field : dst.GetDescriptor().GetTopLevelFields()) { @@ -1046,6 +1125,13 @@ class RNTupleMergeStrategyDefault final : public RNTupleMergeStrategy { } } + ROOT::NTupleSize_t GetNDstEntries() const final + { + // Since with default merging we always commit clusters at source boundary, we can simply rely on + // the sink's internal counter. + return fDestination.GetNEntries(); + } + void InitSource(RPageSource &source) final { source.Attach(RNTupleSerializer::EDescriptorDeserializeMode::kForWriting); @@ -1061,8 +1147,8 @@ class RNTupleMergeStrategyDefault final : public RNTupleMergeStrategy { [[nodiscard]] ROOT::RResult ExtendDestinationModel(RDescriptorsComparison &descCmp) final { // late model extension for all fExtraSrcFields in Union mode - auto res = ::ExtendDestinationModel(descCmp, *fDstModel, fDestination, *fSrcDescriptor, - fDestination.GetNEntries(), fMergeOpts.fExtraVerbose, + auto res = ::ExtendDestinationModel(descCmp, *fDstModel, fDestination, *fSrcDescriptor, GetNDstEntries(), + fMergeOpts.fExtraVerbose, /*forSlowMerge=*/false); return res; } @@ -1575,14 +1661,22 @@ class RNTupleMergeStrategySlow final : public RNTupleMergeStrategy { RNTupleMergeActionContext fMergeContext; public: - RNTupleMergeStrategySlow(std::unique_ptr &dstModel, ROOT::Internal::RPagePersistentSink &dst, - const RNTupleMergeOptions &mergeOpts) - : RNTupleMergeStrategy(dstModel, dst, mergeOpts) + RNTupleMergeStrategySlow(std::unique_ptr dstModel, ROOT::Internal::RPagePersistentSink &dst, + const RNTupleMergeOptions &mergeOpts, + std::unique_ptr actions = std::make_unique()) + : RNTupleMergeStrategy(std::move(dstModel), dst, mergeOpts), fActions(std::move(actions)) { // dstModel is non null iff we are doing an incremental merge. Currently we don't support it. assert(!dstModel); } + ROOT::NTupleSize_t GetNDstEntries() const final + { + // During slow merging clusters are not necessarily committed at source boundary, so we need to + // manually track this number. + return fMergeContext.fDestinationEntryIndex; + } + void InitSource(RPageSource &source) final { source.Attach(); @@ -1658,8 +1752,7 @@ ROOT::RResult RNTupleMergeStrategySlow::MergeSource(RPageSource &source, R // We must not use the descriptor guard to prevent recursive locking in field.ConnectPageSource ROOT::DescriptorId_t fieldZeroId = fSrcDescriptor->GetFieldZeroId(); fieldZero.SetOnDiskId(fieldZeroId); - const auto resetFieldIdAndColumnRepresentatives = [](ROOT::RFieldBase &field, - const auto &recursiveCall) -> void { + const auto resetFieldIdAndColumnRepresentatives = [](ROOT::RFieldBase &field, const auto &recursiveCall) -> void { field.SetOnDiskId(ROOT::kInvalidDescriptorId); field.SetColumnRepresentatives({}); for (auto *f : field.GetMutableSubfields()) { @@ -1693,8 +1786,12 @@ ROOT::RResult RNTupleMergeStrategySlow::MergeSource(RPageSource &source, R // Create and connect fields to source and sink and bind them together. // This causes a read on the src entry to populate the same memory used by the dst entry to write. for (const auto &[srcFieldDesc, _dstFieldDesc] : descCmp.fCommonFields) { - if (!srcFieldDesc->IsProjectedField()) + if (!srcFieldDesc->IsProjectedField()) { + // Reset even common fields to their default value, since they may be Record fields with missing subfields + // in the source (in which case we won't overwrite them with new values and they'd keep the old ones) + fDstEntry->EmplaceNewValue(srcFieldDesc->GetFieldName()); srcEntry->BindValue(srcFieldDesc->GetFieldName(), fDstEntry->GetPtr(srcFieldDesc->GetFieldName())); + } } // reset the extraDstfields, to make sure any subfield that's missing from the current source // is default-initialized (rather than keeping its previous value). @@ -1719,6 +1816,16 @@ ROOT::RResult RNTupleMergeStrategySlow::MergeSource(RPageSource &source, R } } + while (true) { + auto res = fActions->OnSourceEnd(*fDstEntry, fMergeContext); + if (res == ENTupleMergeActionResult::kDrop) + break; + fMergeContext.fDestinationEntryIndex += 1; + fDstEntry->Append(); + if (res == ENTupleMergeActionResult::kEmit) + break; + } + if (fMergeContext.fDestinationEntryIndex > 0) { for (auto &field : ROOT::Internal::GetFieldZeroOfModel(*fDstModel)) { ROOT::Internal::CallFlushColumnsOnField(field); @@ -1787,10 +1894,25 @@ ROOT::RResult RNTupleMerger::Merge(std::span sources, const return R__FAIL(errMsg); \ } - auto strat = - doSlowMerge - ? std::unique_ptr(new RNTupleMergeStrategySlow(fModel, *fDestination, mergeOpts)) - : std::unique_ptr(new RNTupleMergeStrategyDefault(fModel, *fDestination, mergeOpts)); + auto strat = doSlowMerge ? std::unique_ptr( + new RNTupleMergeStrategySlow(std::move(fModel), *fDestination, mergeOpts)) + : std::unique_ptr( + new RNTupleMergeStrategyDefault(std::move(fModel), *fDestination, mergeOpts)); + struct RAttrSetMergeData { + std::unique_ptr fSink; + RNTupleMergeStrategySlow fStrategy; + + RAttrSetMergeData(std::unique_ptr attrSink, + const RNTupleMergeStrategy &parentStrategy, const RNTupleMergeOptions &mergeOpts, + ROOT::NTupleSize_t initialStartOffset) + : fSink(std::move(attrSink)), + fStrategy(nullptr, *fSink, mergeOpts, + std::make_unique(parentStrategy, initialStartOffset)) + { + } + }; + // Map { attrSetName => mergeData } + std::unordered_map attrSetsData; // Merge main loop for (RPageSource *source : sources) { @@ -1854,14 +1976,69 @@ ROOT::RResult RNTupleMerger::Merge(std::span sources, const } } + const auto prevNEntries = strat->GetNDstEntries(); + auto res = strat->MergeSource(*source, descCmp); if (!res) return R__FORWARD_ERROR(res); + + // Slow-merge attributes + for (const auto &attrSetDesc : source->GetSharedDescriptorGuard()->GetAttrSetIterable()) { + // Initialize output sink and strategy if it's the first time we encounter this attribute set name + auto attrSetDataIt = attrSetsData.find(attrSetDesc.GetName()); + if (attrSetDataIt == attrSetsData.end()) { + auto sinkOpts = RNTupleWriteOptions(); + sinkOpts.SetCompression(mergeOpts.fCompressionSettings.value()); + auto sink = + std::unique_ptr(static_cast( + fDestination->CloneAsHidden(attrSetDesc.GetName(), sinkOpts).release())); + auto [insertedIt, inserted] = + attrSetsData.try_emplace(attrSetDesc.GetName(), std::move(sink), *strat, mergeOpts, prevNEntries); + assert(inserted); + attrSetDataIt = insertedIt; + } + auto &attrSetData = attrSetDataIt->second; + + const auto attrLink = + ROOT::Internal::RNTupleLink{attrSetDesc.GetAnchorLocator(), attrSetDesc.GetAnchorLength()}; + auto attrSource = source->OpenWithDifferentAnchor(attrLink); + auto &attrStrat = attrSetData.fStrategy; + attrStrat.InitSource(*attrSource); + + descCmpRes = CompareDescriptorStructure(attrSetData.fSink->GetDescriptor(), + attrSource->GetSharedDescriptorGuard().GetConstRef(), + /*forAttributeMerging=*/true); + if (!descCmpRes) { + SKIP_OR_ABORT(std::string("Source RNTuple has an incompatible schema with the destination:\n") + + descCmpRes.GetError()->GetReport()) + } + descCmp = descCmpRes.Unwrap(); + + if (!descCmp.fExtraSrcFields.empty() || !descCmp.fExtraSrcSubfields.empty()) { + auto edmRes = attrStrat.ExtendDestinationModel(descCmp); + if (!edmRes) + return R__FORWARD_ERROR(edmRes); + } + + auto mergeSourceRes = attrStrat.MergeSource(*attrSource, descCmp); + if (!mergeSourceRes) + return R__FORWARD_ERROR(mergeSourceRes); + } } // end loop over sources // Commit the output + for (auto &[attrSetName, attrSetData] : attrSetsData) { + attrSetData.fStrategy.BeforeCommit(); + if (attrSetData.fSink->GetNEntries() > 0) { + attrSetData.fSink->CommitClusterGroup(); + } + auto anchorLink = attrSetData.fSink->CommitDataset(); + fDestination->CommitAttributeSet(attrSetName, anchorLink); + } + strat->BeforeCommit(); + // Commit the output if (fDestination->GetNEntries() == 0) R__LOG_WARNING(NTupleMergeLog()) << "Output RNTuple '" << fDestination->GetNTupleName() << "' has no entries."; else diff --git a/tree/ntuple/test/CMakeLists.txt b/tree/ntuple/test/CMakeLists.txt index 198fd39954f12..babfab714f248 100644 --- a/tree/ntuple/test/CMakeLists.txt +++ b/tree/ntuple/test/CMakeLists.txt @@ -43,6 +43,7 @@ if(NOT MSVC) # These unit tests rely on fork(), which is not available on Windows. ROOT_ADD_GTEST(ntuple_evolution_shape ntuple_evolution_shape.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_evolution_soa ntuple_evolution_soa.cxx LIBRARIES ROOTNTuple) + ROOT_ADD_GTEST(ntuple_evolution_merge ntuple_evolution_merge.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_emulated ntuple_emulated.cxx LIBRARIES ROOTNTuple) ROOT_ADD_GTEST(ntuple_merge_emulated ntuple_merge_emulated.cxx ntuple_test.cxx LIBRARIES ROOTNTuple xxHash::xxHash) endif() diff --git a/tree/ntuple/test/ntuple_attributes.cxx b/tree/ntuple/test/ntuple_attributes.cxx index c8ae7496d31b9..fe7c4d8415032 100644 --- a/tree/ntuple/test/ntuple_attributes.cxx +++ b/tree/ntuple/test/ntuple_attributes.cxx @@ -636,7 +636,7 @@ TEST(RNTupleAttributes, MultipleSetsTFile) auto reader = RNTupleReader::Open(*ntpl); EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 2); - // Extract the headers' offsets + // Extract the headers' offsets std::uint64_t mainHeaderOff = ntpl->GetSeekHeader(); std::uint64_t attrSet1Off = 0, attrSet2Off = 0; int n = 0; @@ -1348,3 +1348,24 @@ TEST(RNTupleAttributes, WriteZeroAttributes) auto iterCG = attrSetReader->GetDescriptor().GetClusterGroupIterable(); EXPECT_EQ(iterCG.begin(), iterCG.end()); } + +TEST(RNTupleAttributes, UntypedRecord) +{ + FileRaii fileGuard1("test_ntuple_attr_untyped_record_1.root"); + + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + std::vector> children; + children.push_back(std::make_unique>("int")); + children.push_back(std::make_unique>("char")); + auto record = std::make_unique("record", std::move(children)); + attrSetModel->AddField(std::move(record)); + try { + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + FAIL() << "using Untyped records as attributes should fail"; + } catch (const ROOT::RException &ex) { + EXPECT_THAT(ex.what(), testing::HasSubstr("Untyped records or collections are currently unsupported")); + } +} diff --git a/tree/ntuple/test/ntuple_evolution_merge.cxx b/tree/ntuple/test/ntuple_evolution_merge.cxx new file mode 100644 index 0000000000000..88406a222217c --- /dev/null +++ b/tree/ntuple/test/ntuple_evolution_merge.cxx @@ -0,0 +1,119 @@ +#include "ntuple_test.hxx" + +#include "ntuple_fork.hxx" +#include "ntuple_test_evolution.hxx" + +#include + +#include +#include + +TEST(RNTupleMerger, MergeWithAttributesSchemaEvolve) +{ + ROOT::TestSupport::CheckDiagsRAII diagsRAII; + diagsRAII.requiredDiag(kWarning, "ROOT.NTuple", "RNTuple Attributes are experimental", false); + + // Write two test ntuples to be merged. + // These files have 1 attribute set with the same name and they contain a custom class field. + // The second class field has a newer version that has been schema evolved from the first. + FileRaii fileGuard1("test_ntuple_merge_attr_schemaevolve_1.root"); + ExecInFork([&] { + fileGuard1.PreserveFile(); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct MyClass { + int fInt1 = 1; + int fInt3 = 3; +}; +)")); + + auto model = RNTupleModel::Create(); + auto pInt = model->MakeField("mainInt"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrModel = RNTupleModel::Create(); + attrModel->AddField(RFieldBase::Create("attr", "MyClass").Unwrap()); + auto attrSet = writer->CreateAttributeSet(std::move(attrModel), "AttrSet1"); + void *pAttr = attrSet->GetModel().GetDefaultEntry().GetPtr("attr").get(); + auto attrRange = attrSet->BeginRange(); + DeclarePointer("MyClass", "pMyClass", pAttr); + ProcessLine("pMyClass->fInt1 = 11;"); + ProcessLine("pMyClass->fInt3 = 33;"); + *pInt = 100; + writer->Fill(); + attrSet->CommitRange(std::move(attrRange)); + + // Reset / close the writer and flush the file. + writer.reset(); + }); + + ASSERT_TRUE(gInterpreter->Declare(R"( +struct MyClass { + int fInt1 = 1; + int fInt2 = 2; + int fInt3 = 3; +}; +)")); + + FileRaii fileGuard2("test_ntuple_merge_attr_schemaevolve_2.root"); + { + auto model = RNTupleModel::Create(); + auto pInt = model->MakeField("mainInt"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + auto attrModel = RNTupleModel::Create(); + attrModel->AddField(RFieldBase::Create("attr", "MyClass").Unwrap()); + auto attrSet = writer->CreateAttributeSet(std::move(attrModel), "AttrSet1"); + void *pAttr = attrSet->GetModel().GetDefaultEntry().GetPtr("attr").get(); + auto attrRange = attrSet->BeginRange(); + DeclarePointer("MyClass", "pMyClass", pAttr); + ProcessLine("pMyClass->fInt1 = 22;"); + ProcessLine("pMyClass->fInt2 = 44;"); + ProcessLine("pMyClass->fInt3 = 66;"); + *pInt = 200; + writer->Fill(); + attrSet->CommitRange(std::move(attrRange)); + } + + std::vector> sources; + sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); + sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) { + sourcePtrs.push_back(s.get()); + } + + // Now merge the inputs + for (const auto mmode : {ENTupleMergingMode::kFilter, ENTupleMergingMode::kStrict, ENTupleMergingMode::kUnion}) { + const auto modeString = ToString(mmode); + SCOPED_TRACE(std::string("with merging mode = ") + modeString); + FileRaii fileGuardOut("test_ntuple_merge_attr_schemaevolve_out.root"); + { + auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), RNTupleWriteOptions()); + RNTupleMerger merger{std::move(destination)}; + RNTupleMergeOptions opts; + opts.fMergingMode = mmode; + auto res = merger.Merge(sourcePtrs, opts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); + } + { + auto reader = ROOT::RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 2); + EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 1); + { + auto attrSet1 = reader->OpenAttributeSet("AttrSet1"); + EXPECT_EQ(attrSet1->GetNEntries(), 2); + const auto &attrEntry1 = attrSet1->GetModel().GetDefaultEntry(); + auto pAttr = attrEntry1.GetPtr("attr").get(); + const auto ptrName = std::string("pMyClass_") + modeString; + DeclarePointer("MyClass", ptrName, pAttr); + for (auto idx : attrSet1->GetAttributes()) { + auto range1 = attrSet1->LoadEntry(idx); + EXPECT_EQ(range1.GetFirst(), idx); + EXPECT_EQ(range1.GetLast(), idx); + EXPECT_EVALUATE_EQ((ptrName + "->fInt1").c_str(), 11 * (idx + 1)); + EXPECT_EVALUATE_EQ((ptrName + "->fInt2").c_str(), idx == 1 ? 44 : 2); + EXPECT_EVALUATE_EQ((ptrName + "->fInt3").c_str(), 33 * (idx + 1)); + } + } + } + } +} diff --git a/tree/ntuple/test/ntuple_merger.cxx b/tree/ntuple/test/ntuple_merger.cxx index 347f6d32acb13..d59bf9a64d7f3 100644 --- a/tree/ntuple/test/ntuple_merger.cxx +++ b/tree/ntuple/test/ntuple_merger.cxx @@ -4918,3 +4918,397 @@ TEST(RNTupleMerger, MergeRealRegularQuantMixed) } } } + +TEST(RNTupleMerger, MergeWithAttributesSimple) +{ + ROOT::TestSupport::CheckDiagsRAII diagsRAII; + diagsRAII.requiredDiag(kWarning, "ROOT.NTuple", "RNTuple Attributes are experimental", false); + + // Write two test ntuples to be merged. + // These files have a single attribute set in common. + // We expect the output file to contain it, with the union of all the sources' attribute entries. + FileRaii fileGuard1("test_ntuple_merge_attr1.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrInt = attrSetModel->MakeField("attrInt"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrInt = 1; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + FileRaii fileGuard2("test_ntuple_merge_attr2.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrInt = attrSetModel->MakeField("attrInt"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrInt = 2; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 567; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + std::vector> sources; + sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); + sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) { + sourcePtrs.push_back(s.get()); + } + + // Now merge the inputs + for (const auto mmode : {ENTupleMergingMode::kFilter, ENTupleMergingMode::kStrict, ENTupleMergingMode::kUnion}) { + SCOPED_TRACE(std::string("with merging mode = ") + ToString(mmode)); + FileRaii fileGuardOut("test_ntuple_merge_attr_out.root"); + { + auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), RNTupleWriteOptions()); + RNTupleMerger merger{std::move(destination)}; + RNTupleMergeOptions opts; + opts.fMergingMode = mmode; + auto res = merger.Merge(sourcePtrs, opts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); + } + { + auto reader = ROOT::RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 20); + EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 1); + { + auto attrSet = reader->OpenAttributeSet("AttrSet1"); + auto attrEntry = attrSet->CreateEntry(); + auto pAttrInt = attrEntry->GetPtr("attrInt"); + EXPECT_EQ(attrSet->GetNEntries(), 2); + for (auto idx : attrSet->GetAttributes()) { + auto range1 = attrSet->LoadEntry(idx, *attrEntry); + EXPECT_EQ(range1.GetFirst(), 10 * idx); + EXPECT_EQ(range1.GetLast(), 10 * idx + 9); + EXPECT_EQ(*pAttrInt, idx + 1); + } + } + } + } +} + +TEST(RNTupleMerger, MergeWithAttributes) +{ + ROOT::TestSupport::CheckDiagsRAII diagsRAII; + diagsRAII.requiredDiag(kWarning, "ROOT.NTuple", "RNTuple Attributes are experimental", false); + + // Write two test ntuples to be merged. + // These files both have 2 attribute sets, one of which (AttrSet1) is common to both. + // We expect the output file to contain all 3 attribute sets, where AttrSet1 has the union of both sets' entries. + FileRaii fileGuard1("test_ntuple_merge_attr2_1.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto fieldBar = model->MakeField("bar"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + attrSetModel->MakeField("int"); + auto attrSet1 = writer->CreateAttributeSet(attrSetModel->Clone(), "AttrSet1"); + attrSetModel->MakeField("long"); + auto attrSet2 = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet2"); + auto &attrEntry1 = attrSet1->GetModel().GetDefaultEntry(); + auto &attrEntry2 = attrSet2->GetModel().GetDefaultEntry(); + auto attrRange1 = attrSet1->BeginRange(); + auto attrRange2 = attrSet2->BeginRange(); + *attrEntry1.GetPtr("int") = 1; + *attrEntry2.GetPtr("int") = 2; + *attrEntry2.GetPtr("long") = 3; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + *fieldBar = i * 321; + writer->Fill(); + } + attrSet1->CommitRange(std::move(attrRange1)); + attrSet2->CommitRange(std::move(attrRange2)); + } + + FileRaii fileGuard2("test_ntuple_merge_attr2_2.root"); + { + auto model = RNTupleModel::Create(); + auto fieldBar = model->MakeField("bar"); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + attrSetModel->MakeField("int"); + auto attrSet1 = writer->CreateAttributeSet(attrSetModel->Clone(), "AttrSet1"); + attrSetModel->MakeField("string"); + auto attrSet3 = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet3"); + auto &attrEntry1 = attrSet1->GetModel().GetDefaultEntry(); + auto &attrEntry3 = attrSet3->GetModel().GetDefaultEntry(); + auto attrRange1 = attrSet1->BeginRange(); + auto attrRange3 = attrSet3->BeginRange(); + *attrEntry1.GetPtr("int") = 4; + *attrEntry3.GetPtr("int") = 5; + *attrEntry3.GetPtr("string") = "6"; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 567; + *fieldBar = i * 765; + writer->Fill(); + } + attrSet1->CommitRange(std::move(attrRange1)); + attrSet3->CommitRange(std::move(attrRange3)); + } + + std::vector> sources; + sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); + sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) { + sourcePtrs.push_back(s.get()); + } + + // Now merge the inputs + for (const auto mmode : {ENTupleMergingMode::kFilter, ENTupleMergingMode::kStrict, ENTupleMergingMode::kUnion}) { + SCOPED_TRACE(std::string("with merging mode = ") + ToString(mmode)); + FileRaii fileGuardOut("test_ntuple_merge_attr2_out.root"); + { + auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), RNTupleWriteOptions()); + RNTupleMerger merger{std::move(destination)}; + RNTupleMergeOptions opts; + opts.fMergingMode = mmode; + auto res = merger.Merge(sourcePtrs, opts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); + } + { + auto reader = ROOT::RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 20); + EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 3); + { + auto attrSet1 = reader->OpenAttributeSet("AttrSet1"); + auto attrEntry1 = attrSet1->CreateEntry(); + auto pInt1 = attrEntry1->GetPtr("int"); + EXPECT_EQ(attrSet1->GetNEntries(), 2); + for (auto idx : attrSet1->GetAttributes()) { + auto range1 = attrSet1->LoadEntry(idx, *attrEntry1); + EXPECT_EQ(range1.GetFirst(), 10 * idx); + EXPECT_EQ(range1.GetLast(), 10 * idx + 9); + EXPECT_EQ(*pInt1, idx < 1 ? 1 : 4); + } + } + { + auto attrSet2 = reader->OpenAttributeSet("AttrSet2"); + auto attrEntry2 = attrSet2->CreateEntry(); + auto pInt2 = attrEntry2->GetPtr("int"); + auto pLong2 = attrEntry2->GetPtr("long"); + EXPECT_EQ(attrSet2->GetNEntries(), 1); + for (auto idx : attrSet2->GetAttributes()) { + auto range2 = attrSet2->LoadEntry(idx, *attrEntry2); + EXPECT_EQ(range2.GetFirst(), 0); + EXPECT_EQ(range2.GetLast(), 9); + EXPECT_EQ(*pInt2, 2); + EXPECT_EQ(*pLong2, 3); + } + } + { + auto attrSet3 = reader->OpenAttributeSet("AttrSet3"); + auto attrEntry3 = attrSet3->CreateEntry(); + auto pInt3 = attrEntry3->GetPtr("int"); + auto pStr3 = attrEntry3->GetPtr("string"); + EXPECT_EQ(attrSet3->GetNEntries(), 1); + for (auto idx : attrSet3->GetAttributes()) { + auto range3 = attrSet3->LoadEntry(idx, *attrEntry3); + EXPECT_EQ(range3.GetFirst(), 10); + EXPECT_EQ(range3.GetLast(), 19); + EXPECT_EQ(*pInt3, 5); + EXPECT_EQ(*pStr3, "6"); + } + } + } + } +} + +TEST(RNTupleMerger, MergeWithAttributesMissingFields) +{ + ROOT::TestSupport::CheckDiagsRAII diagsRAII; + diagsRAII.requiredDiag(kWarning, "ROOT.NTuple", "RNTuple Attributes are experimental", false); + + // Write two test ntuples to be merged. + // These files have 1 attribute set with the same name, but the first contains extra fields. + // We expect the output file to contain that attribute field with the union of all attributes. + FileRaii fileGuard1("test_ntuple_merge_attr_missing_1.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrInt = attrSetModel->MakeField("int"); + auto pAttrChar = attrSetModel->MakeField("char"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrInt = 0x42; + *pAttrChar = 'f'; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + FileRaii fileGuard2("test_ntuple_merge_attr_missing_2.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrChar = attrSetModel->MakeField("char"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrChar = 'g'; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + std::vector> sources; + sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); + sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) { + sourcePtrs.push_back(s.get()); + } + + // Now merge the inputs + for (const auto mmode : {ENTupleMergingMode::kFilter, ENTupleMergingMode::kStrict, ENTupleMergingMode::kUnion}) { + SCOPED_TRACE(std::string("with merging mode = ") + ToString(mmode)); + FileRaii fileGuardOut("test_ntuple_merge_attr_missing_out.root"); + { + auto wopts = RNTupleWriteOptions(); + wopts.SetCompression(0); + auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), wopts); + RNTupleMerger merger{std::move(destination)}; + RNTupleMergeOptions opts; + opts.fMergingMode = mmode; + opts.fCompressionSettings = 0; + auto res = merger.Merge(sourcePtrs, opts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); + } + { + auto reader = ROOT::RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 20); + EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 1); + { + auto attrSet1 = reader->OpenAttributeSet("AttrSet1"); + EXPECT_EQ(attrSet1->GetNEntries(), 2); + const auto &attrEntry1 = attrSet1->GetModel().GetDefaultEntry(); + auto pAttrInt = attrEntry1.GetPtr("int"); + auto pAttrChar = attrEntry1.GetPtr("char"); + for (auto idx : attrSet1->GetAttributes()) { + auto range1 = attrSet1->LoadEntry(idx); + EXPECT_EQ(range1.GetFirst(), 10 * idx); + EXPECT_EQ(range1.GetLast(), 10 * idx + 9); + EXPECT_EQ(*pAttrInt, idx == 0 ? 0x42 : 0); + EXPECT_EQ(*pAttrChar, idx == 0 ? 'f' : 'g'); + } + } + } + + fileGuardOut.PreserveFile(); + break; + } +} + +TEST(RNTupleMerger, MergeWithAttributesLateModelExtend) +{ + ROOT::TestSupport::CheckDiagsRAII diagsRAII; + diagsRAII.requiredDiag(kWarning, "ROOT.NTuple", "RNTuple Attributes are experimental", false); + + // Write two test ntuples to be merged. + // These files have 1 attribute set with the same name, but the second contains extra fields. + // We expect the output file to contain that attribute field with the union of all attributes. + FileRaii fileGuard1("test_ntuple_merge_attr_ext_1.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard1.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrInt = attrSetModel->MakeField("int"); + auto pAttrChar = attrSetModel->MakeField("char"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrInt = 42; + *pAttrChar = 'f'; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + FileRaii fileGuard2("test_ntuple_merge_attr_ext_2.root"); + { + auto model = RNTupleModel::Create(); + auto fieldFoo = model->MakeField("foo"); + auto writer = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard2.GetPath()); + auto attrSetModel = RNTupleModel::Create(); + auto pAttrFloat = attrSetModel->MakeField("float"); + auto pAttrChar = attrSetModel->MakeField("char"); + auto attrSet = writer->CreateAttributeSet(std::move(attrSetModel), "AttrSet1"); + auto attrRange = attrSet->BeginRange(); + *pAttrFloat = 84.f; + *pAttrChar = 'g'; + for (size_t i = 0; i < 10; ++i) { + *fieldFoo = i * 123; + writer->Fill(); + } + attrSet->CommitRange(std::move(attrRange)); + } + + std::vector> sources; + sources.push_back(RPageSource::Create("ntuple", fileGuard1.GetPath(), RNTupleReadOptions())); + sources.push_back(RPageSource::Create("ntuple", fileGuard2.GetPath(), RNTupleReadOptions())); + std::vector sourcePtrs; + for (const auto &s : sources) { + sourcePtrs.push_back(s.get()); + } + + // Now merge the inputs + for (const auto mmode : {ENTupleMergingMode::kFilter, ENTupleMergingMode::kStrict, ENTupleMergingMode::kUnion}) { + SCOPED_TRACE(std::string("with merging mode = ") + ToString(mmode)); + FileRaii fileGuardOut("test_ntuple_merge_attr_ext_out.root"); + { + auto destination = std::make_unique("ntuple", fileGuardOut.GetPath(), RNTupleWriteOptions()); + RNTupleMerger merger{std::move(destination)}; + RNTupleMergeOptions opts; + opts.fMergingMode = mmode; + auto res = merger.Merge(sourcePtrs, opts); + ASSERT_TRUE(bool(res)) << res.GetError()->GetReport(); + } + { + auto reader = ROOT::RNTupleReader::Open("ntuple", fileGuardOut.GetPath()); + EXPECT_EQ(reader->GetNEntries(), 20); + EXPECT_EQ(reader->GetDescriptor().GetNAttributeSets(), 1); + { + auto attrSet1 = reader->OpenAttributeSet("AttrSet1"); + EXPECT_EQ(attrSet1->GetNEntries(), 2); + const auto &attrEntry1 = attrSet1->GetModel().GetDefaultEntry(); + auto pAttrInt = attrEntry1.GetPtr("int"); + auto pAttrChar = attrEntry1.GetPtr("char"); + auto pAttrFloat = attrEntry1.GetPtr("float"); + for (auto idx : attrSet1->GetAttributes()) { + auto range1 = attrSet1->LoadEntry(idx); + EXPECT_EQ(range1.GetFirst(), 10 * idx); + EXPECT_EQ(range1.GetLast(), 10 * idx + 9); + EXPECT_EQ(*pAttrInt, idx == 0 ? 42 : 0); + EXPECT_EQ(*pAttrChar, idx == 0 ? 'f' : 'g'); + EXPECT_FLOAT_EQ(*pAttrFloat, idx == 0 ? 0.f : 84.f); + } + } + } + } +} diff --git a/tree/ntuple/test/ntuple_test.hxx b/tree/ntuple/test/ntuple_test.hxx index 6328362aa0a7f..c2c98c9172605 100644 --- a/tree/ntuple/test/ntuple_test.hxx +++ b/tree/ntuple/test/ntuple_test.hxx @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include