From acfab2c2e196f9987a28ed9396acaa41fc136eed Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 8 Sep 2026 16:43:49 +0200 Subject: [PATCH 1/6] [ntuple] add RRuleField common base class Add a new RRuleField internal base class containing the code dealing with I/O customization rules. This class is initially used by RClassField and will later be reused by RSoAField. --- tree/ntuple/inc/ROOT/RField.hxx | 74 ++++--- tree/ntuple/src/RFieldMeta.cxx | 356 ++++++++++++++++---------------- 2 files changed, 225 insertions(+), 205 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 1429a8e3f0b5c..34eb792ee2812 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -131,17 +131,9 @@ public: size_t GetAlignment() const final { return 0; } }; // RInvalidField -/// The field for a class with dictionary -class RClassField : public RFieldBase { -private: - enum ESubfieldRole { - kBaseClass, - kDataMember, - }; - struct RSubfieldInfo { - ESubfieldRole fRole; - std::size_t fOffset; - }; +/// Base class for fields that are subject to I/O customization rules. Used by the class field and the SoA field. +class RRuleField : public RFieldBase { +protected: // Information to read into the staging area a field that is used as an input to an I/O customization rule struct RStagingItem { /// The field used to read the on-disk data. The fields type may be different from the on-disk type as long @@ -152,19 +144,6 @@ private: /// Prefix used in the subfield names generated for base classes static constexpr const char *kPrefixInherited{":"}; - class RClassDeleter : public RDeleter { - private: - TClass *fClass; - - public: - explicit RClassDeleter(TClass *cl); - void operator()(void *objPtr, bool dtorOnly) final; - }; - - TClass *fClass; - /// Additional information kept for each entry in `fSubfields` - std::vector fSubfieldsInfo; - /// The staging area stores inputs to I/O rules according to the offsets given by the streamer info of /// "TypeName@@Version". The area is allocated depending on I/O rules resp. the source members of the I/O rules. std::unique_ptr fStagingArea; @@ -175,15 +154,15 @@ private: TClass *fStagingClass = nullptr; std::unordered_map fStagingItems; ///< Lookup staging items by member name -private: - RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl - RClassField(std::string_view fieldName, TClass *classp); - void Attach(std::unique_ptr child, RSubfieldInfo info); + RRuleField(std::string_view name, std::string_view type, ROOT::ENTupleStructure structure); + + /// Derived classes should return the TClass instance representing the current in-memory layout. + virtual TClass *GetInMemoryClass() const = 0; /// Returns the id of member 'name' in the class field given by 'fieldId', or kInvalidDescriptorId if no such /// member exist. Looks recursively in base classes. - ROOT::DescriptorId_t - LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, ROOT::DescriptorId_t classFieldId); + ROOT::DescriptorId_t LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const; /// Sets fStagingClass according to the given name and version void SetStagingClass(const std::string &className, unsigned int classVersion); /// If there are rules with inputs (source members), create the staging area according to the TClass instance @@ -196,7 +175,38 @@ private: /// to the class field at hand, to which the fieldDesc descriptor, if provided, must correspond. /// Fields may not have an on-disk representation (e.g., when inserted by schema evolution), in which case the passed /// field descriptor is nullptr. - std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc); + std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc) const; +}; + +/// The field for a class with dictionary +class RClassField : public RRuleField { +private: + enum ESubfieldRole { + kBaseClass, + kDataMember, + }; + struct RSubfieldInfo { + ESubfieldRole fRole; + std::size_t fOffset; + }; + + class RClassDeleter : public RDeleter { + private: + TClass *fClass; + + public: + explicit RClassDeleter(TClass *cl); + void operator()(void *objPtr, bool dtorOnly) final; + }; + + TClass *fClass; + /// Additional information kept for each entry in `fSubfields` + std::vector fSubfieldsInfo; + +private: + RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl + RClassField(std::string_view fieldName, TClass *classp); + void Attach(std::unique_ptr child, RSubfieldInfo info); protected: std::unique_ptr CloneImpl(std::string_view newName) const final; @@ -211,6 +221,8 @@ protected: std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; + TClass *GetInMemoryClass() const final { return fClass; } + public: RClassField(std::string_view fieldName, std::string_view className); RClassField(RClassField &&other) = default; diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index e500b761c5dd6..5422b3ce1af4f 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -191,8 +191,188 @@ std::string BuildMapTypeName(ROOT::RMapField::EMapType mapType, const ROOT::RFie } // anonymous namespace +ROOT::RRuleField::RRuleField(std::string_view fieldName, std::string_view typeName, ROOT::ENTupleStructure structure) + : RFieldBase(fieldName, typeName, structure, false /* isSimple */) +{ +} + +ROOT::DescriptorId_t ROOT::RRuleField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const +{ + auto idSourceMember = desc.FindFieldId(memberName, classFieldId); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + + for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { + const auto &subFieldName = subFieldDesc.GetFieldName(); + if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { + idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + } + } + + return ROOT::kInvalidDescriptorId; +} + +void ROOT::RRuleField::SetStagingClass(const std::string &className, unsigned int classVersion) +{ + TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); + if (classVersion != GetTypeVersion() || className != GetTypeName()) { + fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); + if (!fStagingClass) { + // For a rename rule, we may simply ask for the old class name + fStagingClass = TClass::GetClass(className.c_str()); + } + } else { + fStagingClass = GetInMemoryClass(); + } + R__ASSERT(fStagingClass); + R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); +} + +void ROOT::RRuleField::PrepareStagingArea(const std::vector &rules, + const ROOT::RNTupleDescriptor &desc, + const ROOT::RFieldDescriptor &classFieldDesc) +{ + std::size_t stagingAreaSize = 0; + for (const auto rule : rules) { + for (auto source : TRangeDynCast(rule->GetSource())) { + auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); + if (!isNew) { + // This source member has already been processed by another rule (and we only support one type per member) + continue; + } + RStagingItem &stagingItem = itr->second; + + const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); + if (memberFieldId == kInvalidDescriptorId) { + throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + + source->GetName())); + } + + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); + memberField->SetOnDiskId(memberFieldId); + auto fieldZero = std::make_unique(); + Internal::SetAllowFieldSubstitutions(*fieldZero, true); + fieldZero->Attach(std::move(memberField)); + stagingItem.fField = std::move(fieldZero); + + stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); + // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it + // to be present in the TClass instance, too. + R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); + stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); + } + } + + if (stagingAreaSize) { + R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules + // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. + fStagingArea = std::make_unique(stagingAreaSize); + + for (const auto &[_, si] : fStagingItems) { + const auto &memberField = *si.fField->cbegin(); + if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { + CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); + } + } + } +} + +std::vector ROOT::RRuleField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) const +{ + ROOT::Detail::TSchemaRuleSet::TMatches rules; + auto cl = GetInMemoryClass(); + + const auto ruleset = cl->GetSchemaRules(); + if (!ruleset) + return rules; + + if (!fieldDesc) { + // If we have no on-disk information for the field, we still process the rules on the current in-memory version + // of the class + rules = ruleset->FindRules(cl->GetName(), cl->GetClassVersion(), cl->GetCheckSum()); + } else { + // We need to change (back) the name normalization from RNTuple to ROOT Meta + std::string normalizedName; + TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); + // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the + // on-disk version of the field. + if (fieldDesc->GetTypeChecksum()) { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); + } else { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); + } + } + + // Cleanup and sort rules + // Check that any any given source member uses the same type in all rules + std::unordered_map sourceNameAndType; + std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector + for (auto itr = rules.begin(); itr != rules.end() - nskip;) { + const auto rule = *itr; + + // Erase unknown rule types + if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); + itr = rules.erase(itr); + continue; + } + + bool hasConflictingSourceMembers = false; + for (auto source : TRangeDynCast(rule->GetSource())) { + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); + if (!isNew && (itrSrc->second != memberType)) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " + << memberType << " for member " << source->GetName(); + hasConflictingSourceMembers = true; + break; + } + } + if (hasConflictingSourceMembers) { + itr = rules.erase(itr); + continue; + } + + // Rules targeting the entire object need to be executed at the end + if (rule->GetTarget() == nullptr) { + nskip++; + if (itr != rules.end() - nskip) + std::iter_swap(itr++, rules.end() - nskip); + continue; + } + + ++itr; + } + + return rules; +} + +void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule) +{ + auto func = rule->GetReadFunctionPointer(); + if (func == nullptr) { + // Can happen for rename rules + return; + } + fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { + TVirtualObject onfileObj{nullptr}; + onfileObj.fClass = stagingClass; + onfileObj.fObject = stagingArea; + func(static_cast(target), &onfileObj); + onfileObj.fObject = nullptr; // TVirtualObject does not own the value + }); +} + +//------------------------------------------------------------------------------ + ROOT::RClassField::RClassField(std::string_view fieldName, const RClassField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord, false /* isSimple */), + : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord), fClass(source.fClass), fSubfieldsInfo(source.fSubfieldsInfo) { @@ -208,8 +388,7 @@ ROOT::RClassField::RClassField(std::string_view fieldName, std::string_view clas } ROOT::RClassField::RClassField(std::string_view fieldName, TClass *classp) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord, - false /* isSimple */), + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord), fClass(classp) { EnsureValidUserClass(fClass, *this, "RClassField"); @@ -291,76 +470,6 @@ void ROOT::RClassField::Attach(std::unique_ptr child, RSubfieldInfo RFieldBase::Attach(std::move(child)); } -std::vector ROOT::RClassField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) -{ - ROOT::Detail::TSchemaRuleSet::TMatches rules; - const auto ruleset = fClass->GetSchemaRules(); - if (!ruleset) - return rules; - - if (!fieldDesc) { - // If we have no on-disk information for the field, we still process the rules on the current in-memory version - // of the class - rules = ruleset->FindRules(fClass->GetName(), fClass->GetClassVersion(), fClass->GetCheckSum()); - } else { - // We need to change (back) the name normalization from RNTuple to ROOT Meta - std::string normalizedName; - TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); - // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the - // on-disk version of the field. - if (fieldDesc->GetTypeChecksum()) { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); - } else { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); - } - } - - // Cleanup and sort rules - // Check that any any given source member uses the same type in all rules - std::unordered_map sourceNameAndType; - std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector - for (auto itr = rules.begin(); itr != rules.end() - nskip;) { - const auto rule = *itr; - - // Erase unknown rule types - if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); - itr = rules.erase(itr); - continue; - } - - bool hasConflictingSourceMembers = false; - for (auto source : TRangeDynCast(rule->GetSource())) { - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); - if (!isNew && (itrSrc->second != memberType)) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " - << memberType << " for member " << source->GetName(); - hasConflictingSourceMembers = true; - break; - } - } - if (hasConflictingSourceMembers) { - itr = rules.erase(itr); - continue; - } - - // Rules targeting the entire object need to be executed at the end - if (rule->GetTarget() == nullptr) { - nskip++; - if (itr != rules.end() - nskip) - std::iter_swap(itr++, rules.end() - nskip); - continue; - } - - ++itr; - } - - return rules; -} - std::unique_ptr ROOT::RClassField::CloneImpl(std::string_view newName) const { return std::unique_ptr(new RClassField(newName, *this)); @@ -395,107 +504,6 @@ void ROOT::RClassField::ReadInClusterImpl(RNTupleLocalIndex localIndex, void *to } } -ROOT::DescriptorId_t ROOT::RClassField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, - ROOT::DescriptorId_t classFieldId) -{ - auto idSourceMember = desc.FindFieldId(memberName, classFieldId); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - - for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { - const auto &subFieldName = subFieldDesc.GetFieldName(); - if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { - idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - } - } - - return ROOT::kInvalidDescriptorId; -} - -void ROOT::RClassField::SetStagingClass(const std::string &className, unsigned int classVersion) -{ - TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); - if (classVersion != GetTypeVersion() || className != GetTypeName()) { - fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); - if (!fStagingClass) { - // For a rename rule, we may simply ask for the old class name - fStagingClass = TClass::GetClass(className.c_str()); - } - } else { - fStagingClass = fClass; - } - R__ASSERT(fStagingClass); - R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); -} - -void ROOT::RClassField::PrepareStagingArea(const std::vector &rules, - const ROOT::RNTupleDescriptor &desc, - const ROOT::RFieldDescriptor &classFieldDesc) -{ - std::size_t stagingAreaSize = 0; - for (const auto rule : rules) { - for (auto source : TRangeDynCast(rule->GetSource())) { - auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); - if (!isNew) { - // This source member has already been processed by another rule (and we only support one type per member) - continue; - } - RStagingItem &stagingItem = itr->second; - - const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); - if (memberFieldId == kInvalidDescriptorId) { - throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + - source->GetName())); - } - - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); - memberField->SetOnDiskId(memberFieldId); - auto fieldZero = std::make_unique(); - Internal::SetAllowFieldSubstitutions(*fieldZero, true); - fieldZero->Attach(std::move(memberField)); - stagingItem.fField = std::move(fieldZero); - - stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); - // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it - // to be present in the TClass instance, too. - R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); - stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); - } - } - - if (stagingAreaSize) { - R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules - // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. - fStagingArea = std::make_unique(stagingAreaSize); - - for (const auto &[_, si] : fStagingItems) { - const auto &memberField = *si.fField->cbegin(); - if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { - CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); - } - } - } -} - -void ROOT::RClassField::AddReadCallbacksFromIORule(const TSchemaRule *rule) -{ - auto func = rule->GetReadFunctionPointer(); - if (func == nullptr) { - // Can happen for rename rules - return; - } - fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { - TVirtualObject onfileObj{nullptr}; - onfileObj.fClass = stagingClass; - onfileObj.fObject = stagingArea; - func(static_cast(target), &onfileObj); - onfileObj.fObject = nullptr; // TVirtualObject does not own the value - }); -} - std::unique_ptr ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) { std::vector rules; From 4bcaa0accda4061833f39aeca19562d240dd94a4 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 8 Sep 2026 16:49:46 +0200 Subject: [PATCH 2/6] [ntuple] let RSoAField inherit from RRuleField --- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 4 +++- tree/ntuple/src/RFieldMeta.cxx | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index af503964a865c..b9536af6ea912 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -53,7 +53,7 @@ namespace Experimental { /// /// Since the on-disk representation is a collection of record type, the class version and checksum of the SoA type /// itself is ignored. -class RSoAField : public RFieldBase { +class RSoAField : public RRuleField { class RSoADeleter : public RDeleter { private: TClass *fSoAClass; @@ -116,6 +116,8 @@ protected: void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; + TClass *GetInMemoryClass() const final { return fSoAClass; } + public: RSoAField(std::string_view fieldName, std::string_view className); RSoAField(RSoAField &&other) = default; diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index 5422b3ce1af4f..548698dc3fa8d 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -690,7 +690,7 @@ void ROOT::RClassField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) cons //------------------------------------------------------------------------------ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, const RSoAField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection, false /* isSimple */), + : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection), fSoAClass(source.fSoAClass), fSoAMemberOffsets(source.fSoAMemberOffsets) { @@ -880,8 +880,7 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection, - false /* isSimple */), + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection), fSoAClass(clSoA) { static std::once_flag once; From 40609d573434ddb021f1b2023fe4501c50f0011d Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Wed, 9 Sep 2026 10:20:41 +0200 Subject: [PATCH 3/6] [ntuple] basic support rules in SoA classes Adds support for rename and whole-object rules. Errors out when rules with targets or sources are found. --- tree/ntuple/inc/ROOT/RField.hxx | 4 +- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 14 ++- tree/ntuple/src/RFieldMeta.cxx | 66 ++++++++++-- tree/ntuple/test/SoAField.hxx | 99 +++++++++++++++++ tree/ntuple/test/SoAFieldLinkDef.h | 39 +++++++ tree/ntuple/test/ntuple_soa.cxx | 126 ++++++++++++++++++++-- 6 files changed, 329 insertions(+), 19 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 34eb792ee2812..889220f2dc2ca 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -170,7 +170,9 @@ protected: void PrepareStagingArea(const std::vector &rules, const ROOT::RNTupleDescriptor &desc, const ROOT::RFieldDescriptor &classFieldId); /// Register post-read callback corresponding to a ROOT I/O customization rules. - void AddReadCallbacksFromIORule(const TSchemaRule *rule); + /// The sub object offset allows to apply the rule to a nested object within the passed target. + /// This is used to execute rules on base classes and nested classes in a SoA field. + void AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset = 0); /// Given the on-disk information from the page source, find all the I/O customization rules that apply /// to the class field at hand, to which the fieldDesc descriptor, if provided, must correspond. /// Fields may not have an on-disk representation (e.g., when inserted by schema evolution), in which case the passed diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index b9536af6ea912..bc001198d4980 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -63,6 +63,13 @@ class RSoAField : public RRuleField { void operator()(void *objPtr, bool dtorOnly) final; }; + // A rule of the SoA type itself or one of its nested SoA types or base classes, togther with the offset + // of the nested object in the SoA type. + struct RRule { + const TSchemaRule *fRule = nullptr; + std::size_t fOffset = 0; + }; + TClass *fSoAClass = nullptr; /// Direct access to the member fields of the underlying record. In case of a nested SoA type, this vector /// contains the contents of the inner fRecordMemberFields, too. Effectively, this record will contain all the @@ -75,6 +82,10 @@ class RSoAField : public RRuleField { std::vector> fRecordMemberDeleters; ROOT::Internal::RColumnIndex fNWritten; + /// Contains the I/O customization rules for fSoAClass and all nested SoA classes and base classes, + /// and the relative offset at which the rule needs to be applied. + std::vector fRules; + /// For reading and writing, the RVecs of the SoA class do not have a dedicated field. The in-memory RVecs of the /// SoA object are used directly with the subfields of the underlying record type. For splitting a SoA class object /// (SplitValue()), however, we need actual RRVecFields so that we can recursively split the in-memory SoA value. @@ -90,7 +101,7 @@ class RSoAField : public RRuleField { RSoAField(std::string_view fieldName, TClass *clSoA); /// Called during construction, picks up the (nested) member fields of the underlying record type(s) and its - /// base classes. + /// base classes. Also fills fRules. void CollectRecordMemberFields(); /// For a nested SoA struct (either as a member of as a base class), use their fRecordMemberFields in this class, /// i.e. "unroll" the vectors in the nested SoA struct into the SoA base class. @@ -114,6 +125,7 @@ protected: void CommitClusterImpl() final { fNWritten = 0; } + std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; TClass *GetInMemoryClass() const final { return fSoAClass; } diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index 548698dc3fa8d..3f1f64ec0f063 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -353,20 +353,21 @@ std::vector ROOT::RRuleField::FindRules(const ROOT::R return rules; } -void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule) +void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset) { auto func = rule->GetReadFunctionPointer(); if (func == nullptr) { // Can happen for rename rules return; } - fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { - TVirtualObject onfileObj{nullptr}; - onfileObj.fClass = stagingClass; - onfileObj.fObject = stagingArea; - func(static_cast(target), &onfileObj); - onfileObj.fObject = nullptr; // TVirtualObject does not own the value - }); + fReadCallbacks.emplace_back( + [func, subObjectOffset, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { + TVirtualObject onfileObj{nullptr}; + onfileObj.fClass = stagingClass; + onfileObj.fObject = stagingArea; + func(static_cast(target) + subObjectOffset, &onfileObj); + onfileObj.fObject = nullptr; // TVirtualObject does not own the value + }); } //------------------------------------------------------------------------------ @@ -817,6 +818,11 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + for (const RRule &r : soaBaseField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += base->GetDelta(); + } + baseIdx++; } @@ -852,6 +858,11 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() GraftNestedMemberFields(*soaField, dataMember->GetOffset(), [&](const std::string &name) { return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + + for (const RRule &r : soaField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += dataMember->GetOffset(); + } } else if (auto vecField = dynamic_cast(dmField.get())) { if (vecField->begin()->GetTypeName() != underlyingField->GetTypeName() || vecField->begin()->GetTypeAlias() != underlyingField->GetTypeAlias()) { @@ -877,6 +888,10 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() if (nDirectRecordSubfields != nMembers) { throw RException(R__FAIL("missing SoA members")); } + + auto schemaRules = FindRules(nullptr); + for (const auto &r : schemaRules) + fRules.emplace_back(RRule{r, 0}); } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) @@ -1022,9 +1037,42 @@ void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalInde } } +std::unique_ptr +ROOT::Experimental::RSoAField::BeforeConnectPageSource(ROOT::Internal::RPageSource & /*pageSource*/) +{ + // Most of the heavy lifting is done by the actual subfields that map onto the on-disk AoS schema. + // In the SoA field itself, we only need to take care of the in-memory part of the rule processing. + // One complication is that for non-rename rules, we manually need to recurse into base (SoA) classes and + // nested (SoA) classes because we don't have them in the field tree. + + // For now, allow only rename rules and whole-object rules + + const bool hasSources = std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { + return r.fRule->GetSource() && (r.fRule->GetSource()->GetEntries() > 0); + }); + const bool hasTargets = + std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { return r.fRule->GetTarget(); }); + + if (hasSources) { + throw RException(R__FAIL("I/O customization rules with sources are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + if (hasTargets) { + throw RException(R__FAIL("I/O customization rules with targets are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + for (const auto &r : fRules) { + AddReadCallbacksFromIORule(r.fRule, r.fOffset); + } + + return nullptr; +} + void ROOT::Experimental::RSoAField::ReconcileOnDiskField(const RNTupleDescriptor &desc) { - EnsureMatchingOnDiskField(desc, kDiffTypeVersion).ThrowOnError(); + EnsureMatchingOnDiskField(desc, kDiffTypeName | kDiffTypeVersion).ThrowOnError(); } void ROOT::Experimental::RSoAField::ConstructValue(void *where) const diff --git a/tree/ntuple/test/SoAField.hxx b/tree/ntuple/test/SoAField.hxx index e74192aa84c88..2da64e1612258 100644 --- a/tree/ntuple/test/SoAField.hxx +++ b/tree/ntuple/test/SoAField.hxx @@ -170,4 +170,103 @@ struct SoADerivedFail2 : public SoABase { ClassDefNV(SoADerivedFail2, 2); }; +struct RecordBaseOld { + float fBase; + ClassDefNV(RecordBaseOld, 2); +}; + +struct SoABaseOld { + ROOT::RVec fBase; + ClassDefNV(SoABaseOld, 2); +}; + +struct RecordIntermediateOld : public RecordBaseOld { + float fIntermediate; + ClassDefNV(RecordIntermediateOld, 2); +}; + +struct SoAIntermediateOld : public SoABaseOld { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateOld, 2); +}; + +struct RecordLeafOld : public RecordIntermediateOld { + float fLeaf; + ClassDefNV(RecordLeafOld, 2); +}; + +struct SoALeafOld : public SoAIntermediateOld { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafOld, 2); +}; + +struct RecordBaseNew { + float fBase; + float fNew; + ClassDefNV(RecordBaseNew, 2); +}; + +struct SoABaseNew { + ROOT::RVec fBase; + ROOT::RVec fNew; + ClassDefNV(SoABaseNew, 2); +}; + +struct RecordIntermediateNew : public RecordBaseNew { + float fIntermediate; + ClassDefNV(RecordIntermediateNew, 2); +}; + +struct SoAIntermediateNew : public SoABaseNew { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateNew, 2); +}; + +struct RecordLeafNew : public RecordIntermediateNew { + float fLeaf; + ClassDefNV(RecordLeafNew, 2); +}; + +struct SoALeafNew : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafNew, 2); +}; + +struct RecordNested { + float fInner; + ClassDefNV(RecordNested, 2); +}; + +struct SoANested { + ROOT::RVec fInner; + ClassDefNV(SoANested, 2); +}; + +struct RecordOuter { + float fOuter; + RecordNested fNested; + ClassDefNV(RecordOuter, 2); +}; + +struct SoAOuter { + ROOT::RVec fOuter; + SoANested fNested; + ClassDefNV(SoAOuter, 2); +}; + +struct RecordRuleFail { + float fX; + ClassDefNV(RecordRuleFail, 2); +}; + +struct SoARuleFailSource { + ROOT::RVec fX; + ClassDefNV(SoARuleFailSource, 2); +}; + +struct SoARuleFailTarget { + ROOT::RVec fX; + ClassDefNV(SoARuleFailTarget, 2); +}; + #endif // ROOT_RNTuple_Test_SoAField diff --git a/tree/ntuple/test/SoAFieldLinkDef.h b/tree/ntuple/test/SoAFieldLinkDef.h index 0af69cec4189a..34b45d1f036ff 100644 --- a/tree/ntuple/test/SoAFieldLinkDef.h +++ b/tree/ntuple/test/SoAFieldLinkDef.h @@ -34,4 +34,43 @@ #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail1+; #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail2+; +#pragma link C++ class RecordBaseOld+; +#pragma link C++ class RecordIntermediateOld+; +#pragma link C++ class RecordLeafOld+; +#pragma link C++ options=rntupleSoARecord(RecordBaseOld) class SoABaseOld+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateOld) class SoAIntermediateOld+; +#pragma link C++ options=rntupleSoARecord(RecordLeafOld) class SoALeafOld+; +#pragma link C++ class RecordBaseNew+; +#pragma link C++ class RecordIntermediateNew+; +#pragma link C++ class RecordLeafNew+; +#pragma link C++ options=rntupleSoARecord(RecordBaseNew) class SoABaseNew+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateNew) class SoAIntermediateNew+; +#pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNew+; + +#pragma read sourceClass = "RecordBaseOld" targetClass = "RecordBaseNew" version = "[1-]"; +#pragma read sourceClass = "RecordIntermediateOld" targetClass = "RecordIntermediateNew" version = "[1-]"; +#pragma read sourceClass = "RecordLeafOld" targetClass = "RecordLeafNew" version = "[1-]"; +#pragma read sourceClass = "SoABaseOld" targetClass = "SoABaseNew" version = "[1-]"; +#pragma read sourceClass = "SoAIntermediateOld" targetClass = "SoAIntermediateNew" version = "[1-]"; +#pragma read sourceClass = "SoALeafOld" targetClass = "SoALeafNew" version = "[1-]"; + +#pragma link C++ class RecordNested+; +#pragma link C++ class RecordOuter+; +#pragma link C++ options=rntupleSoARecord(RecordNested) class SoANested+; +#pragma link C++ options=rntupleSoARecord(RecordOuter) class SoAOuter+; + +#pragma read sourceClass="SoANested" version="[1-]" targetClass="SoANested" source="" target="" \ + code="{ newObj->fInner *= 2.; }" +#pragma read sourceClass="SoAOuter" version="[1-]" targetClass="SoAOuter" source="" target="" \ + code="{ newObj->fOuter *= 4.; }" + +#pragma link C++ class RecordRuleFail+; +#pragma link C++ options=rntupleSoARecord(RecordRuleFail) class SoARuleFailSource+; +#pragma link C++ options=rntupleSoARecord(RecordRuleFail) class SoARuleFailTarget+; + +#pragma read sourceClass="SoARuleFailSource" version="[1-]" targetClass="SoARuleFailSource" \ + source="ROOT::RVec fX" target="" code="{ }"; +#pragma read sourceClass="SoARuleFailTarget" version="[1-]" targetClass="SoARuleFailTarget" \ + source="" target="fX" code="{ (void)fX; }"; + #endif // __CLING__ diff --git a/tree/ntuple/test/ntuple_soa.cxx b/tree/ntuple/test/ntuple_soa.cxx index 62b7bf42b4656..02076e2e6e2c0 100644 --- a/tree/ntuple/test/ntuple_soa.cxx +++ b/tree/ntuple/test/ntuple_soa.cxx @@ -385,6 +385,7 @@ TEST(RNTuple, SoAFromVector) auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); v->emplace_back(RecordSimple{1.0, 2.0}); + v->emplace_back(RecordSimple{3.0, 4.0}); writer->Fill(); } @@ -392,14 +393,16 @@ TEST(RNTuple, SoAFromVector) auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); SoASimple soa; - // Until SoA schema evolution is implemented, the reading the vector as SoA will - try { - reader->GetView("simple", &soa, "SoASimple"); - FAIL() << "reading a vector with a SoA field should fail"; - } catch (const ROOT::RException &e) { - EXPECT_THAT(e.what(), testing::HasSubstr( - "in-memory field simple of type SoASimple is incompatible with on-disk field simple")); - } + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "simple": [{"fX": 1, "fY": 2}, {"fX": 3, "fY": 4}] +} +)"}; + // clang-format on + EXPECT_EQ(expected, os.str()); } TEST(RNTuple, SoAShow) @@ -576,3 +579,110 @@ R"({ // clang-format on EXPECT_EQ(expected, os.str()); } + +TEST(RNTuple, SoARename) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_rename.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("leaf", "SoALeafOld")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto leafSoA = writer->GetModel().GetDefaultEntry().GetPtr("leaf"); + leafSoA->fBase = {1.0, 2.0}; + leafSoA->fIntermediate = {3.0, 4.0}; + leafSoA->fLeaf = {5.0, 6.0}; + + writer->Fill(); + } + + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafNew")); + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + + // We cannot use "Show()" because that will reconstruct the original model as a display model, it will not + // use the imposed model. + + auto leafSoA = reader->GetModel().GetDefaultEntry().GetPtr("leaf"); + reader->LoadEntry(0); + + EXPECT_EQ(2u, leafSoA->fBase.size()); + EXPECT_EQ(2u, leafSoA->fNew.size()); + EXPECT_EQ(2u, leafSoA->fIntermediate.size()); + EXPECT_EQ(2u, leafSoA->fLeaf.size()); + EXPECT_FLOAT_EQ(1.0, leafSoA->fBase[0]); + EXPECT_FLOAT_EQ(2.0, leafSoA->fBase[1]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[0]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[1]); + EXPECT_FLOAT_EQ(3.0, leafSoA->fIntermediate[0]); + EXPECT_FLOAT_EQ(4.0, leafSoA->fIntermediate[1]); + EXPECT_FLOAT_EQ(5.0, leafSoA->fLeaf[0]); + EXPECT_FLOAT_EQ(6.0, leafSoA->fLeaf[1]); +} + +TEST(RNTuple, SoARuleFailures) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_whole_object_rule.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("source", "SoARuleFailSource")); + model->AddField(std::make_unique("target", "SoARuleFailSource")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + + SoARuleFailSource source; + try { + auto viewSoASource = reader->GetView("source", &source, "SoARuleFailSource"); + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("rules with sources are currently unsupported for SoA fields")); + } + + SoARuleFailTarget target; + try { + auto viewSoASource = reader->GetView("target", &target, "SoARuleFailTarget"); + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("rules with targets are currently unsupported for SoA fields")); + } +} + +TEST(RNTuple, SoAWholeObjectRule) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_whole_object_rule.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("outer", "SoAOuter")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto outerSoA = writer->GetModel().GetDefaultEntry().GetPtr("outer"); + outerSoA->fNested.fInner = {1.0, 2.0}; + outerSoA->fOuter = {3.0, 4.0}; + + writer->Fill(); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "outer": { + "fOuter": [12, 16], + "fNested": { + "fInner": [2, 4] + } + } +} +)" }; + // clang-format on + EXPECT_EQ(expected, os.str()); +} From 8eb85334d3b99bc6859ceb602afa086b397be9b3 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 6 Oct 2026 09:40:43 +0200 Subject: [PATCH 4/6] [NFC][ntuple] minor comment fix --- tree/ntuple/src/RFieldMeta.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index 3f1f64ec0f063..ace086a67446a 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -308,7 +308,7 @@ std::vector ROOT::RRuleField::FindRules(const ROOT::R } // Cleanup and sort rules - // Check that any any given source member uses the same type in all rules + // Check that any given source member uses the same type in all rules std::unordered_map sourceNameAndType; std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector for (auto itr = rules.begin(); itr != rules.end() - nskip;) { From 220233def9b56ba318e9374aaec5ece7fc6a542e Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 6 Oct 2026 11:25:32 +0200 Subject: [PATCH 5/6] [ntuple] store protected fClass pointer in RRuleField ... and remove the GetInMemoryClass() virtual method. --- tree/ntuple/inc/ROOT/RField.hxx | 11 ++-- tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx | 9 ++-- tree/ntuple/src/RFieldMeta.cxx | 62 +++++++++++------------ 3 files changed, 36 insertions(+), 46 deletions(-) diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 889220f2dc2ca..e211610d3be7b 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -144,6 +144,9 @@ protected: /// Prefix used in the subfield names generated for base classes static constexpr const char *kPrefixInherited{":"}; + /// Set by derived class to the TClass instance representing the current in-memory layout. + TClass *fClass = nullptr; + /// The staging area stores inputs to I/O rules according to the offsets given by the streamer info of /// "TypeName@@Version". The area is allocated depending on I/O rules resp. the source members of the I/O rules. std::unique_ptr fStagingArea; @@ -154,10 +157,7 @@ protected: TClass *fStagingClass = nullptr; std::unordered_map fStagingItems; ///< Lookup staging items by member name - RRuleField(std::string_view name, std::string_view type, ROOT::ENTupleStructure structure); - - /// Derived classes should return the TClass instance representing the current in-memory layout. - virtual TClass *GetInMemoryClass() const = 0; + RRuleField(std::string_view name, std::string_view type, TClass *classp, ROOT::ENTupleStructure structure); /// Returns the id of member 'name' in the class field given by 'fieldId', or kInvalidDescriptorId if no such /// member exist. Looks recursively in base classes. @@ -201,7 +201,6 @@ private: void operator()(void *objPtr, bool dtorOnly) final; }; - TClass *fClass; /// Additional information kept for each entry in `fSubfields` std::vector fSubfieldsInfo; @@ -223,8 +222,6 @@ protected: std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; - TClass *GetInMemoryClass() const final { return fClass; } - public: RClassField(std::string_view fieldName, std::string_view className); RClassField(RClassField &&other) = default; diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index bc001198d4980..ddc844cc17049 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -70,7 +70,6 @@ class RSoAField : public RRuleField { std::size_t fOffset = 0; }; - TClass *fSoAClass = nullptr; /// Direct access to the member fields of the underlying record. In case of a nested SoA type, this vector /// contains the contents of the inner fRecordMemberFields, too. Effectively, this record will contain all the /// fields of the underlying record type that correspond to terminal RVec members in a nested SoA type. @@ -82,7 +81,7 @@ class RSoAField : public RRuleField { std::vector> fRecordMemberDeleters; ROOT::Internal::RColumnIndex fNWritten; - /// Contains the I/O customization rules for fSoAClass and all nested SoA classes and base classes, + /// Contains the I/O customization rules for fClass and all nested SoA classes and base classes, /// and the relative offset at which the rule needs to be applied. std::vector fRules; @@ -118,7 +117,7 @@ protected: void GenerateColumns(const ROOT::RNTupleDescriptor &desc) final; void ConstructValue(void *where) const final; - std::unique_ptr GetDeleter() const final { return std::make_unique(fSoAClass); } + std::unique_ptr GetDeleter() const final { return std::make_unique(fClass); } std::size_t AppendImpl(const void *from) final; void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final; @@ -128,8 +127,6 @@ protected: std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; - TClass *GetInMemoryClass() const final { return fSoAClass; } - public: RSoAField(std::string_view fieldName, std::string_view className); RSoAField(RSoAField &&other) = default; @@ -147,7 +144,7 @@ public: const std::type_info *GetPolymorphicTypeInfo() const; void AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const final; - TClass *GetSoAClass() const { return fSoAClass; } + TClass *GetSoAClass() const { return fClass; } }; } // namespace Experimental diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index ace086a67446a..dd1d70955a31d 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -191,8 +191,9 @@ std::string BuildMapTypeName(ROOT::RMapField::EMapType mapType, const ROOT::RFie } // anonymous namespace -ROOT::RRuleField::RRuleField(std::string_view fieldName, std::string_view typeName, ROOT::ENTupleStructure structure) - : RFieldBase(fieldName, typeName, structure, false /* isSimple */) +ROOT::RRuleField::RRuleField(std::string_view fieldName, std::string_view typeName, TClass *classp, + ROOT::ENTupleStructure structure) + : RFieldBase(fieldName, typeName, structure, false /* isSimple */), fClass(classp) { } @@ -225,7 +226,7 @@ void ROOT::RRuleField::SetStagingClass(const std::string &className, unsigned in fStagingClass = TClass::GetClass(className.c_str()); } } else { - fStagingClass = GetInMemoryClass(); + fStagingClass = fClass; } R__ASSERT(fStagingClass); R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); @@ -284,16 +285,15 @@ void ROOT::RRuleField::PrepareStagingArea(const std::vector std::vector ROOT::RRuleField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) const { ROOT::Detail::TSchemaRuleSet::TMatches rules; - auto cl = GetInMemoryClass(); - const auto ruleset = cl->GetSchemaRules(); + const auto ruleset = fClass->GetSchemaRules(); if (!ruleset) return rules; if (!fieldDesc) { // If we have no on-disk information for the field, we still process the rules on the current in-memory version // of the class - rules = ruleset->FindRules(cl->GetName(), cl->GetClassVersion(), cl->GetCheckSum()); + rules = ruleset->FindRules(fClass->GetName(), fClass->GetClassVersion(), fClass->GetCheckSum()); } else { // We need to change (back) the name normalization from RNTuple to ROOT Meta std::string normalizedName; @@ -373,8 +373,7 @@ void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule, std:: //------------------------------------------------------------------------------ ROOT::RClassField::RClassField(std::string_view fieldName, const RClassField &source) - : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord), - fClass(source.fClass), + : ROOT::RRuleField(fieldName, source.GetTypeName(), source.fClass, ROOT::ENTupleStructure::kRecord), fSubfieldsInfo(source.fSubfieldsInfo) { for (const auto &f : source.GetConstSubfields()) { @@ -389,8 +388,7 @@ ROOT::RClassField::RClassField(std::string_view fieldName, std::string_view clas } ROOT::RClassField::RClassField(std::string_view fieldName, TClass *classp) - : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord), - fClass(classp) + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(classp->GetName()), classp, ROOT::ENTupleStructure::kRecord) { EnsureValidUserClass(fClass, *this, "RClassField"); @@ -691,8 +689,7 @@ void ROOT::RClassField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) cons //------------------------------------------------------------------------------ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, const RSoAField &source) - : ROOT::RRuleField(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection), - fSoAClass(source.fSoAClass), + : ROOT::RRuleField(fieldName, source.GetTypeName(), source.fClass, ROOT::ENTupleStructure::kCollection), fSoAMemberOffsets(source.fSoAMemberOffsets) { fTraits = source.GetTraits(); @@ -782,13 +779,13 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } // Base classes are treated as unrolled nested SoA classes - const auto *soaBases = fSoAClass->GetListOfBases(); + const auto *soaBases = fClass->GetListOfBases(); if (soaBases->GetSize() != static_cast(nDirectRecordBases)) { throw RException(R__FAIL(std::string("number of base classes don't match between SoA class ") + GetFieldName() + " and its underlying record type")); } unsigned int baseIdx = 0; - for (auto base : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfBases())) { + for (auto base : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfBases())) { if (base->GetDelta() < 0) { throw RException(R__FAIL(std::string("virtual inheritance is not supported: ") + GetTypeName() + " virtually inherits from " + base->GetName())); @@ -827,7 +824,7 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } unsigned int nMembers = 0; - for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfDataMembers())) { + for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfDataMembers())) { // NOTE: ReconstructSplitFields() will also traverse the data members and need to apply the same rules for // skipping members @@ -849,7 +846,7 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() assert(dmField->GetFieldName() == underlyingField->GetFieldName()); if (auto soaField = dynamic_cast(dmField.get())) { - if (ROOT::Internal::GetRNTupleSoARecord(soaField->fSoAClass) != underlyingField->GetTypeName()) { + if (ROOT::Internal::GetRNTupleSoARecord(soaField->fClass) != underlyingField->GetTypeName()) { throw RException(R__FAIL(std::string("nested SoA field ") + soaField->GetQualifiedFieldName() + " [" + soaField->GetTypeName() + "] does not match underlying type " + underlyingField->GetTypeName())); @@ -895,16 +892,15 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) - : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection), - fSoAClass(clSoA) + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(clSoA->GetName()), clSoA, ROOT::ENTupleStructure::kCollection) { static std::once_flag once; std::call_once(once, []() { R__LOG_WARNING(ROOT::Internal::NTupleLog()) << "The SoA field is experimental and still under development."; }); - EnsureValidUserClass(fSoAClass, *this, "RSoAField"); - const auto recordTypeName = ROOT::Internal::GetRNTupleSoARecord(fSoAClass); + EnsureValidUserClass(fClass, *this, "RSoAField"); + const auto recordTypeName = ROOT::Internal::GetRNTupleSoARecord(fClass); if (recordTypeName.empty()) { throw ROOT::RException(R__FAIL(std::string("class ") + GetTypeName() + " is not marked with the rntupleSoARecord " @@ -915,17 +911,17 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clS } catch (ROOT::RException &e) { throw RException(R__FAIL("invalid record type of SoA field " + GetTypeName() + " [" + e.what() + "]")); } - R__ASSERT(fSoAClass->GetClassVersion() >= 0); - if (static_cast(fSoAClass->GetClassVersion()) != fSubfields[0]->GetTypeVersion()) { + R__ASSERT(fClass->GetClassVersion() >= 0); + if (static_cast(fClass->GetClassVersion()) != fSubfields[0]->GetTypeVersion()) { throw RException(R__FAIL(std::string("version mismatch between SoA type and underlying record type: ") + - std::to_string(fSoAClass->GetClassVersion()) + " vs. " + + std::to_string(fClass->GetClassVersion()) + " vs. " + std::to_string(fSubfields[0]->GetTypeVersion()))); } CollectRecordMemberFields(); std::string renormalizedAlias; - if (ROOT::Internal::NeedsMetaNameAsAlias(fSoAClass->GetName(), renormalizedAlias)) + if (ROOT::Internal::NeedsMetaNameAsAlias(fClass->GetName(), renormalizedAlias)) fTypeAlias = renormalizedAlias; fTraits |= kTraitSoACollection | kTraitTypeChecksum; @@ -1077,7 +1073,7 @@ void ROOT::Experimental::RSoAField::ReconcileOnDiskField(const RNTupleDescriptor void ROOT::Experimental::RSoAField::ConstructValue(void *where) const { - fSoAClass->New(where); + fClass->New(where); } ROOT::Experimental::RSoAField::RSoADeleter::RSoADeleter(TClass *cl) : RDeleter(cl->GetClassAlignment()), fSoAClass(cl) @@ -1100,7 +1096,7 @@ void ROOT::Experimental::RSoAField::ReconstructSplitFields() const fSplitOffsets = std::make_unique>(); unsigned int baseIdx = 0; - for (auto base : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfBases())) { + for (auto base : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfBases())) { TClass *cl = base->GetClassPointer(); auto baseField = RFieldBase::Create(std::string(":_" + std::to_string(baseIdx)), cl->GetName()).Unwrap(); fSplitFields->emplace_back(std::move(baseField)); @@ -1108,7 +1104,7 @@ void ROOT::Experimental::RSoAField::ReconstructSplitFields() const baseIdx++; } - for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfDataMembers())) { + for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfDataMembers())) { if ((dataMember->Property() & kIsStatic) || !dataMember->IsPersistent()) continue; @@ -1136,22 +1132,22 @@ std::vector ROOT::Experimental::RSoAField::SplitValue( std::size_t ROOT::Experimental::RSoAField::GetValueSize() const { - return fSoAClass->GetClassSize(); + return fClass->GetClassSize(); } std::uint32_t ROOT::Experimental::RSoAField::GetTypeVersion() const { - return fSoAClass->GetClassVersion(); + return fClass->GetClassVersion(); } std::uint32_t ROOT::Experimental::RSoAField::GetTypeChecksum() const { - return fSoAClass->GetCheckSum(); + return fClass->GetCheckSum(); } std::size_t ROOT::Experimental::RSoAField::GetAlignment() const { - const auto align = fSoAClass->GetClassAlignment(); + const auto align = fClass->GetClassAlignment(); EnsureValidAlignment(align); return align; } @@ -1159,11 +1155,11 @@ std::size_t ROOT::Experimental::RSoAField::GetAlignment() const const std::type_info *ROOT::Experimental::RSoAField::GetPolymorphicTypeInfo() const { // TODO(jblomer): factor out - bool polymorphic = fSoAClass->ClassProperty() & kClassHasVirtual; + bool polymorphic = fClass->ClassProperty() & kClassHasVirtual; if (!polymorphic) { return nullptr; } - return fSoAClass->GetTypeInfo(); + return fClass->GetTypeInfo(); } void ROOT::Experimental::RSoAField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const From 60be7b4e4d09346edcd922477ae04582aa5f1247 Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Thu, 8 Oct 2026 00:10:43 +0200 Subject: [PATCH 6/6] [ntuple] add test for SoA 'evolution' without rename rule --- tree/ntuple/test/SoAField.hxx | 15 ++++++++ tree/ntuple/test/SoAFieldLinkDef.h | 3 ++ tree/ntuple/test/ntuple_soa.cxx | 55 ++++++++++++++++++------------ 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/tree/ntuple/test/SoAField.hxx b/tree/ntuple/test/SoAField.hxx index 2da64e1612258..89704b2765da0 100644 --- a/tree/ntuple/test/SoAField.hxx +++ b/tree/ntuple/test/SoAField.hxx @@ -227,11 +227,26 @@ struct RecordLeafNew : public RecordIntermediateNew { ClassDefNV(RecordLeafNew, 2); }; +struct RecordLeafLikeNew : public RecordIntermediateNew { + float fLeaf; + ClassDefNV(RecordLeafLikeNew, 2); +}; + struct SoALeafNew : public SoAIntermediateNew { ROOT::RVec fLeaf; ClassDefNV(SoALeafNew, 2); }; +struct SoALeafNewNoRenameRule : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafNewNoRenameRule, 2); +}; + +struct SoALeafLikeNew : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafLikeNew, 2); +}; + struct RecordNested { float fInner; ClassDefNV(RecordNested, 2); diff --git a/tree/ntuple/test/SoAFieldLinkDef.h b/tree/ntuple/test/SoAFieldLinkDef.h index 34b45d1f036ff..44e0ffe512c63 100644 --- a/tree/ntuple/test/SoAFieldLinkDef.h +++ b/tree/ntuple/test/SoAFieldLinkDef.h @@ -43,9 +43,12 @@ #pragma link C++ class RecordBaseNew+; #pragma link C++ class RecordIntermediateNew+; #pragma link C++ class RecordLeafNew+; +#pragma link C++ class RecordLeafLikeNew+; #pragma link C++ options=rntupleSoARecord(RecordBaseNew) class SoABaseNew+; #pragma link C++ options=rntupleSoARecord(RecordIntermediateNew) class SoAIntermediateNew+; #pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNew+; +#pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNewNoRenameRule+; +#pragma link C++ options=rntupleSoARecord(RecordLeafLikeNew) class SoALeafLikeNew+; #pragma read sourceClass = "RecordBaseOld" targetClass = "RecordBaseNew" version = "[1-]"; #pragma read sourceClass = "RecordIntermediateOld" targetClass = "RecordIntermediateNew" version = "[1-]"; diff --git a/tree/ntuple/test/ntuple_soa.cxx b/tree/ntuple/test/ntuple_soa.cxx index 02076e2e6e2c0..6bf4f198199f8 100644 --- a/tree/ntuple/test/ntuple_soa.cxx +++ b/tree/ntuple/test/ntuple_soa.cxx @@ -598,28 +598,41 @@ TEST(RNTuple, SoARename) writer->Fill(); } - auto model = ROOT::RNTupleModel::Create(); - model->AddField(std::make_unique("leaf", "SoALeafNew")); - auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); - - // We cannot use "Show()" because that will reconstruct the original model as a display model, it will not - // use the imposed model. - - auto leafSoA = reader->GetModel().GetDefaultEntry().GetPtr("leaf"); - reader->LoadEntry(0); + { + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafNew")); + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + + // We cannot use "Show()" because that will reconstruct the original model as a display model, it will not + // use the imposed model. + + auto leafSoA = reader->GetModel().GetDefaultEntry().GetPtr("leaf"); + reader->LoadEntry(0); + + EXPECT_EQ(2u, leafSoA->fBase.size()); + EXPECT_EQ(2u, leafSoA->fNew.size()); + EXPECT_EQ(2u, leafSoA->fIntermediate.size()); + EXPECT_EQ(2u, leafSoA->fLeaf.size()); + EXPECT_FLOAT_EQ(1.0, leafSoA->fBase[0]); + EXPECT_FLOAT_EQ(2.0, leafSoA->fBase[1]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[0]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[1]); + EXPECT_FLOAT_EQ(3.0, leafSoA->fIntermediate[0]); + EXPECT_FLOAT_EQ(4.0, leafSoA->fIntermediate[1]); + EXPECT_FLOAT_EQ(5.0, leafSoA->fLeaf[0]); + EXPECT_FLOAT_EQ(6.0, leafSoA->fLeaf[1]); + } - EXPECT_EQ(2u, leafSoA->fBase.size()); - EXPECT_EQ(2u, leafSoA->fNew.size()); - EXPECT_EQ(2u, leafSoA->fIntermediate.size()); - EXPECT_EQ(2u, leafSoA->fLeaf.size()); - EXPECT_FLOAT_EQ(1.0, leafSoA->fBase[0]); - EXPECT_FLOAT_EQ(2.0, leafSoA->fBase[1]); - EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[0]); - EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[1]); - EXPECT_FLOAT_EQ(3.0, leafSoA->fIntermediate[0]); - EXPECT_FLOAT_EQ(4.0, leafSoA->fIntermediate[1]); - EXPECT_FLOAT_EQ(5.0, leafSoA->fLeaf[0]); - EXPECT_FLOAT_EQ(6.0, leafSoA->fLeaf[1]); + { + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafLikeNew")); + try { + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + FAIL() << "mismatch in underlying record field should fail"; + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("RecordLeafLikeNew vs. RecordLeafOld")); + } + } } TEST(RNTuple, SoARuleFailures)