diff --git a/tree/ntuple/inc/ROOT/RField.hxx b/tree/ntuple/inc/ROOT/RField.hxx index 1429a8e3f0b5c..e211610d3be7b 100644 --- a/tree/ntuple/inc/ROOT/RField.hxx +++ b/tree/ntuple/inc/ROOT/RField.hxx @@ -131,17 +131,9 @@ public: size_t GetAlignment() const final { return 0; } }; // RInvalidField -/// The field for a class with dictionary -class RClassField : public RFieldBase { -private: - enum ESubfieldRole { - kBaseClass, - kDataMember, - }; - struct RSubfieldInfo { - ESubfieldRole fRole; - std::size_t fOffset; - }; +/// Base class for fields that are subject to I/O customization rules. Used by the class field and the SoA field. +class RRuleField : public RFieldBase { +protected: // Information to read into the staging area a field that is used as an input to an I/O customization rule struct RStagingItem { /// The field used to read the on-disk data. The fields type may be different from the on-disk type as long @@ -152,18 +144,8 @@ private: /// Prefix used in the subfield names generated for base classes static constexpr const char *kPrefixInherited{":"}; - class RClassDeleter : public RDeleter { - private: - TClass *fClass; - - public: - explicit RClassDeleter(TClass *cl); - void operator()(void *objPtr, bool dtorOnly) final; - }; - - TClass *fClass; - /// Additional information kept for each entry in `fSubfields` - std::vector fSubfieldsInfo; + /// Set by derived class to the TClass instance representing the current in-memory layout. + TClass *fClass = nullptr; /// The staging area stores inputs to I/O rules according to the offsets given by the streamer info of /// "TypeName@@Version". The area is allocated depending on I/O rules resp. the source members of the I/O rules. @@ -175,15 +157,12 @@ private: TClass *fStagingClass = nullptr; std::unordered_map fStagingItems; ///< Lookup staging items by member name -private: - RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl - RClassField(std::string_view fieldName, TClass *classp); - void Attach(std::unique_ptr child, RSubfieldInfo info); + RRuleField(std::string_view name, std::string_view type, TClass *classp, ROOT::ENTupleStructure structure); /// Returns the id of member 'name' in the class field given by 'fieldId', or kInvalidDescriptorId if no such /// member exist. Looks recursively in base classes. - ROOT::DescriptorId_t - LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, ROOT::DescriptorId_t classFieldId); + ROOT::DescriptorId_t LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const; /// Sets fStagingClass according to the given name and version void SetStagingClass(const std::string &className, unsigned int classVersion); /// If there are rules with inputs (source members), create the staging area according to the TClass instance @@ -191,12 +170,44 @@ private: void PrepareStagingArea(const std::vector &rules, const ROOT::RNTupleDescriptor &desc, const ROOT::RFieldDescriptor &classFieldId); /// Register post-read callback corresponding to a ROOT I/O customization rules. - void AddReadCallbacksFromIORule(const TSchemaRule *rule); + /// The sub object offset allows to apply the rule to a nested object within the passed target. + /// This is used to execute rules on base classes and nested classes in a SoA field. + void AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset = 0); /// Given the on-disk information from the page source, find all the I/O customization rules that apply /// to the class field at hand, to which the fieldDesc descriptor, if provided, must correspond. /// Fields may not have an on-disk representation (e.g., when inserted by schema evolution), in which case the passed /// field descriptor is nullptr. - std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc); + std::vector FindRules(const ROOT::RFieldDescriptor *fieldDesc) const; +}; + +/// The field for a class with dictionary +class RClassField : public RRuleField { +private: + enum ESubfieldRole { + kBaseClass, + kDataMember, + }; + struct RSubfieldInfo { + ESubfieldRole fRole; + std::size_t fOffset; + }; + + class RClassDeleter : public RDeleter { + private: + TClass *fClass; + + public: + explicit RClassDeleter(TClass *cl); + void operator()(void *objPtr, bool dtorOnly) final; + }; + + /// Additional information kept for each entry in `fSubfields` + std::vector fSubfieldsInfo; + +private: + RClassField(std::string_view fieldName, const RClassField &source); ///< Used by CloneImpl + RClassField(std::string_view fieldName, TClass *classp); + void Attach(std::unique_ptr child, RSubfieldInfo info); protected: std::unique_ptr CloneImpl(std::string_view newName) const final; diff --git a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx index af503964a865c..ddc844cc17049 100644 --- a/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx +++ b/tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx @@ -53,7 +53,7 @@ namespace Experimental { /// /// Since the on-disk representation is a collection of record type, the class version and checksum of the SoA type /// itself is ignored. -class RSoAField : public RFieldBase { +class RSoAField : public RRuleField { class RSoADeleter : public RDeleter { private: TClass *fSoAClass; @@ -63,7 +63,13 @@ class RSoAField : public RFieldBase { void operator()(void *objPtr, bool dtorOnly) final; }; - TClass *fSoAClass = nullptr; + // A rule of the SoA type itself or one of its nested SoA types or base classes, togther with the offset + // of the nested object in the SoA type. + struct RRule { + const TSchemaRule *fRule = nullptr; + std::size_t fOffset = 0; + }; + /// Direct access to the member fields of the underlying record. In case of a nested SoA type, this vector /// contains the contents of the inner fRecordMemberFields, too. Effectively, this record will contain all the /// fields of the underlying record type that correspond to terminal RVec members in a nested SoA type. @@ -75,6 +81,10 @@ class RSoAField : public RFieldBase { std::vector> fRecordMemberDeleters; ROOT::Internal::RColumnIndex fNWritten; + /// Contains the I/O customization rules for fClass and all nested SoA classes and base classes, + /// and the relative offset at which the rule needs to be applied. + std::vector fRules; + /// For reading and writing, the RVecs of the SoA class do not have a dedicated field. The in-memory RVecs of the /// SoA object are used directly with the subfields of the underlying record type. For splitting a SoA class object /// (SplitValue()), however, we need actual RRVecFields so that we can recursively split the in-memory SoA value. @@ -90,7 +100,7 @@ class RSoAField : public RFieldBase { RSoAField(std::string_view fieldName, TClass *clSoA); /// Called during construction, picks up the (nested) member fields of the underlying record type(s) and its - /// base classes. + /// base classes. Also fills fRules. void CollectRecordMemberFields(); /// For a nested SoA struct (either as a member of as a base class), use their fRecordMemberFields in this class, /// i.e. "unroll" the vectors in the nested SoA struct into the SoA base class. @@ -107,13 +117,14 @@ protected: void GenerateColumns(const ROOT::RNTupleDescriptor &desc) final; void ConstructValue(void *where) const final; - std::unique_ptr GetDeleter() const final { return std::make_unique(fSoAClass); } + std::unique_ptr GetDeleter() const final { return std::make_unique(fClass); } std::size_t AppendImpl(const void *from) final; void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final; void CommitClusterImpl() final { fNWritten = 0; } + std::unique_ptr BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final; void ReconcileOnDiskField(const RNTupleDescriptor &desc) final; public: @@ -133,7 +144,7 @@ public: const std::type_info *GetPolymorphicTypeInfo() const; void AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const final; - TClass *GetSoAClass() const { return fSoAClass; } + TClass *GetSoAClass() const { return fClass; } }; } // namespace Experimental diff --git a/tree/ntuple/src/RFieldMeta.cxx b/tree/ntuple/src/RFieldMeta.cxx index e500b761c5dd6..dd1d70955a31d 100644 --- a/tree/ntuple/src/RFieldMeta.cxx +++ b/tree/ntuple/src/RFieldMeta.cxx @@ -191,9 +191,189 @@ std::string BuildMapTypeName(ROOT::RMapField::EMapType mapType, const ROOT::RFie } // anonymous namespace +ROOT::RRuleField::RRuleField(std::string_view fieldName, std::string_view typeName, TClass *classp, + ROOT::ENTupleStructure structure) + : RFieldBase(fieldName, typeName, structure, false /* isSimple */), fClass(classp) +{ +} + +ROOT::DescriptorId_t ROOT::RRuleField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, + ROOT::DescriptorId_t classFieldId) const +{ + auto idSourceMember = desc.FindFieldId(memberName, classFieldId); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + + for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { + const auto &subFieldName = subFieldDesc.GetFieldName(); + if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { + idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); + if (idSourceMember != ROOT::kInvalidDescriptorId) + return idSourceMember; + } + } + + return ROOT::kInvalidDescriptorId; +} + +void ROOT::RRuleField::SetStagingClass(const std::string &className, unsigned int classVersion) +{ + TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); + if (classVersion != GetTypeVersion() || className != GetTypeName()) { + fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); + if (!fStagingClass) { + // For a rename rule, we may simply ask for the old class name + fStagingClass = TClass::GetClass(className.c_str()); + } + } else { + fStagingClass = fClass; + } + R__ASSERT(fStagingClass); + R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); +} + +void ROOT::RRuleField::PrepareStagingArea(const std::vector &rules, + const ROOT::RNTupleDescriptor &desc, + const ROOT::RFieldDescriptor &classFieldDesc) +{ + std::size_t stagingAreaSize = 0; + for (const auto rule : rules) { + for (auto source : TRangeDynCast(rule->GetSource())) { + auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); + if (!isNew) { + // This source member has already been processed by another rule (and we only support one type per member) + continue; + } + RStagingItem &stagingItem = itr->second; + + const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); + if (memberFieldId == kInvalidDescriptorId) { + throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + + source->GetName())); + } + + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); + memberField->SetOnDiskId(memberFieldId); + auto fieldZero = std::make_unique(); + Internal::SetAllowFieldSubstitutions(*fieldZero, true); + fieldZero->Attach(std::move(memberField)); + stagingItem.fField = std::move(fieldZero); + + stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); + // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it + // to be present in the TClass instance, too. + R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); + stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); + } + } + + if (stagingAreaSize) { + R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules + // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. + fStagingArea = std::make_unique(stagingAreaSize); + + for (const auto &[_, si] : fStagingItems) { + const auto &memberField = *si.fField->cbegin(); + if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { + CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); + } + } + } +} + +std::vector ROOT::RRuleField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) const +{ + ROOT::Detail::TSchemaRuleSet::TMatches rules; + + const auto ruleset = fClass->GetSchemaRules(); + if (!ruleset) + return rules; + + if (!fieldDesc) { + // If we have no on-disk information for the field, we still process the rules on the current in-memory version + // of the class + rules = ruleset->FindRules(fClass->GetName(), fClass->GetClassVersion(), fClass->GetCheckSum()); + } else { + // We need to change (back) the name normalization from RNTuple to ROOT Meta + std::string normalizedName; + TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); + // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the + // on-disk version of the field. + if (fieldDesc->GetTypeChecksum()) { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); + } else { + rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); + } + } + + // Cleanup and sort rules + // Check that any given source member uses the same type in all rules + std::unordered_map sourceNameAndType; + std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector + for (auto itr = rules.begin(); itr != rules.end() - nskip;) { + const auto rule = *itr; + + // Erase unknown rule types + if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); + itr = rules.erase(itr); + continue; + } + + bool hasConflictingSourceMembers = false; + for (auto source : TRangeDynCast(rule->GetSource())) { + auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); + auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); + if (!isNew && (itrSrc->second != memberType)) { + R__LOG_WARNING(ROOT::Internal::NTupleLog()) + << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " + << memberType << " for member " << source->GetName(); + hasConflictingSourceMembers = true; + break; + } + } + if (hasConflictingSourceMembers) { + itr = rules.erase(itr); + continue; + } + + // Rules targeting the entire object need to be executed at the end + if (rule->GetTarget() == nullptr) { + nskip++; + if (itr != rules.end() - nskip) + std::iter_swap(itr++, rules.end() - nskip); + continue; + } + + ++itr; + } + + return rules; +} + +void ROOT::RRuleField::AddReadCallbacksFromIORule(const TSchemaRule *rule, std::size_t subObjectOffset) +{ + auto func = rule->GetReadFunctionPointer(); + if (func == nullptr) { + // Can happen for rename rules + return; + } + fReadCallbacks.emplace_back( + [func, subObjectOffset, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { + TVirtualObject onfileObj{nullptr}; + onfileObj.fClass = stagingClass; + onfileObj.fObject = stagingArea; + func(static_cast(target) + subObjectOffset, &onfileObj); + onfileObj.fObject = nullptr; // TVirtualObject does not own the value + }); +} + +//------------------------------------------------------------------------------ + ROOT::RClassField::RClassField(std::string_view fieldName, const RClassField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kRecord, false /* isSimple */), - fClass(source.fClass), + : ROOT::RRuleField(fieldName, source.GetTypeName(), source.fClass, ROOT::ENTupleStructure::kRecord), fSubfieldsInfo(source.fSubfieldsInfo) { for (const auto &f : source.GetConstSubfields()) { @@ -208,9 +388,7 @@ ROOT::RClassField::RClassField(std::string_view fieldName, std::string_view clas } ROOT::RClassField::RClassField(std::string_view fieldName, TClass *classp) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(classp->GetName()), ROOT::ENTupleStructure::kRecord, - false /* isSimple */), - fClass(classp) + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(classp->GetName()), classp, ROOT::ENTupleStructure::kRecord) { EnsureValidUserClass(fClass, *this, "RClassField"); @@ -291,76 +469,6 @@ void ROOT::RClassField::Attach(std::unique_ptr child, RSubfieldInfo RFieldBase::Attach(std::move(child)); } -std::vector ROOT::RClassField::FindRules(const ROOT::RFieldDescriptor *fieldDesc) -{ - ROOT::Detail::TSchemaRuleSet::TMatches rules; - const auto ruleset = fClass->GetSchemaRules(); - if (!ruleset) - return rules; - - if (!fieldDesc) { - // If we have no on-disk information for the field, we still process the rules on the current in-memory version - // of the class - rules = ruleset->FindRules(fClass->GetName(), fClass->GetClassVersion(), fClass->GetCheckSum()); - } else { - // We need to change (back) the name normalization from RNTuple to ROOT Meta - std::string normalizedName; - TClassEdit::GetNormalizedName(normalizedName, fieldDesc->GetTypeName()); - // We do have an on-disk field that correspond to the current RClassField instance. Ask for rules matching the - // on-disk version of the field. - if (fieldDesc->GetTypeChecksum()) { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion(), *fieldDesc->GetTypeChecksum()); - } else { - rules = ruleset->FindRules(normalizedName, fieldDesc->GetTypeVersion()); - } - } - - // Cleanup and sort rules - // Check that any any given source member uses the same type in all rules - std::unordered_map sourceNameAndType; - std::size_t nskip = 0; // skip whole-object-rules that were moved to the end of the rules vector - for (auto itr = rules.begin(); itr != rules.end() - nskip;) { - const auto rule = *itr; - - // Erase unknown rule types - if (rule->GetRuleType() != ROOT::TSchemaRule::kReadRule) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule with unsupported type: " << rule->GetRuleType(); - itr = rules.erase(itr); - continue; - } - - bool hasConflictingSourceMembers = false; - for (auto source : TRangeDynCast(rule->GetSource())) { - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto [itrSrc, isNew] = sourceNameAndType.emplace(source->GetName(), memberType); - if (!isNew && (itrSrc->second != memberType)) { - R__LOG_WARNING(ROOT::Internal::NTupleLog()) - << "ignoring I/O customization rule due to conflicting source member type: " << itrSrc->second << " vs. " - << memberType << " for member " << source->GetName(); - hasConflictingSourceMembers = true; - break; - } - } - if (hasConflictingSourceMembers) { - itr = rules.erase(itr); - continue; - } - - // Rules targeting the entire object need to be executed at the end - if (rule->GetTarget() == nullptr) { - nskip++; - if (itr != rules.end() - nskip) - std::iter_swap(itr++, rules.end() - nskip); - continue; - } - - ++itr; - } - - return rules; -} - std::unique_ptr ROOT::RClassField::CloneImpl(std::string_view newName) const { return std::unique_ptr(new RClassField(newName, *this)); @@ -395,107 +503,6 @@ void ROOT::RClassField::ReadInClusterImpl(RNTupleLocalIndex localIndex, void *to } } -ROOT::DescriptorId_t ROOT::RClassField::LookupMember(const ROOT::RNTupleDescriptor &desc, std::string_view memberName, - ROOT::DescriptorId_t classFieldId) -{ - auto idSourceMember = desc.FindFieldId(memberName, classFieldId); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - - for (const auto &subFieldDesc : desc.GetFieldIterable(classFieldId)) { - const auto &subFieldName = subFieldDesc.GetFieldName(); - if (subFieldName.length() > 2 && subFieldName[0] == ':' && subFieldName[1] == '_') { - idSourceMember = LookupMember(desc, memberName, subFieldDesc.GetId()); - if (idSourceMember != ROOT::kInvalidDescriptorId) - return idSourceMember; - } - } - - return ROOT::kInvalidDescriptorId; -} - -void ROOT::RClassField::SetStagingClass(const std::string &className, unsigned int classVersion) -{ - TClass::GetClass(className.c_str())->GetStreamerInfo(classVersion); - if (classVersion != GetTypeVersion() || className != GetTypeName()) { - fStagingClass = TClass::GetClass((className + std::string("@@") + std::to_string(classVersion)).c_str()); - if (!fStagingClass) { - // For a rename rule, we may simply ask for the old class name - fStagingClass = TClass::GetClass(className.c_str()); - } - } else { - fStagingClass = fClass; - } - R__ASSERT(fStagingClass); - R__ASSERT(static_cast(fStagingClass->GetClassVersion()) == classVersion); -} - -void ROOT::RClassField::PrepareStagingArea(const std::vector &rules, - const ROOT::RNTupleDescriptor &desc, - const ROOT::RFieldDescriptor &classFieldDesc) -{ - std::size_t stagingAreaSize = 0; - for (const auto rule : rules) { - for (auto source : TRangeDynCast(rule->GetSource())) { - auto [itr, isNew] = fStagingItems.emplace(source->GetName(), RStagingItem()); - if (!isNew) { - // This source member has already been processed by another rule (and we only support one type per member) - continue; - } - RStagingItem &stagingItem = itr->second; - - const auto memberFieldId = LookupMember(desc, source->GetName(), classFieldDesc.GetId()); - if (memberFieldId == kInvalidDescriptorId) { - throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." + - source->GetName())); - } - - auto memberType = source->GetTypeForDeclaration() + source->GetDimensions(); - auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap(); - memberField->SetOnDiskId(memberFieldId); - auto fieldZero = std::make_unique(); - Internal::SetAllowFieldSubstitutions(*fieldZero, true); - fieldZero->Attach(std::move(memberField)); - stagingItem.fField = std::move(fieldZero); - - stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName()); - // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it - // to be present in the TClass instance, too. - R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing); - stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize()); - } - } - - if (stagingAreaSize) { - R__ASSERT(static_cast(stagingAreaSize) <= fStagingClass->Size()); // we may have removed rules - // We use std::make_unique instead of MakeUninitArray to zero-initialize the staging area. - fStagingArea = std::make_unique(stagingAreaSize); - - for (const auto &[_, si] : fStagingItems) { - const auto &memberField = *si.fField->cbegin(); - if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) { - CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset); - } - } - } -} - -void ROOT::RClassField::AddReadCallbacksFromIORule(const TSchemaRule *rule) -{ - auto func = rule->GetReadFunctionPointer(); - if (func == nullptr) { - // Can happen for rename rules - return; - } - fReadCallbacks.emplace_back([func, stagingClass = fStagingClass, stagingArea = fStagingArea.get()](void *target) { - TVirtualObject onfileObj{nullptr}; - onfileObj.fClass = stagingClass; - onfileObj.fObject = stagingArea; - func(static_cast(target), &onfileObj); - onfileObj.fObject = nullptr; // TVirtualObject does not own the value - }); -} - std::unique_ptr ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) { std::vector rules; @@ -682,8 +689,7 @@ void ROOT::RClassField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) cons //------------------------------------------------------------------------------ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, const RSoAField &source) - : ROOT::RFieldBase(fieldName, source.GetTypeName(), ROOT::ENTupleStructure::kCollection, false /* isSimple */), - fSoAClass(source.fSoAClass), + : ROOT::RRuleField(fieldName, source.GetTypeName(), source.fClass, ROOT::ENTupleStructure::kCollection), fSoAMemberOffsets(source.fSoAMemberOffsets) { fTraits = source.GetTraits(); @@ -773,13 +779,13 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() } // Base classes are treated as unrolled nested SoA classes - const auto *soaBases = fSoAClass->GetListOfBases(); + const auto *soaBases = fClass->GetListOfBases(); if (soaBases->GetSize() != static_cast(nDirectRecordBases)) { throw RException(R__FAIL(std::string("number of base classes don't match between SoA class ") + GetFieldName() + " and its underlying record type")); } unsigned int baseIdx = 0; - for (auto base : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfBases())) { + for (auto base : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfBases())) { if (base->GetDelta() < 0) { throw RException(R__FAIL(std::string("virtual inheritance is not supported: ") + GetTypeName() + " virtually inherits from " + base->GetName())); @@ -809,11 +815,16 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + for (const RRule &r : soaBaseField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += base->GetDelta(); + } + baseIdx++; } unsigned int nMembers = 0; - for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfDataMembers())) { + for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfDataMembers())) { // NOTE: ReconstructSplitFields() will also traverse the data members and need to apply the same rules for // skipping members @@ -835,7 +846,7 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() assert(dmField->GetFieldName() == underlyingField->GetFieldName()); if (auto soaField = dynamic_cast(dmField.get())) { - if (ROOT::Internal::GetRNTupleSoARecord(soaField->fSoAClass) != underlyingField->GetTypeName()) { + if (ROOT::Internal::GetRNTupleSoARecord(soaField->fClass) != underlyingField->GetTypeName()) { throw RException(R__FAIL(std::string("nested SoA field ") + soaField->GetQualifiedFieldName() + " [" + soaField->GetTypeName() + "] does not match underlying type " + underlyingField->GetTypeName())); @@ -844,6 +855,11 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() GraftNestedMemberFields(*soaField, dataMember->GetOffset(), [&](const std::string &name) { return realRecordMemberFields[recordFieldNameToIdx[name]]; }); + + for (const RRule &r : soaField->fRules) { + fRules.emplace_back(r); + fRules.back().fOffset += dataMember->GetOffset(); + } } else if (auto vecField = dynamic_cast(dmField.get())) { if (vecField->begin()->GetTypeName() != underlyingField->GetTypeName() || vecField->begin()->GetTypeAlias() != underlyingField->GetTypeAlias()) { @@ -869,20 +885,22 @@ void ROOT::Experimental::RSoAField::CollectRecordMemberFields() if (nDirectRecordSubfields != nMembers) { throw RException(R__FAIL("missing SoA members")); } + + auto schemaRules = FindRules(nullptr); + for (const auto &r : schemaRules) + fRules.emplace_back(RRule{r, 0}); } ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA) - : ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection, - false /* isSimple */), - fSoAClass(clSoA) + : ROOT::RRuleField(fieldName, GetRenormalizedTypeName(clSoA->GetName()), clSoA, ROOT::ENTupleStructure::kCollection) { static std::once_flag once; std::call_once(once, []() { R__LOG_WARNING(ROOT::Internal::NTupleLog()) << "The SoA field is experimental and still under development."; }); - EnsureValidUserClass(fSoAClass, *this, "RSoAField"); - const auto recordTypeName = ROOT::Internal::GetRNTupleSoARecord(fSoAClass); + EnsureValidUserClass(fClass, *this, "RSoAField"); + const auto recordTypeName = ROOT::Internal::GetRNTupleSoARecord(fClass); if (recordTypeName.empty()) { throw ROOT::RException(R__FAIL(std::string("class ") + GetTypeName() + " is not marked with the rntupleSoARecord " @@ -893,17 +911,17 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clS } catch (ROOT::RException &e) { throw RException(R__FAIL("invalid record type of SoA field " + GetTypeName() + " [" + e.what() + "]")); } - R__ASSERT(fSoAClass->GetClassVersion() >= 0); - if (static_cast(fSoAClass->GetClassVersion()) != fSubfields[0]->GetTypeVersion()) { + R__ASSERT(fClass->GetClassVersion() >= 0); + if (static_cast(fClass->GetClassVersion()) != fSubfields[0]->GetTypeVersion()) { throw RException(R__FAIL(std::string("version mismatch between SoA type and underlying record type: ") + - std::to_string(fSoAClass->GetClassVersion()) + " vs. " + + std::to_string(fClass->GetClassVersion()) + " vs. " + std::to_string(fSubfields[0]->GetTypeVersion()))); } CollectRecordMemberFields(); std::string renormalizedAlias; - if (ROOT::Internal::NeedsMetaNameAsAlias(fSoAClass->GetName(), renormalizedAlias)) + if (ROOT::Internal::NeedsMetaNameAsAlias(fClass->GetName(), renormalizedAlias)) fTypeAlias = renormalizedAlias; fTraits |= kTraitSoACollection | kTraitTypeChecksum; @@ -1015,14 +1033,47 @@ void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalInde } } +std::unique_ptr +ROOT::Experimental::RSoAField::BeforeConnectPageSource(ROOT::Internal::RPageSource & /*pageSource*/) +{ + // Most of the heavy lifting is done by the actual subfields that map onto the on-disk AoS schema. + // In the SoA field itself, we only need to take care of the in-memory part of the rule processing. + // One complication is that for non-rename rules, we manually need to recurse into base (SoA) classes and + // nested (SoA) classes because we don't have them in the field tree. + + // For now, allow only rename rules and whole-object rules + + const bool hasSources = std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { + return r.fRule->GetSource() && (r.fRule->GetSource()->GetEntries() > 0); + }); + const bool hasTargets = + std::any_of(fRules.begin(), fRules.end(), [](const auto &r) { return r.fRule->GetTarget(); }); + + if (hasSources) { + throw RException(R__FAIL("I/O customization rules with sources are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + if (hasTargets) { + throw RException(R__FAIL("I/O customization rules with targets are currently unsupported for SoA fields (" + + GetTypeName() + ")")); + } + + for (const auto &r : fRules) { + AddReadCallbacksFromIORule(r.fRule, r.fOffset); + } + + return nullptr; +} + void ROOT::Experimental::RSoAField::ReconcileOnDiskField(const RNTupleDescriptor &desc) { - EnsureMatchingOnDiskField(desc, kDiffTypeVersion).ThrowOnError(); + EnsureMatchingOnDiskField(desc, kDiffTypeName | kDiffTypeVersion).ThrowOnError(); } void ROOT::Experimental::RSoAField::ConstructValue(void *where) const { - fSoAClass->New(where); + fClass->New(where); } ROOT::Experimental::RSoAField::RSoADeleter::RSoADeleter(TClass *cl) : RDeleter(cl->GetClassAlignment()), fSoAClass(cl) @@ -1045,7 +1096,7 @@ void ROOT::Experimental::RSoAField::ReconstructSplitFields() const fSplitOffsets = std::make_unique>(); unsigned int baseIdx = 0; - for (auto base : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfBases())) { + for (auto base : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfBases())) { TClass *cl = base->GetClassPointer(); auto baseField = RFieldBase::Create(std::string(":_" + std::to_string(baseIdx)), cl->GetName()).Unwrap(); fSplitFields->emplace_back(std::move(baseField)); @@ -1053,7 +1104,7 @@ void ROOT::Experimental::RSoAField::ReconstructSplitFields() const baseIdx++; } - for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fSoAClass->GetListOfDataMembers())) { + for (auto dataMember : ROOT::Detail::TRangeStaticCast(*fClass->GetListOfDataMembers())) { if ((dataMember->Property() & kIsStatic) || !dataMember->IsPersistent()) continue; @@ -1081,22 +1132,22 @@ std::vector ROOT::Experimental::RSoAField::SplitValue( std::size_t ROOT::Experimental::RSoAField::GetValueSize() const { - return fSoAClass->GetClassSize(); + return fClass->GetClassSize(); } std::uint32_t ROOT::Experimental::RSoAField::GetTypeVersion() const { - return fSoAClass->GetClassVersion(); + return fClass->GetClassVersion(); } std::uint32_t ROOT::Experimental::RSoAField::GetTypeChecksum() const { - return fSoAClass->GetCheckSum(); + return fClass->GetCheckSum(); } std::size_t ROOT::Experimental::RSoAField::GetAlignment() const { - const auto align = fSoAClass->GetClassAlignment(); + const auto align = fClass->GetClassAlignment(); EnsureValidAlignment(align); return align; } @@ -1104,11 +1155,11 @@ std::size_t ROOT::Experimental::RSoAField::GetAlignment() const const std::type_info *ROOT::Experimental::RSoAField::GetPolymorphicTypeInfo() const { // TODO(jblomer): factor out - bool polymorphic = fSoAClass->ClassProperty() & kClassHasVirtual; + bool polymorphic = fClass->ClassProperty() & kClassHasVirtual; if (!polymorphic) { return nullptr; } - return fSoAClass->GetTypeInfo(); + return fClass->GetTypeInfo(); } void ROOT::Experimental::RSoAField::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const diff --git a/tree/ntuple/test/SoAField.hxx b/tree/ntuple/test/SoAField.hxx index e74192aa84c88..89704b2765da0 100644 --- a/tree/ntuple/test/SoAField.hxx +++ b/tree/ntuple/test/SoAField.hxx @@ -170,4 +170,118 @@ struct SoADerivedFail2 : public SoABase { ClassDefNV(SoADerivedFail2, 2); }; +struct RecordBaseOld { + float fBase; + ClassDefNV(RecordBaseOld, 2); +}; + +struct SoABaseOld { + ROOT::RVec fBase; + ClassDefNV(SoABaseOld, 2); +}; + +struct RecordIntermediateOld : public RecordBaseOld { + float fIntermediate; + ClassDefNV(RecordIntermediateOld, 2); +}; + +struct SoAIntermediateOld : public SoABaseOld { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateOld, 2); +}; + +struct RecordLeafOld : public RecordIntermediateOld { + float fLeaf; + ClassDefNV(RecordLeafOld, 2); +}; + +struct SoALeafOld : public SoAIntermediateOld { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafOld, 2); +}; + +struct RecordBaseNew { + float fBase; + float fNew; + ClassDefNV(RecordBaseNew, 2); +}; + +struct SoABaseNew { + ROOT::RVec fBase; + ROOT::RVec fNew; + ClassDefNV(SoABaseNew, 2); +}; + +struct RecordIntermediateNew : public RecordBaseNew { + float fIntermediate; + ClassDefNV(RecordIntermediateNew, 2); +}; + +struct SoAIntermediateNew : public SoABaseNew { + ROOT::RVec fIntermediate; + ClassDefNV(SoAIntermediateNew, 2); +}; + +struct RecordLeafNew : public RecordIntermediateNew { + float fLeaf; + ClassDefNV(RecordLeafNew, 2); +}; + +struct RecordLeafLikeNew : public RecordIntermediateNew { + float fLeaf; + ClassDefNV(RecordLeafLikeNew, 2); +}; + +struct SoALeafNew : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafNew, 2); +}; + +struct SoALeafNewNoRenameRule : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafNewNoRenameRule, 2); +}; + +struct SoALeafLikeNew : public SoAIntermediateNew { + ROOT::RVec fLeaf; + ClassDefNV(SoALeafLikeNew, 2); +}; + +struct RecordNested { + float fInner; + ClassDefNV(RecordNested, 2); +}; + +struct SoANested { + ROOT::RVec fInner; + ClassDefNV(SoANested, 2); +}; + +struct RecordOuter { + float fOuter; + RecordNested fNested; + ClassDefNV(RecordOuter, 2); +}; + +struct SoAOuter { + ROOT::RVec fOuter; + SoANested fNested; + ClassDefNV(SoAOuter, 2); +}; + +struct RecordRuleFail { + float fX; + ClassDefNV(RecordRuleFail, 2); +}; + +struct SoARuleFailSource { + ROOT::RVec fX; + ClassDefNV(SoARuleFailSource, 2); +}; + +struct SoARuleFailTarget { + ROOT::RVec fX; + ClassDefNV(SoARuleFailTarget, 2); +}; + #endif // ROOT_RNTuple_Test_SoAField diff --git a/tree/ntuple/test/SoAFieldLinkDef.h b/tree/ntuple/test/SoAFieldLinkDef.h index 0af69cec4189a..44e0ffe512c63 100644 --- a/tree/ntuple/test/SoAFieldLinkDef.h +++ b/tree/ntuple/test/SoAFieldLinkDef.h @@ -34,4 +34,46 @@ #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail1+; #pragma link C++ options=rntupleSoARecord(RecordDerived) class SoADerivedFail2+; +#pragma link C++ class RecordBaseOld+; +#pragma link C++ class RecordIntermediateOld+; +#pragma link C++ class RecordLeafOld+; +#pragma link C++ options=rntupleSoARecord(RecordBaseOld) class SoABaseOld+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateOld) class SoAIntermediateOld+; +#pragma link C++ options=rntupleSoARecord(RecordLeafOld) class SoALeafOld+; +#pragma link C++ class RecordBaseNew+; +#pragma link C++ class RecordIntermediateNew+; +#pragma link C++ class RecordLeafNew+; +#pragma link C++ class RecordLeafLikeNew+; +#pragma link C++ options=rntupleSoARecord(RecordBaseNew) class SoABaseNew+; +#pragma link C++ options=rntupleSoARecord(RecordIntermediateNew) class SoAIntermediateNew+; +#pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNew+; +#pragma link C++ options=rntupleSoARecord(RecordLeafNew) class SoALeafNewNoRenameRule+; +#pragma link C++ options=rntupleSoARecord(RecordLeafLikeNew) class SoALeafLikeNew+; + +#pragma read sourceClass = "RecordBaseOld" targetClass = "RecordBaseNew" version = "[1-]"; +#pragma read sourceClass = "RecordIntermediateOld" targetClass = "RecordIntermediateNew" version = "[1-]"; +#pragma read sourceClass = "RecordLeafOld" targetClass = "RecordLeafNew" version = "[1-]"; +#pragma read sourceClass = "SoABaseOld" targetClass = "SoABaseNew" version = "[1-]"; +#pragma read sourceClass = "SoAIntermediateOld" targetClass = "SoAIntermediateNew" version = "[1-]"; +#pragma read sourceClass = "SoALeafOld" targetClass = "SoALeafNew" version = "[1-]"; + +#pragma link C++ class RecordNested+; +#pragma link C++ class RecordOuter+; +#pragma link C++ options=rntupleSoARecord(RecordNested) class SoANested+; +#pragma link C++ options=rntupleSoARecord(RecordOuter) class SoAOuter+; + +#pragma read sourceClass="SoANested" version="[1-]" targetClass="SoANested" source="" target="" \ + code="{ newObj->fInner *= 2.; }" +#pragma read sourceClass="SoAOuter" version="[1-]" targetClass="SoAOuter" source="" target="" \ + code="{ newObj->fOuter *= 4.; }" + +#pragma link C++ class RecordRuleFail+; +#pragma link C++ options=rntupleSoARecord(RecordRuleFail) class SoARuleFailSource+; +#pragma link C++ options=rntupleSoARecord(RecordRuleFail) class SoARuleFailTarget+; + +#pragma read sourceClass="SoARuleFailSource" version="[1-]" targetClass="SoARuleFailSource" \ + source="ROOT::RVec fX" target="" code="{ }"; +#pragma read sourceClass="SoARuleFailTarget" version="[1-]" targetClass="SoARuleFailTarget" \ + source="" target="fX" code="{ (void)fX; }"; + #endif // __CLING__ diff --git a/tree/ntuple/test/ntuple_soa.cxx b/tree/ntuple/test/ntuple_soa.cxx index 62b7bf42b4656..6bf4f198199f8 100644 --- a/tree/ntuple/test/ntuple_soa.cxx +++ b/tree/ntuple/test/ntuple_soa.cxx @@ -385,6 +385,7 @@ TEST(RNTuple, SoAFromVector) auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); v->emplace_back(RecordSimple{1.0, 2.0}); + v->emplace_back(RecordSimple{3.0, 4.0}); writer->Fill(); } @@ -392,14 +393,16 @@ TEST(RNTuple, SoAFromVector) auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); SoASimple soa; - // Until SoA schema evolution is implemented, the reading the vector as SoA will - try { - reader->GetView("simple", &soa, "SoASimple"); - FAIL() << "reading a vector with a SoA field should fail"; - } catch (const ROOT::RException &e) { - EXPECT_THAT(e.what(), testing::HasSubstr( - "in-memory field simple of type SoASimple is incompatible with on-disk field simple")); - } + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "simple": [{"fX": 1, "fY": 2}, {"fX": 3, "fY": 4}] +} +)"}; + // clang-format on + EXPECT_EQ(expected, os.str()); } TEST(RNTuple, SoAShow) @@ -576,3 +579,123 @@ R"({ // clang-format on EXPECT_EQ(expected, os.str()); } + +TEST(RNTuple, SoARename) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_rename.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("leaf", "SoALeafOld")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto leafSoA = writer->GetModel().GetDefaultEntry().GetPtr("leaf"); + leafSoA->fBase = {1.0, 2.0}; + leafSoA->fIntermediate = {3.0, 4.0}; + leafSoA->fLeaf = {5.0, 6.0}; + + writer->Fill(); + } + + { + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafNew")); + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + + // We cannot use "Show()" because that will reconstruct the original model as a display model, it will not + // use the imposed model. + + auto leafSoA = reader->GetModel().GetDefaultEntry().GetPtr("leaf"); + reader->LoadEntry(0); + + EXPECT_EQ(2u, leafSoA->fBase.size()); + EXPECT_EQ(2u, leafSoA->fNew.size()); + EXPECT_EQ(2u, leafSoA->fIntermediate.size()); + EXPECT_EQ(2u, leafSoA->fLeaf.size()); + EXPECT_FLOAT_EQ(1.0, leafSoA->fBase[0]); + EXPECT_FLOAT_EQ(2.0, leafSoA->fBase[1]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[0]); + EXPECT_FLOAT_EQ(0.0, leafSoA->fNew[1]); + EXPECT_FLOAT_EQ(3.0, leafSoA->fIntermediate[0]); + EXPECT_FLOAT_EQ(4.0, leafSoA->fIntermediate[1]); + EXPECT_FLOAT_EQ(5.0, leafSoA->fLeaf[0]); + EXPECT_FLOAT_EQ(6.0, leafSoA->fLeaf[1]); + } + + { + auto model = ROOT::RNTupleModel::Create(); + model->AddField(std::make_unique("leaf", "SoALeafLikeNew")); + try { + auto reader = ROOT::RNTupleReader::Open(std::move(model), "ntpl", fileGuard.GetPath()); + FAIL() << "mismatch in underlying record field should fail"; + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("RecordLeafLikeNew vs. RecordLeafOld")); + } + } +} + +TEST(RNTuple, SoARuleFailures) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_whole_object_rule.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("source", "SoARuleFailSource")); + model->AddField(std::make_unique("target", "SoARuleFailSource")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + + SoARuleFailSource source; + try { + auto viewSoASource = reader->GetView("source", &source, "SoARuleFailSource"); + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("rules with sources are currently unsupported for SoA fields")); + } + + SoARuleFailTarget target; + try { + auto viewSoASource = reader->GetView("target", &target, "SoARuleFailTarget"); + } catch (const ROOT::RException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("rules with targets are currently unsupported for SoA fields")); + } +} + +TEST(RNTuple, SoAWholeObjectRule) +{ + ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_whole_object_rule.root"); + + { + auto model = ROOT::RNTupleModel::Create(); + + model->AddField(std::make_unique("outer", "SoAOuter")); + auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath()); + + auto outerSoA = writer->GetModel().GetDefaultEntry().GetPtr("outer"); + outerSoA->fNested.fInner = {1.0, 2.0}; + outerSoA->fOuter = {3.0, 4.0}; + + writer->Fill(); + } + + auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath()); + + std::ostringstream os; + reader->Show(0, os); + // clang-format off + std::string expected{ +R"({ + "outer": { + "fOuter": [12, 16], + "fNested": { + "fInner": [2, 4] + } + } +} +)" }; + // clang-format on + EXPECT_EQ(expected, os.str()); +}