Skip to content

[ntuple] implement nested SoA I/O #22761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jblomer wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[ntuple] implement nested SoA I/O #22761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5e806dc
[ntuple] add helper method to RSoAField constructor
jblomer Jun 24, 2026
9db1a50
[ntuple] prepare SoA field construction for nested types
jblomer Jun 24, 2026
d282aaf
[ntuple] add support for nested SoA types
jblomer Jul 3, 2026
6ca7c23
[ntuple] add support for inheritance in SoA fields
jblomer Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <ROOT/RNTupleTypes.hxx>

#include <cstddef>
#include <functional>
#include <memory>
#include <mutex>
#include <string_view>
Expand Down Expand Up @@ -63,9 +64,12 @@ class RSoAField : public RFieldBase {
};

TClass *fSoAClass = nullptr;
std::vector<RFieldBase *> fRecordMemberFields; ///< Direct access to the member fields of the underlying record
/// The offset of the RVec members in the SoA type in the order of subfields of the underlying record type.
/// In particular, the order is not necessarily the same then the order of RVec members in the SoA class.
/// Direct access to the member fields of the underlying record. In case of a nested SoA type, this vector
/// contains the contents of the inner fRecordMemberFields, too. Effectively, this record will contain all the
/// fields of the underlying record type that correspond to terminal RVec members in a nested SoA type.
/// Pointers in this vector point into the field hierarchy owned by fSubfields[0].
std::vector<RFieldBase *> fRecordMemberFields;
/// The offset of the RVec members in the SoA type in the same order as fRecordMemberFields.
std::vector<std::size_t> fSoAMemberOffsets;
///< A deleter returned by each record member's GetDeleter()
std::vector<std::unique_ptr<RDeleter>> fRecordMemberDeleters;
Expand All @@ -75,12 +79,26 @@ class RSoAField : public RFieldBase {
/// SoA object are used directly with the subfields of the underlying record type. For splitting a SoA class object
/// (SplitValue()), however, we need actual RRVecFields so that we can recursively split the in-memory SoA value.
/// The split fields are created only when SplitField() is called.
mutable std::unique_ptr<std::vector<std::unique_ptr<ROOT::RRVecField>>> fSplitFields;
/// In general, the number of split fields is different from the number of record member fields because there is
/// one split field for every _direct_ member of the SoA class. Because nested SoA structs can have transient
/// members, we can also not reuse fSoAMemberOffsets but we need to store the data member offsets.
mutable std::unique_ptr<std::vector<std::unique_ptr<ROOT::RFieldBase>>> fSplitFields;
mutable std::unique_ptr<std::vector<std::size_t>> fSplitOffsets;
mutable std::unique_ptr<std::mutex> fLockSplitFields; ///< protects the fSplitFields member.

RSoAField(std::string_view fieldName, const RSoAField &source); ///< Used by CloneImpl
RSoAField(std::string_view fieldName, TClass *clSoA);

/// Called during construction, picks up the (nested) member fields of the underlying record type(s) and its
/// base classes.
void CollectRecordMemberFields();
/// For a nested SoA struct (either as a member of as a base class), use their fRecordMemberFields in this class,
/// i.e. "unroll" the vectors in the nested SoA struct into the SoA base class.
void GraftNestedMemberFields(const RSoAField &nestedSoA, std::size_t offsetInParent,
std::function<RFieldBase *(const std::string &)> fnRecordFieldFinder);

void ReconstructSplitFields() const;

protected:
std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const final;

Expand Down
283 changes: 199 additions & 84 deletions tree/ntuple/src/RFieldMeta.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,171 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, std::string
{
}

void ROOT::Experimental::RSoAField::GraftNestedMemberFields(
const RSoAField &nestedSoA, std::size_t offsetInParent,
std::function<RFieldBase *(const std::string &)> fnRecordFieldFinder)
{
const std::size_t nNestedRecordMemberFields = nestedSoA.fRecordMemberFields.size();

// The qualified field name of fields in nestedSoA->fRecordMemberFields will have a "<field name>._0."
// prefix because these fields are rooted in a collection named after the nested SoA field.
//
// E.g., in the following example:
//
// struct SoA_A { struct Record_A {
// SoA_B fB; Record_B fB;
// }; };
//
// struct SoA_B { struct Record_B {
// ROOT::RVec<float> fX; float fX;
// }; };
//
// The on-disk schema of SoA_A is "collection of Record_A", and the on-disk schema of SoA_B is
// "collection of Record_B".
// The qualified field name of fX in the subfield hiararchy of SoA_A is <field name>._0.fB.fX.
// The qualified field name of fX in the subfield hiararchy of SoA_B is <field name>._0.fX.
const auto lenPrefix = nestedSoA.GetFieldName().length() + strlen("._0.");

for (std::size_t i = 0; i < nNestedRecordMemberFields; ++i) {
const auto fieldNameForMatching =
nestedSoA.GetFieldName() + "." + nestedSoA.fRecordMemberFields[i]->GetQualifiedFieldName().substr(lenPrefix);

fRecordMemberFields.emplace_back(fnRecordFieldFinder(fieldNameForMatching));
fRecordMemberDeleters.emplace_back(GetDeleterOf(*nestedSoA.fRecordMemberFields[i]));
fSoAMemberOffsets.emplace_back(offsetInParent + nestedSoA.fSoAMemberOffsets[i]);
}
}

void ROOT::Experimental::RSoAField::CollectRecordMemberFields()
{
// Build a map of all subfields (nested) of the underlying record type. Map the fully qualified name of the
// subfields to their field pointer, so that we can later match the subfields of the SoA class to their corresponding
// fields in the underlying record type. Note that the members of the SoA class and the underlying record type
// can have different ordering. However, the base classes of the SoA class and the underlying record type must match
// in order.

std::vector<RFieldBase *> realRecordMemberFields; // Contains all subfields of the underlying record type
// Qualified field name --> index in realRecordMemberFields
std::unordered_map<std::string, std::size_t> recordFieldNameToIdx;

// Count the top-level subfields of the underlying record type for cross-check with the SoA type
unsigned int nDirectRecordSubfields = 0;
unsigned int nDirectRecordBases = 0;

for (auto itr = fSubfields[0]->begin(), iEnd = fSubfields[0]->end(); itr != iEnd; ++itr) {
if (itr->GetParent() == fSubfields[0].get()) {
(itr->GetFieldName()[0] == ':') ? nDirectRecordBases++ : nDirectRecordSubfields++;
}

// Build the qualified field name for matching. We root the qualified field name at the underlying record type.
auto qualifiedName = itr->GetFieldName();
auto parent = itr->GetParent();
while (parent != fSubfields[0].get()) {
qualifiedName = parent->GetFieldName() + "." + qualifiedName;
parent = parent->GetParent();
}
recordFieldNameToIdx[qualifiedName] = realRecordMemberFields.size();

realRecordMemberFields.emplace_back(&(*itr));
}

// Base classes are treated as unrolled nested SoA classes
const auto *soaBases = fSoAClass->GetListOfBases();
if (soaBases->GetSize() != static_cast<Int_t>(nDirectRecordBases)) {
throw RException(R__FAIL(std::string("number of base classes don't match between SoA class ") + GetFieldName() +
" and its underlying record type"));
}
for (unsigned int i = 0; i < static_cast<unsigned int>(soaBases->GetSize()); ++i) {
auto base = static_cast<TBaseClass *>(soaBases->At(i));
if (base->GetDelta() < 0) {
throw RException(R__FAIL(std::string("virtual inheritance is not supported: ") + GetTypeName() +
" virtually inherits from " + base->GetName()));
}
TClass *cl = base->GetClassPointer();

const auto baseFieldName = std::string(":_") + std::to_string(i);

// SoA class `A` inherits from a SoA class `B` whose underlying record type is `X` if and only if the underlying
// record type of `A` inherits from a type `X`.
const auto underlyingBaseTypeName = ROOT::Internal::GetRNTupleSoARecord(cl);
auto recordBaseField = realRecordMemberFields[recordFieldNameToIdx[baseFieldName]];
if (underlyingBaseTypeName != recordBaseField->GetTypeName()) {
throw RException(R__FAIL(std::string("inheritance of SoA class ") + GetFieldName() +
" does not match its underlying record type"));
}

std::unique_ptr<RSoAField> soaBaseField;
try {
soaBaseField = std::make_unique<RSoAField>(baseFieldName, cl->GetName());
} catch (const RException &e) {
throw RException(R__FAIL(std::string("invalid field type in base class: ") + cl->GetName() + " of SoA field " +
GetFieldName() + " (" + e.what() + ")"));
}

GraftNestedMemberFields(*soaBaseField, base->GetDelta(), [&](const std::string &name) {
return realRecordMemberFields[recordFieldNameToIdx[name]];
});
}

unsigned int nMembers = 0;
for (auto dataMember : ROOT::Detail::TRangeStaticCast<TDataMember>(*fSoAClass->GetListOfDataMembers())) {
// NOTE: ReconstructSplitFields() will also traverse the data members and need to apply the same rules for
// skipping members

if ((dataMember->Property() & kIsStatic) || !dataMember->IsPersistent())
continue;

if (dataMember->Property() & kIsArray) {
throw RException(R__FAIL(std::string("unsupported array type in SoA class: ") + dataMember->GetName()));
}

const std::string typeName{dataMember->GetTrueTypeName()};
auto dmField = RFieldBase::Create(dataMember->GetName(), typeName).Unwrap();

auto itr = recordFieldNameToIdx.find(dmField->GetFieldName());
if (itr == recordFieldNameToIdx.end()) {
throw RException(R__FAIL(std::string("unexpected SoA member: ") + dmField->GetFieldName()));
}
auto underlyingField = realRecordMemberFields[itr->second];
assert(dmField->GetFieldName() == underlyingField->GetFieldName());

if (auto soaField = dynamic_cast<RSoAField *>(dmField.get())) {
if (ROOT::Internal::GetRNTupleSoARecord(soaField->fSoAClass) != underlyingField->GetTypeName()) {
throw RException(R__FAIL(std::string("nested SoA field ") + soaField->GetQualifiedFieldName() + " [" +
soaField->GetTypeName() + "] does not match underlying type " +
underlyingField->GetTypeName()));
}

GraftNestedMemberFields(*soaField, dataMember->GetOffset(), [&](const std::string &name) {
return realRecordMemberFields[recordFieldNameToIdx[name]];
});
} else if (auto vecField = dynamic_cast<RRVecField *>(dmField.get())) {
if (vecField->begin()->GetTypeName() != underlyingField->GetTypeName() ||
vecField->begin()->GetTypeAlias() != underlyingField->GetTypeAlias()) {
const std::string leftType =
vecField->begin()->GetTypeName() +
(vecField->begin()->GetTypeAlias().empty() ? "" : " [" + vecField->begin()->GetTypeAlias() + "]");
const std::string rightType =
underlyingField->GetTypeName() +
(underlyingField->GetTypeAlias().empty() ? "" : " [" + underlyingField->GetTypeAlias() + "]");
throw RException(R__FAIL(std::string("SoA member type mismatch: ") + vecField->GetFieldName() + " (" +
leftType + " vs. " + rightType + ")"));
}

fRecordMemberFields.emplace_back(underlyingField);
fRecordMemberDeleters.emplace_back(GetDeleterOf(*underlyingField));
fSoAMemberOffsets.emplace_back(dataMember->GetOffset());
} else {
throw RException(R__FAIL("invalid field type in SoA class: " + dmField->GetTypeName()));
}

nMembers++;
}
if (nDirectRecordSubfields != nMembers) {
throw RException(R__FAIL("missing SoA members"));
}
}

ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clSoA)
: ROOT::RFieldBase(fieldName, GetRenormalizedTypeName(clSoA->GetName()), ROOT::ENTupleStructure::kCollection,
false /* isSimple */),
Expand Down Expand Up @@ -714,75 +879,8 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clS
std::to_string(fSoAClass->GetClassVersion()) + " vs. " +
std::to_string(fSubfields[0]->GetTypeVersion())));
}
fRecordMemberFields = fSubfields[0]->GetMutableSubfields();

std::unordered_map<std::string, std::size_t> recordFieldNameToIdx;
fRecordMemberDeleters.reserve(fRecordMemberFields.size());
recordFieldNameToIdx.reserve(fRecordMemberFields.size());
for (std::size_t i = 0; i < fRecordMemberFields.size(); ++i) {
const RFieldBase *f = fRecordMemberFields[i];
assert(!f->GetFieldName().empty());
if (f->GetFieldName()[0] == ':') {
throw RException(R__FAIL("SoA fields with inheritance are currently unsupported"));
}
recordFieldNameToIdx[f->GetFieldName()] = i;
fRecordMemberDeleters.emplace_back(GetDeleterOf(*f));
}

const auto *bases = fSoAClass->GetListOfBases();
assert(bases);
for (auto baseClass : ROOT::Detail::TRangeStaticCast<TBaseClass>(*bases)) {
if (baseClass->GetDelta() < 0) {
throw RException(R__FAIL(std::string("virtual inheritance is not supported: ") + GetTypeName() +
" virtually inherits from " + baseClass->GetName()));
}
// At a later point, we will support inheritance
throw RException(R__FAIL("SoA fields with inheritance are currently unsupported"));
}

fSoAMemberOffsets.resize(fRecordMemberFields.size());
unsigned int nMembers = 0;
for (auto dataMember : ROOT::Detail::TRangeStaticCast<TDataMember>(*fSoAClass->GetListOfDataMembers())) {
if ((dataMember->Property() & kIsStatic) || !dataMember->IsPersistent())
continue;

if (dataMember->Property() & kIsArray) {
throw RException(R__FAIL(std::string("unsupported array type in SoA class: ") + dataMember->GetName()));
}

const std::string typeName{dataMember->GetTrueTypeName()};
auto subField = RFieldBase::Create(dataMember->GetName(), typeName).Unwrap();
auto vecFieldPtr = dynamic_cast<RRVecField *>(subField.get());
if (!vecFieldPtr) {
throw RException(R__FAIL("invalid field type in SoA class: " + subField->GetTypeName()));
}
subField.release();
auto vecField = std::unique_ptr<RRVecField>(vecFieldPtr);

auto itr = recordFieldNameToIdx.find(vecField->GetFieldName());
if (itr == recordFieldNameToIdx.end()) {
throw RException(R__FAIL(std::string("unexpected SoA member: ") + vecField->GetFieldName()));
}
const RFieldBase *memberField = fRecordMemberFields[itr->second];
if (vecField->begin()->GetTypeName() != memberField->GetTypeName() ||
vecField->begin()->GetTypeAlias() != memberField->GetTypeAlias()) {
const std::string leftType =
vecField->begin()->GetTypeName() +
(vecField->begin()->GetTypeAlias().empty() ? "" : " [" + vecField->begin()->GetTypeAlias() + "]");
const std::string rightType =
memberField->GetTypeName() +
(memberField->GetTypeAlias().empty() ? "" : " [" + memberField->GetTypeAlias() + "]");
throw RException(R__FAIL(std::string("SoA member type mismatch: ") + vecField->GetFieldName() + " (" +
leftType + " vs. " + rightType + ")"));
}

assert(itr->second < fSoAMemberOffsets.size());
fSoAMemberOffsets[itr->second] = dataMember->GetOffset();
nMembers++;
}
if (recordFieldNameToIdx.size() != nMembers) {
throw RException(R__FAIL("missing SoA members"));
}
CollectRecordMemberFields();

std::string renormalizedAlias;
if (ROOT::Internal::NeedsMetaNameAsAlias(fSoAClass->GetName(), renormalizedAlias))
Expand Down Expand Up @@ -898,29 +996,46 @@ void ROOT::Experimental::RSoAField::RSoADeleter::operator()(void *objPtr, bool d
RDeleter::operator()(objPtr, dtorOnly);
}

std::vector<ROOT::RFieldBase::RValue> ROOT::Experimental::RSoAField::SplitValue(const RValue &value) const
void ROOT::Experimental::RSoAField::ReconstructSplitFields() const
{
const auto nSoAMembers = fSoAMemberOffsets.size();
std::lock_guard<std::mutex> lockGuard(*fLockSplitFields);
if (fSplitFields)
return;

{
std::lock_guard<std::mutex> lockGuard(*fLockSplitFields);
if (!fSplitFields) {
fSplitFields = std::make_unique<std::vector<std::unique_ptr<ROOT::RRVecField>>>();
fSplitFields->reserve(nSoAMembers);
for (std::size_t i = 0; i < nSoAMembers; ++i) {
const auto itemField = fRecordMemberFields[i];
fSplitFields->emplace_back(std::make_unique<RRVecField>(itemField->GetFieldName(), itemField->Clone("_0")));
}
}
fSplitFields = std::make_unique<std::vector<std::unique_ptr<ROOT::RFieldBase>>>();
fSplitOffsets = std::make_unique<std::vector<std::size_t>>();

unsigned int iBase = 0;
for (auto base : ROOT::Detail::TRangeStaticCast<TBaseClass>(*fSoAClass->GetListOfBases())) {
TClass *cl = base->GetClassPointer();
auto baseField = RFieldBase::Create(std::string(":_" + std::to_string(iBase)), cl->GetName()).Unwrap();
fSplitFields->emplace_back(std::move(baseField));
fSplitOffsets->emplace_back(base->GetDelta());
iBase++;
}

for (auto dataMember : ROOT::Detail::TRangeStaticCast<TDataMember>(*fSoAClass->GetListOfDataMembers())) {
if ((dataMember->Property() & kIsStatic) || !dataMember->IsPersistent())
continue;

const std::string typeName{dataMember->GetTrueTypeName()};
auto dmField = RFieldBase::Create(dataMember->GetName(), typeName).Unwrap();
fSplitFields->emplace_back(std::move(dmField));
fSplitOffsets->emplace_back(dataMember->GetOffset());
}
}

std::vector<ROOT::RFieldBase::RValue> ROOT::Experimental::RSoAField::SplitValue(const RValue &value) const
{
ReconstructSplitFields();
const auto nSplitFields = fSplitFields->size();

auto valuePtr = value.GetPtr<void>();
auto soaPtr = static_cast<unsigned char *>(valuePtr.get());
std::vector<RValue> values;
values.reserve(nSoAMembers);
for (std::size_t i = 0; i < nSoAMembers; ++i) {
values.emplace_back(
(*fSplitFields)[i]->BindValue(std::shared_ptr<void>(valuePtr, soaPtr + fSoAMemberOffsets[i])));
values.reserve(nSplitFields);
for (std::size_t i = 0; i < nSplitFields; ++i) {
values.emplace_back((*fSplitFields)[i]->BindValue(std::shared_ptr<void>(valuePtr, soaPtr + (*fSplitOffsets)[i])));
}
return values;
}
Expand Down
Loading
Loading