siliataider · Oct 20, 2026 · Oct 16, 2026 · Oct 16, 2026 · Oct 19, 2026 · Oct 19, 2026
@@ -3606,6 +3606,23 @@ def test16_vector_padding(self):
             self.teardown_file(self.file_name5)
             raise
 
+    def test17_dataframes_without_cluster_metadata(self):
+        # the dataframes are not read from a file, so their entries are counted with Count()
+        dl = ROOT.Experimental.ML.RDataLoader(
+            [self.define_rdf1(), self.define_rdf2()],
+            batch_size=3,
+            target="b2",
+            shuffle=False,
+            drop_remainder=False,
+            load_eager=True,
+        )
+        gen_train, gen_validation = dl.train_test_split(0.4)
+
+        x_train = [v for x, _ in gen_train.as_numpy() for v in x.flatten().tolist()]
+        x_val = [v for x, _ in gen_validation.as_numpy() for v in x.flatten().tolist()]
+        self.assertEqual(x_train, [0.0, 1.0, 2.0, 5.0, 6.0, 7.0])
+        self.assertEqual(x_val, [3.0, 4.0, 8.0, 9.0])
+
 
 class DataLoaderLazyLoadingMultipleDataframes(unittest.TestCase):
     file_name1 = "first_half.root"
@@ -7838,6 +7855,37 @@ def test16_vector_padding(self):
             self.teardown_file(self.file_name5)
             raise
 
+    def test17_set_seed_resamples_different_entries(self):
+        # with a fixed seed, the resampled minority entries used to all be the same one
+        self.create_file_major()
+        self.create_file_minor()
+
+        try:
+            df_major = ROOT.RDataFrame(self.tree_name, self.file_name1)
+            df_minor = ROOT.RDataFrame(self.tree_name, self.file_name2)
+
+            dl = ROOT.Experimental.ML.RDataLoader(
+                [df_major, df_minor],
+                batch_size=2,
+                target="b2",
+                shuffle=True,
+                drop_remainder=False,
+                set_seed=42,
+                load_eager=True,
+                sampling_type="oversampling",
+                sampling_ratio=1.0,
+            )
+
+            x = np.concatenate([x for x, _ in dl.as_numpy()]).flatten()
+            minor = x[x % 2 == 1]
+
+            self.assertEqual(len(minor), 10)
+            self.assertGreater(len(np.unique(minor)), 1)
+
+        finally:
+            self.teardown_file(self.file_name1)
+            self.teardown_file(self.file_name2)
+
 
 class DataLoaderSave(unittest.TestCase):
     file_name = "dataloader_save_input.root"

@@ -34,6 +34,9 @@ class R__CLING_PTRCHECK(off) RActionCustomLoader final : public RActionBase {
    /// Column readers per slot and per input column
    std::vector<std::vector<RColumnReaderBase *>> fValues;
 
+   /// Value addresses of the current entry per slot, reused for every entry
+   std::vector<std::vector<void *>> fValueAddresses;
+
    // Types of input columns
    std::vector<const std::type_info *> fColTypeIDs;
 
@@ -47,6 +50,7 @@ public:
         fHelper(std::move(h)),
         fPrevNodes{std::static_pointer_cast<PrevNodeCommon_t>(pd)},
         fValues(GetNSlots()),
+        fValueAddresses(GetNSlots()),
         fColTypeIDs(colTypeIDs)
    {
       fLoopManager->Register(this);
@@ -93,11 +97,11 @@ public:
 
    void CallExec(unsigned int slot, Long64_t entry)
    {
-      std::vector<void *> untypedValues;
+      auto &untypedValues = fValueAddresses[slot];
       auto nReaders = fValues[slot].size();
-      untypedValues.reserve(nReaders);
+      untypedValues.resize(nReaders);
       for (decltype(nReaders) readerIdx{}; readerIdx < nReaders; readerIdx++)
-         untypedValues.push_back(GetValue(slot, readerIdx, entry));
+         untypedValues[readerIdx] = GetValue(slot, readerIdx, entry);
 
       fHelper.Exec(slot, untypedValues);
    }

@@ -109,18 +109,35 @@ private:
    std::size_t fNumValidationEntries;
 
    // flattened buffers for chunks and temporary tensors (rows * cols)
-   std::vector<RFlat2DMatrix> fTrainingDatasets;
-   std::vector<RFlat2DMatrix> fValidationDatasets;
+   RFlat2DMatrix fDataset;
 
-   RFlat2DMatrix fTrainingDataset;
-   RFlat2DMatrix fValidationDataset;
+   // rows of fDataset in the training and validation sets, one set per dataframe when sampling
+   std::vector<std::vector<std::size_t>> fTrainingRows;
+   std::vector<std::vector<std::size_t>> fValidationRows;
 
    RFlat2DMatrix fSampledTrainingDataset;
    RFlat2DMatrix fSampledValidationDataset;
 
    std::size_t fTrainingEpochCount{0};
    std::size_t fValidationEpochCount{0};
 
+   /// \brief Rows of an eager epoch, turned into batches as they are requested
+   struct REagerEpoch {
+      // non-owning pointer: the data stays in the engine's own members (fDataset for the normal path,
+      // fSampledTraining/ValidationDataset for the sampler path)
+      const RFlat2DMatrix *fSource = nullptr;
+      std::vector<std::size_t> fPermutation;
+      std::size_t fNextRow = 0;
+      std::size_t fEndRow = 0;
+   };
+
+   REagerEpoch fTrainingEagerEpoch;
+   REagerEpoch fValidationEagerEpoch;
+
+   REagerEpoch MakeEagerEpoch(const std::vector<std::size_t> &rows);
+
+   void CreateNextEagerBatch(RBatchLoader &batchLoader, REagerEpoch &epoch);
+
    std::vector<RColumnLayout> MakeColumnLayout();
 
    /// \brief Opens a training or validation epoch and closes it again when done

@@ -46,6 +46,8 @@ struct RFlat2DMatrix {
       fRVec.resize(rows * cols);
    }
 
+   void Reserve(std::size_t rows, std::size_t cols) { fRVec.reserve(rows * cols); }
+
    void Reshape(std::size_t rows, std::size_t cols)
    {
       // We don't reallocate: require matching sizes

@@ -5,6 +5,8 @@
 #include <numeric>
 #include <utility>
 
+#include "RFlat2DMatrixOperators.hxx"
+
 namespace ROOT::Experimental::Internal::ML {
 
 RBatchLoader::RBatchLoader(std::size_t batchSize, const std::vector<std::string> &cols, std::mutex &sharedMutex,
@@ -24,7 +26,6 @@ RBatchLoader::RBatchLoader(std::size_t batchSize, const std::vector<std::string>
    RecalculateBatchCounts(numEntries);
 
    fPrimaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
-   fSecondaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
 }
 
 /// \brief Activate the batchloader. This means that batches can be created and loaded.
@@ -67,7 +68,6 @@ void RBatchLoader::Reset()
 
       fCurrentBatch.reset();
       fPrimaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
-      fSecondaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
    }
 
    fCV.notify_all();
@@ -83,13 +83,16 @@ void RBatchLoader::MarkProducerDone()
 /// \brief Return a batch of data as a unique pointer.
 /// After the batch has been processed, it should be destroyed.
 /// \param[in] chunkTensor Tensor with the data from the chunk
-/// \param[in] idxs Index of batch in the chunk
+/// \param[in] permutation Order in which to read the rows of the chunk, empty to read them in order
+/// \param[in] firstRow First row of the batch in the chunk
 /// \return Batch
-std::unique_ptr<RFlat2DMatrix> RBatchLoader::CreateBatch(RFlat2DMatrix &chunkTensor, std::size_t idxs)
+std::unique_ptr<RFlat2DMatrix> RBatchLoader::CreateBatch(const RFlat2DMatrix &chunkTensor,
+                                                         const std::vector<std::size_t> &permutation,
+                                                         std::size_t firstRow)
 {
-   auto batch = std::make_unique<RFlat2DMatrix>(fBatchSize, fNumColumns);
-   std::copy(chunkTensor.GetData() + (idxs * fBatchSize * fNumColumns),
-             chunkTensor.GetData() + ((idxs + 1) * fBatchSize * fNumColumns), batch->GetData());
+   auto batch = std::make_unique<RFlat2DMatrix>();
+   batch->Reserve(fBatchSize, fNumColumns);
+   RFlat2DMatrixOperators::AppendRows(*batch, chunkTensor, permutation, firstRow, firstRow + fBatchSize);
 
    return batch;
 }
@@ -123,86 +126,45 @@ RFlat2DMatrix RBatchLoader::GetBatch()
 
 /// \brief Creating the batches from a chunk and add them to the queue.
 /// \param[in] chunkTensor Tensor with the data from the chunk
+/// \param[in] permutation Order in which to read the rows of the chunk, empty to read them in order
+/// \param[in] beginRow First row of the chunk to create batches from
+/// \param[in] endRow End of the rows of the chunk to create batches from
 /// \param[in] isLastBatch Check if the batch in the chunk is the last one
-void RBatchLoader::CreateBatches(RFlat2DMatrix &chunkTensor, bool isLastBatch)
+void RBatchLoader::CreateBatches(const RFlat2DMatrix &chunkTensor, const std::vector<std::size_t> &permutation,
+                                 std::size_t beginRow, std::size_t endRow, bool isLastBatch)
 {
-   std::size_t chunkSize = chunkTensor.GetRows();
-   std::size_t numCols = chunkTensor.GetCols();
+   std::size_t chunkSize = endRow - beginRow;
    std::size_t numBatches = chunkSize / fBatchSize;
-   std::size_t leftoverBatchSize = chunkSize % fBatchSize;
 
    // create a vector of batches
    std::vector<std::unique_ptr<RFlat2DMatrix>> batches;
 
    // fill the full batches from the chunk into a vector
    for (std::size_t i = 0; i < numBatches; i++) {
-      batches.emplace_back(CreateBatch(chunkTensor, i));
+      batches.emplace_back(CreateBatch(chunkTensor, permutation, beginRow + i * fBatchSize));
    }
 
-   // copy the remaining entries from the chunk into a leftover batch
-   RFlat2DMatrix LeftoverBatch(leftoverBatchSize, numCols);
-   std::copy(chunkTensor.GetData() + (numBatches * fBatchSize * numCols),
-             chunkTensor.GetData() + (numBatches * fBatchSize * numCols + leftoverBatchSize * numCols),
-             LeftoverBatch.GetData());
-
-   // calculate how many empty slots are left in fPrimaryLeftoverBatch
-   std::size_t PrimaryLeftoverSize = fPrimaryLeftoverBatch->GetRows();
-   std::size_t emptySlots = fBatchSize - PrimaryLeftoverSize;
-
-   // copy LeftoverBatch to end of fPrimaryLeftoverBatch
-   if (emptySlots >= leftoverBatchSize) {
-      fPrimaryLeftoverBatch->Resize(PrimaryLeftoverSize + leftoverBatchSize, numCols);
-      std::copy(LeftoverBatch.GetData(), LeftoverBatch.GetData() + (leftoverBatchSize * fNumColumns),
-                fPrimaryLeftoverBatch->GetData() + (PrimaryLeftoverSize * numCols));
-
-      // copy LeftoverBatch to end of fPrimaryLeftoverBatch and add it to the batch
-      if (emptySlots == leftoverBatchSize) {
-         auto copy = std::make_unique<RFlat2DMatrix>(fBatchSize, fNumColumns);
-         std::copy(fPrimaryLeftoverBatch->GetData(), fPrimaryLeftoverBatch->GetData() + (fBatchSize * fNumColumns),
-                   copy->GetData());
-         batches.emplace_back(std::move(copy));
-
-         // reset fPrimaryLeftoverBatch and fSecondaryLeftoverBatch
-         *fPrimaryLeftoverBatch = *fSecondaryLeftoverBatch;
-         fSecondaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
-      }
-   }
+   // append the remaining entries of the chunk to fPrimaryLeftoverBatch, and add it to the batches once it is full
+   for (std::size_t row = beginRow + numBatches * fBatchSize; row < endRow;) {
+      const std::size_t numRows = std::min(endRow - row, fBatchSize - fPrimaryLeftoverBatch->GetRows());
+      fPrimaryLeftoverBatch->Reserve(fBatchSize, fNumColumns);
+      RFlat2DMatrixOperators::AppendRows(*fPrimaryLeftoverBatch, chunkTensor, permutation, row, row + numRows);
+      row += numRows;
 
-   // copy LeftoverBatch to both fPrimaryLeftoverBatch and fSecondaryLeftoverBatch
-   else if (emptySlots < leftoverBatchSize) {
-      // copy the first part of LeftoverBatch to end of fPrimaryLeftoverTrainingBatch
-      fPrimaryLeftoverBatch->Resize(fBatchSize, numCols);
-      std::copy(LeftoverBatch.GetData(), LeftoverBatch.GetData() + (emptySlots * numCols),
-                fPrimaryLeftoverBatch->GetData() + (PrimaryLeftoverSize * numCols));
-
-      // copy the last part of LeftoverBatch to the end of fSecondaryLeftoverBatch
-      fSecondaryLeftoverBatch->Resize(leftoverBatchSize - emptySlots, numCols);
-      std::copy(LeftoverBatch.GetData() + (emptySlots * numCols),
-                LeftoverBatch.GetData() + (leftoverBatchSize * numCols), fSecondaryLeftoverBatch->GetData());
-
-      // add fPrimaryLeftoverBatch to the batch vector
-      auto copy = std::make_unique<RFlat2DMatrix>(fBatchSize, fNumColumns);
-      std::copy(fPrimaryLeftoverBatch->GetData(), fPrimaryLeftoverBatch->GetData() + (fBatchSize * fNumColumns),
-                copy->GetData());
-      batches.emplace_back(std::move(copy));
-
-      // exchange fPrimaryLeftoverBatch and fSecondaryLeftoverBatch
-      *fPrimaryLeftoverBatch = *fSecondaryLeftoverBatch;
-      // reset fSecondaryLeftoverTrainingBatch
-      fSecondaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
+      if (fPrimaryLeftoverBatch->GetRows() == fBatchSize) {
+         batches.emplace_back(std::move(fPrimaryLeftoverBatch));
+         fPrimaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
+      }
    }
 
-   // copy the content of fPrimaryLeftoverBatch to the leftover batch from the chunk
+   // add the content of fPrimaryLeftoverBatch as the leftover batch of the last chunk
    if (isLastBatch) {
       if (!fDropRemainder && fLeftoverBatchSize > 0) {
-         auto copy = std::make_unique<RFlat2DMatrix>(fLeftoverBatchSize, fNumColumns);
-         std::copy(fPrimaryLeftoverBatch->GetData(),
-                   fPrimaryLeftoverBatch->GetData() + (fLeftoverBatchSize * fNumColumns), copy->GetData());
-         batches.emplace_back(std::move(copy));
+         fPrimaryLeftoverBatch->Resize(fLeftoverBatchSize, fNumColumns);
+         batches.emplace_back(std::move(fPrimaryLeftoverBatch));
       }
 
       fPrimaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
-      fSecondaryLeftoverBatch = std::make_unique<RFlat2DMatrix>();
    }
 
    {

@@ -63,7 +63,6 @@ private:
 
    // primary and secondary leftover batches used to create batches from a chunk
    std::unique_ptr<RFlat2DMatrix> fPrimaryLeftoverBatch;
-   std::unique_ptr<RFlat2DMatrix> fSecondaryLeftoverBatch;
 
 public:
    RBatchLoader(std::size_t batchSize, const std::vector<std::string> &cols, std::mutex &sharedMutex,
@@ -75,12 +74,15 @@ public:
    void Reset();
    void MarkProducerDone();
 
-   std::unique_ptr<RFlat2DMatrix> CreateBatch(RFlat2DMatrix &chunkTensor, std::size_t idxs);
+   std::unique_ptr<RFlat2DMatrix>
+   CreateBatch(const RFlat2DMatrix &chunkTensor, const std::vector<std::size_t> &permutation, std::size_t firstRow);
    RFlat2DMatrix GetBatch();
-   void CreateBatches(RFlat2DMatrix &chunkTensor, bool isLastBatch);
+   void CreateBatches(const RFlat2DMatrix &chunkTensor, const std::vector<std::size_t> &permutation,
+                      std::size_t beginRow, std::size_t endRow, bool isLastBatch);
    void RecalculateBatchCounts(std::size_t numEntries);
 
    bool isProducerDone() { return fProducerDone; }
+   std::size_t GetBatchSize() { return fBatchSize; }
    std::size_t GetNumBatches() { return fNumBatches; }
    std::size_t GetNumEntries() { return fNumEntries; }
    std::size_t GetNumRemainderRows() { return fLeftoverBatchSize; }

@@ -205,13 +205,14 @@ std::size_t RClusterLoader::LoadTrainingClusterInto(RFlat2DMatrix &dest, std::si
       }
 
       ROOT::RDF::RNode &rdf = fRdfs[rdfIdx];
+      const std::size_t numLoadedValues = dest.fRVec.size();
 
-      std::vector<ULong64_t> rdfEntries;
-      rdfEntries.reserve(endRow - startRow);
-
+      // load the whole cluster and capture rdfentry_ in the same event loop
       ROOT::Internal::RDF::ChangeBeginAndEndEntries(rdf, startRow, endRow);
-      rdf.Foreach([&](ULong64_t entry) { rdfEntries.push_back(entry); }, {"rdfentry_"});
+      auto entries = rdf.Take<ULong64_t>("rdfentry_");
+      ROOT::Internal::RDF::LoadCustomValues(rdf, fCols, fVecSizes, fVecPadding, dest.fRVec);
       ROOT::Internal::RDF::ChangeBeginAndEndEntries(rdf, 0, fRdfSizes[rdfIdx]);
+      auto &rdfEntries = *entries;
 
       const std::size_t totalFiltered = rdfEntries.size();
       if (totalFiltered == 0) {
@@ -262,8 +263,11 @@ std::size_t RClusterLoader::LoadTrainingClusterInto(RFlat2DMatrix &dest, std::si
       fAccumulatedFilteredForTrain += trainCount;
       fAccumulatedFilteredForVal += valCount;
 
-      if (trainCount > 0)
-         LoadClusterInto(dest, rdfIdx, trainStart, trainEnd);
+      // keep only the training rows of the cluster, the validation ones are loaded by the validation epoch
+      float *rows = dest.fRVec.data() + numLoadedValues;
+      if (!trainIsPrefix)
+         std::copy(rows + valCount * fNumChunkCols, rows + totalFiltered * fNumChunkCols, rows);
+      dest.fRVec.resize(numLoadedValues + trainCount * fNumChunkCols);
 
       return trainCount;
    }