[QC-803] Trend only if all input objects are available (#2397)

* added config for trendingTask that enable trending only if all sources are available

* added test and proper parsing

* fixup! added test and proper parsing

---------

Co-authored-by: Michal Tichák <michal.tichak@cern.ch>

Author: justonedev1

Framework/include/QualityControl/TrendingTask.h

@@ -70,7 +70,8 @@ class TrendingTask : public PostProcessingInterface
   static void formatRunNumberXAxis(TH1* background);
   static std::string deduceGraphLegendOptions(const TrendingTaskConfig::Graph& graphConfig);
 
-  void trendValues(const Trigger& t, repository::DatabaseInterface&);
+  /// returns true only if all datasources were available to update reductor
+  bool trendValues(const Trigger& t, repository::DatabaseInterface&);
   void generatePlots();
   TCanvas* drawPlot(const TrendingTaskConfig::Plot& plotConfig);
   void initializeTrend(repository::DatabaseInterface& qcdb);

Framework/include/QualityControl/TrendingTaskConfig.h

@@ -61,6 +61,7 @@ struct TrendingTaskConfig : PostProcessingConfig {
 
   bool producePlotsOnUpdate{};
   bool resumeTrend{};
+  bool trendIfAllInputs{ false };
   std::vector<Plot> plots;
   std::vector<DataSource> dataSources;
 };

Framework/src/TrendingTask.cxx

@@ -158,8 +158,8 @@ void TrendingTask::update(Trigger t, framework::ServiceRegistryRef services)
 {
   auto& qcdb = services.get<repository::DatabaseInterface>();
 
-  trendValues(t, qcdb);
-  if (mConfig.producePlotsOnUpdate) {
+  const auto allSourcesInvoked = trendValues(t, qcdb);
+  if (mConfig.producePlotsOnUpdate && (!mConfig.trendIfAllInputs || allSourcesInvoked)) {
     generatePlots();
   }
 }
@@ -172,22 +172,28 @@ void TrendingTask::finalize(Trigger, framework::ServiceRegistryRef)
   generatePlots();
 }
 
-void TrendingTask::trendValues(const Trigger& t, repository::DatabaseInterface& qcdb)
+bool TrendingTask::trendValues(const Trigger& t, repository::DatabaseInterface& qcdb)
 {
   mTime = activity_helpers::isLegacyValidity(t.activity.mValidity)
             ? t.timestamp / 1000
             : t.activity.mValidity.getMax() / 1000; // ROOT expects seconds since epoch.
   mMetaData.runNumber = t.activity.mId;
+  bool wereAllSourcesInvoked = true;
 
   for (auto& dataSource : mConfig.dataSources) {
     if (!reductor_helpers::updateReductor(mReductors[dataSource.name].get(), t, dataSource, qcdb, *this)) {
+      wereAllSourcesInvoked = false;
       ILOG(Error, Support) << "Failed to update reductor for data sources with path '" << dataSource.path
                            << "', name '" << dataSource.name
                            << "', type '" << dataSource.type << "'." << ENDM;
     }
   }
 
-  mTrend->Fill();
+  if (!mConfig.trendIfAllInputs || wereAllSourcesInvoked) {
+    mTrend->Fill();
+  }
+
+  return wereAllSourcesInvoked;
 }
 
 void TrendingTask::setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels)

Framework/src/TrendingTaskConfig.cxx

@@ -25,6 +25,8 @@ TrendingTaskConfig::TrendingTaskConfig(std::string id, const boost::property_tre
 {
   producePlotsOnUpdate = config.get<bool>("qc.postprocessing." + id + ".producePlotsOnUpdate", true);
   resumeTrend = config.get<bool>("qc.postprocessing." + id + ".resumeTrend", false);
+  trendIfAllInputs = config.get<bool>("qc.postprocessing." + id + ".trendIfAllInputs", false);
+
   for (const auto& [_, plotConfig] : config.get_child("qc.postprocessing." + id + ".plots")) {
     // since QC-1155 we allow for more than one graph in a single plot (canvas). we support both the new and old ways
     // of configuring the expected plots.

Framework/test/testTrendingTask.cxx

@@ -78,6 +78,7 @@ TEST_CASE("test_trending_task")
         "active": "true",
         "taskName": "TestTrendingTask",
         "className": "o2::quality_control::postprocessing::TrendingTask",
+        "trendIfAllInputs": true,
         "moduleName": "QualityControl",
         "detectorName": "TST",
         "dataSources": [
@@ -133,6 +134,35 @@ TEST_CASE("test_trending_task")
   repository->truncate("qc/TST/MO/" + taskName, "*");
   repository->truncate("qc/TST/QO", checkName);
 
+  // Test "trendIfAllInputs". There should not be anything in DB so we don't have any input sources available
+  {
+    auto objectManager = std::make_shared<ObjectsManager>(taskName, "o2::quality_control::postprocessing::TrendingTask", "TST", "");
+    ServiceRegistry services;
+    services.registerService<DatabaseInterface>(repository.get());
+
+    TrendingTask task;
+    task.setName(trendingTaskName);
+    task.setID(trendingTaskID);
+    task.setObjectsManager(objectManager);
+    REQUIRE_NOTHROW(task.configure(config));
+
+    // test initialize()
+    REQUIRE_NOTHROW(task.initialize({ TriggerType::UserOrControl, true, { 0, "NONE", "", "", "qc" }, 1 }, services));
+    REQUIRE(objectManager->getNumberPublishedObjects() == 1);
+    auto treeMO = objectManager->getMonitorObject(trendingTaskName);
+    REQUIRE(treeMO != nullptr);
+    TTree* tree = dynamic_cast<TTree*>(treeMO->getObject());
+    REQUIRE(tree != nullptr);
+    REQUIRE(tree->GetEntries() == 0);
+
+    // test update()
+    task.update({ TriggerType::NewObject, false, { 0, "NONE", "", "", "qc", { 2, 100000 } }, 100000 - 1 }, services);
+    objectManager->stopPublishing(PublicationPolicy::Once);
+    task.update({ TriggerType::NewObject, false, { 0, "NONE", "", "", "qc", { 100000, 200000 } }, 200000 - 1 }, services);
+    REQUIRE(objectManager->getNumberPublishedObjects() == 1);
+    REQUIRE(tree->GetEntries() == 0);
+  }
+
   // Putting the objects to trend into the database
   {
     TH1I* histo = new TH1I("testHistoTrending", "testHistoTrending", 10, 0, 10.0);
@@ -220,4 +250,4 @@ TEST_CASE("test_trending_task")
   REQUIRE(tree->GetEntries() == 2);
   objectManager->stopPublishing(PublicationPolicy::Once);
   objectManager->stopPublishing(PublicationPolicy::ThroughStop);
-}
+}

doc/PostProcessing.md

@@ -389,6 +389,8 @@ use the boolean flag `"producePlotsOnUpdate"`.
 
 To pick up the last existing trend which matches the specified Activity, set `"resumeTrend"` to `"true"`.
 
+To generate plots only when all input objects are available, set `"trendIfAllInputs"`.
+
 ### The SliceTrendingTask class
 The `SliceTrendingTask` is a complementary task to the standard `TrendingTask`. This task allows the trending of canvas objects that hold multiple histograms (which have to be of the same dimension, e.g. TH1) and the slicing of histograms. The latter option allows the user to divide a histogram into multiple subsections along one or two dimensions which are trended in parallel to each other. The task has specific reductors for `TH1` and `TH2` objects which are `o2::quality_control_modules::common::TH1SliceReductor` and `o2::quality_control_modules::common::TH2SliceReductor`.