CCDB benchmark (#23)

* CCDB benchmark classes and script
* Use truncate to delete ccdb data
* Improve CCDB backend for speed
* Add shield for the alibuild dashboard grafana
* Use new CCDB api to get all the objects names of a task

Author: Barthelemy

Framework/CMakeLists.txt

@@ -11,15 +11,9 @@ project(QualityControl)
 include(O2Utils)
 include(QualityControlDependencies)
 
-# TODO make it required, once in its own repo
-if(ROOT_FOUND)
-    message(STATUS "ROOT ${ROOT_VERSION} found in '${ROOTSYS}'")
-    set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${ROOTSYS}/share/root/cmake/modules)
-    include(RootNewMacros)
-else()
-    message(WARNING "ROOT not found, we won't compile the QC (skip, no error)")
-    return()
-endif()
+message(STATUS "ROOT ${ROOT_VERSION} found in '${ROOTSYS}'")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${ROOTSYS}/share/root/cmake/modules)
+include(RootNewMacros)
 
 # Set the default build type to "RelWithDebInfo"
 if(NOT CMAKE_BUILD_TYPE)
@@ -78,6 +72,7 @@ set(SRCS
         src/TaskDataProcessor.cxx
         src/TaskDataProcessorFactory.cxx
         src/TaskInterfaceDPL.cxx
+        src/CcdbBenchmark.cxx
         )
 
 set(HEADERS # needed for the dictionary generation
@@ -170,45 +165,48 @@ O2_GENERATE_EXECUTABLE(
         BUCKET_NAME ${BUCKET_NAME}
 )
 
-if (FAIRROOT_FOUND)
-    O2_GENERATE_EXECUTABLE(
-            EXE_NAME alfaTestReceiver
-            SOURCES src/alfaTestReceiver.cxx
-            MODULE_LIBRARY_NAME ${LIBRARY_NAME}
-            BUCKET_NAME ${BUCKET_NAME}
-    )
-
-    O2_GENERATE_EXECUTABLE(
-            EXE_NAME qcConsumer
-            SOURCES src/Consumer.cxx
-            MODULE_LIBRARY_NAME ${LIBRARY_NAME}
-            BUCKET_NAME ${BUCKET_NAME}
-    )
-endif()
+O2_GENERATE_EXECUTABLE(
+        EXE_NAME alfaTestReceiver
+        SOURCES src/alfaTestReceiver.cxx
+        MODULE_LIBRARY_NAME ${LIBRARY_NAME}
+        BUCKET_NAME ${BUCKET_NAME}
+)
 
-if(ROOT_FOUND) # TODO remove this once we have our own repo
-    set(TEST_SRCS
-            test/testDbFactory.cxx
-            test/testMonitorObject.cxx
-            test/testPublisher.cxx
-            test/testQcInfoLogger.cxx
-            test/testQCTask.cxx
-            test/testQuality.cxx
-            )
-
-    O2_GENERATE_TESTS(
-            MODULE_LIBRARY_NAME ${LIBRARY_NAME}
-            BUCKET_NAME ${BUCKET_NAME}
-            TEST_SRCS ${TEST_SRCS}
-    )
-
-    O2_GENERATE_EXECUTABLE(
-        EXE_NAME "tobject2json"
-        SOURCES ${SRCS_TOBJECT2JSON}
+O2_GENERATE_EXECUTABLE(
+        EXE_NAME qcConsumer
+        SOURCES src/Consumer.cxx
         MODULE_LIBRARY_NAME ${LIBRARY_NAME}
-        BUCKET_NAME "o2_qc_tobject2json"
-    )
-endif()
+        BUCKET_NAME ${BUCKET_NAME}
+)
+
+O2_GENERATE_EXECUTABLE(
+        EXE_NAME ccdbBenchmark
+        SOURCES src/runCcdbBenchmark.cxx
+        MODULE_LIBRARY_NAME ${LIBRARY_NAME}
+        BUCKET_NAME ${BUCKET_NAME}
+)
+
+set(TEST_SRCS
+        test/testDbFactory.cxx
+        test/testMonitorObject.cxx
+        test/testPublisher.cxx
+        test/testQcInfoLogger.cxx
+        test/testQCTask.cxx
+        test/testQuality.cxx
+        )
+
+O2_GENERATE_TESTS(
+        MODULE_LIBRARY_NAME ${LIBRARY_NAME}
+        BUCKET_NAME ${BUCKET_NAME}
+        TEST_SRCS ${TEST_SRCS}
+)
+
+O2_GENERATE_EXECUTABLE(
+    EXE_NAME "tobject2json"
+    SOURCES ${SRCS_TOBJECT2JSON}
+    MODULE_LIBRARY_NAME ${LIBRARY_NAME}
+    BUCKET_NAME "o2_qc_tobject2json"
+)
 
 # Install extra scripts
 install(PROGRAMS script/qcDatabaseSetup.sh DESTINATION bin)

Framework/example-default.json

@@ -9,7 +9,7 @@
         "password": "qc_user",
         "name": "quality_control",
         "implementation": "CCDB",
-        "host": "localhost:8080"
+        "host": "ccdb-test.cern.ch:8080"
       },
       "Activity": {
         "number": "42",

Framework/include/QualityControl/CcdbDatabase.h

@@ -19,6 +19,7 @@
 #include <ctime>
 #include <chrono>
 #include "QualityControl/DatabaseInterface.h"
+#include <curl/curl.h>
 
 namespace o2 {
 namespace quality_control {
@@ -60,14 +61,37 @@ class CcdbDatabase : public DatabaseInterface
     void prepareTaskDataContainer(std::string taskName) override;
     std::vector<std::string> getListOfTasksWithPublications() override;
     std::vector<std::string> getPublishedObjectNames(std::string taskName) override;
+    /**
+     * Delete all the versions of the specified object.
+     * @param taskName
+     * @param objectName
+     */
+    void truncateObject(std::string taskName, std::string objectName);
+    /**
+     * Delete the matching version of this object.
+     * @todo Raise an exception if no such object exist.
+     * @param taskName
+     * @param objectName
+     * @param timestamp
+     */
+    void deleteObjectVersion(std::string taskName, std::string objectName, std::string timestamp);
 
   private:
     long getCurrentTimestamp();
     std::string getTimestampString(long timestamp);
     long getFutureTimestamp(int secondsInFuture);
-    std::string getListing(std::string subpath = "");
+    /**
+     * Return the listing of folder and/or objects in the subpath.
+     * @param subpath The folder we want to list the children of.
+     * @param accept The format of the returned string as an \"Accept\", i.e. text/plain, application/json, text/xml
+     * @return The listing of folder and/or objects in the format requested and as returned by the http server.
+     */
+    std::string getListing(std::string subpath = "", std::string accept = "text/plain");
 
-    std::string url;
+    std::string mUrl;
+    CURL *mCurl;
+    CURLM *mMultiHandle;
+    void curlInit();
 };
 
 }

Framework/script/ccdb_benchmark.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -e ;# exit on error
+set -u ;# exit when using undeclared variable
+#set -x ;# debugging
+
+### Notes
+# FOR THE TIME BEING IT RUNS EVERYTHING LOCALLY
+# One must have ssh keys to connect to all hosts.
+
+### Define matrix of tests
+NB_OF_TASKS=(1 10) ;#1 2 5 10 25 50 100);
+NB_OF_OBJECTS=(10 100);# 10 100 1000);
+SIZE_OBJECTS=(1);# 10 100 1000);# in kB
+
+### Misc variables
+# The log prefix will be followed by the benchmark description, e.g. 1 task 1 checker... or an id or both
+LOG_FILE_PREFIX=/tmp/logCcdbBenchmark_
+NUMBER_CYCLES=60 ;# 180 ;# 1 sec per cycle -> ~ 3 minutes
+NODES=(
+#"aldaqci@aidrefflp01"
+"ccdb@barth-ccdb-606a6b90-1d83-48d5-8e46-ca72a63fc586"
+"ccdb@barth-ccdb-6f859d93-034c-4151-a4c8-571be0fe90f5"
+"ccdb@barth-ccdb-c518e99c-c05c-4066-a5dd-63613755985f"
+"ccdb@barth-ccdb-1394cef1-a627-4cf0-944f-40bf63f62ce1"
+"ccdb@barth-ccdb-a53241f0-f9f5-4862-8bca-66c081f7bd14"
+) ;# space delimited
+
+
+### Utility functions
+# Start a task in the background
+# \param 1 : host
+# \param 2 : name of the task
+# \param 3 : log file suffix
+# \param 4 : size of the objects
+# \param 5 : number of objects
+# \param 6 : number of tasks
+function startTask {
+  host=$1
+  name=$2
+  log_file_suffix=$3
+  size_objects=$4
+  number_objects=$5
+  number_tasks=$6
+  log_file_name=${LOG_FILE_PREFIX}${log_file_suffix}.log
+  echo "Starting task ${name} on host ${host}, logs in ${log_file_name}"
+  cmd="cd alice ; alienv setenv --no-refresh QualityControl/latest -c ccdbBenchmark --max-iterations ${NUMBER_CYCLES} \
+        --id ${name} --mq-config ~/alice/QualityControl/Framework/alfa.json --number-tasks ${number_tasks}\
+        --delete 0 --control static --size-objects ${size_objects} --number-objects ${number_objects} \
+        --monitoring-url influxdb-udp://aido2mon-gpn.cern.ch:8087 --task-name ${name} > ${log_file_name} 2>&1 "
+  echo "ssh ${host} \"${cmd}\" &"
+  ssh ${host} "${cmd}" &
+  pidLastTask=$!
+}
+
+# Kill all processes named "name"
+# \param 1 : name
+# \param 2 : host
+# \param 3 : extra flag for killall
+function killAll {
+    name=$1
+    host=$2
+    extra=${3:-""}
+    echo "Killing all processes called $name on $host"
+    ssh ${host} "killall ${extra} ${name}> /dev/null 2>&1 || true" ;# ignore errors
+}
+
+# Delete the database content
+# \param 1 : number of tasks
+function cleanDatabase {
+  number_tasks=$1
+  for (( task=0; task<$nb_tasks; task++ )); do
+    name=benchmarkTask_${task}
+    cmd="ccdbBenchmark --id test --mq-config ~/dev/alice/QualityControl/Framework/alfa.json --delete 1 --control static \
+         --task-name ${name}" ;# > /dev/null 2>&1"
+    echo ${cmd}
+    eval ${cmd}
+  done
+}
+
+
+### Benchmark starts here
+# Loop through the matrix of tests
+for nb_tasks in ${NB_OF_TASKS[@]}; do
+  for nb_objects in ${NB_OF_OBJECTS[@]}; do
+    for size_objects in ${SIZE_OBJECTS[@]}; do
+      echo "*************************** $(date)
+      Launching test for $nb_tasks tasks, $nb_objects objects, $size_objects kB objects"
+
+      echo "Kill all old processes"
+      for machine in ${NODES[@]}; do
+        killAll "ccdbBenchmark" ${machine} "-9"
+      done
+
+#      echo "Delete database content"
+#      cleanDatabase $nb_tasks
+
+      echo "Now start the tasks"
+      for (( task=0; task<$nb_tasks; task++ )); do
+        echo "*** ~~~ *** Start task"
+
+        # select a node for this task : task % #node -> index of node
+        node_index=$((task%${#NODES[@]}))
+
+        startTask "${NODES[$node_index]}" benchmarkTask_${task} \
+                  "benchmarkTask_${task}_${nb_tasks}_${nb_objects}_${size_objects}" \
+                  ${size_objects} ${nb_objects} ${nb_tasks}
+        TASKS_PIDS+=($pidLastTask)
+      done
+
+      echo "Now wait for the tasks to finish "
+      wait ${TASKS_PIDS[*]};# the checker never stops, we can't just wait
+
+      sleep 5 # leave time to finish
+
+      for machine in ${NODES[@]}; do
+        killAll "ccdbBenchmark" ${machine} "-9"
+      done
+
+      sleep 5 # leave time to finish
+
+      TASKS_PIDS=()
+
+      echo "OK, ready for the next round !"
+
+    done
+  done
+done
+