diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index 0ffc69e9c..c6fdb2a01 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -190,6 +190,9 @@ def add(cfg, flatconfig): if not getattr(args, "with_ZDC", False) and "ZDC" not in getattr(args, "skipModules", []): add(config, {"ZDCSimParam.buildBeamLine" : "false"}) add(config, {"GenTPCLoopers.colsys" : args.col}) + # MC-DATA embedding simulates small signals at the data collisions; no TPC loopers + if len(getattr(args, "data_anchoring", "")) > 0: + add(config, {"GenTPCLoopers.loopersVeto" : "true"}) # ----- apply external overwrites from command line ------- for keyval in externalConfigString.split(";"): @@ -199,6 +202,15 @@ def add(cfg, flatconfig): return config +def qed_included(pdga, pdgb, with_qed, no_qed, data_anchoring): + """ + Whether to add the QED background. On by default for same-species heavy-ion collisions, + off for MC-DATA embedding (the data already carries it). --with-qed forces it on, + --no-qed forces it off and wins over everything. + """ + default = pdga == pdgb and pdga != 2212 and len(data_anchoring) == 0 + return (default or with_qed) and not no_qed + def constructConfigKeyArg(config): # flattens dictionary constructed in create_geant_config # and constructs the --configKeyValues options for simulation diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index b1c44b7d2..c69eb004b 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -43,7 +43,7 @@ from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector, compute_n_workers, merge_dicts from o2dpg_qc_finalization_workflow import include_all_QC_finalization -from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg, option_if_available, overwrite_config +from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg, option_if_available, overwrite_config, qed_included from o2dpg_dpl_config_tools import dpl_option_from_config, TaskFinalizer, quote_if_needed # for some JAliEn interaction @@ -643,11 +643,9 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True): workflow['stages'].append(GRP_TASK) -# QED background is enabled by default only for same-species (heavy-ion) collisions. -# --with-qed forces it on; --no-qed forces it off and wins over both the default and --with-qed -# (the two flags are mutually exclusive at the argparse level, so they can never both be set). -QED_enabled = True if (PDGA==PDGB and PDGA!=2212) else False -includeQED = (QED_enabled or args.with_qed) and not args.no_qed +# QED background: on by default for same-species heavy-ion collisions, off for MC-DATA embedding; +# --with-qed forces it on, --no-qed forces it off (see qed_included) +includeQED = qed_included(PDGA, PDGB, args.with_qed, args.no_qed, args.data_anchoring) signalprefix='sgn' # No vertexing for event pool generation; otherwise the vertex comes from CCDB and later from CollContext @@ -714,6 +712,10 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True): ' --nontrivial-mu-distribution ccdb://https://alice-ccdb.cern.ch/FT0/Calib/EventsPerBc', f'--QEDinteraction {qedspec}' if includeQED else None ], configname = 'precollcontext') +if len(args.data_anchoring) > 0: + # contexts taken from data carry no interaction rate; store the one used for this production + PreCollContextTask['cmd'] += ' && python3 ${O2DPG_ROOT}/MC/utils/o2dpg_data_embedding_utils.py --set-interaction-rate ' + str(INTRATE) \ + + ' --context ' + ' '.join(f'tf{tf}/collisioncontext.root' for tf in range(1, NTIMEFRAMES + 1)) workflow['stages'].append(PreCollContextTask) if doembedding: diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 56d11a0c9..36810dfa5 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -341,6 +341,16 @@ def shift_anchor_past_ITS_rampup(run_start, first_orbit, orbitsPerTF, ITS_rampup rampupOrbits = -(-milliseconds_to_orbits(math.ceil(ITS_rampup)) // orbitsPerTF) * orbitsPerTF return run_start + int(rampupOrbits * LHCOrbitMUS / 1000.), first_orbit + rampupOrbits +def anchor_origin(run_start, first_orbit, orbitsPerTF, ITS_rampup, explicit_timeframe): + """ + Returns the (start of run in ms, first orbit) from which timeframes are counted. + An explicit timeframe ID (MC-DATA embedding) names a timeframe of the data, counted + from the run's first orbit, so it is not moved past the ITS ramp-up. + """ + if explicit_timeframe: + return run_start, first_orbit + return shift_anchor_past_ITS_rampup(run_start, first_orbit, orbitsPerTF, ITS_rampup) + def retrieve_MinBias_CTPScaler_Rate(raw_rate_at, finaltime, trig_eff_arg, NBunches, ColSystem, eCM, run_number = -1): """ Turns the raw CTP counting rate at finaltime (in milliseconds) into the interaction rate for @@ -506,9 +516,26 @@ def determine_timestamp_from_timeframeID(sor, eor, timeframeID, HBF_per_timefram # this is a closure test. If we had perfect floating point precision everywhere, it wouldn't fail. # But since we don't have that and there are some int casts as well, better check again. assert (timestamp_of_production >= sor) - assert (timestamp_of_production <= eor) + # no upper bound: a timeframe taken from data may start after the end of run recorded in CCDB return int(timestamp_of_production) +def conditions_timestamp(timestamp, eor): + """ + The timestamp for condition lookups of a timeframe taken from data. A timeframe starting after + the recorded end of run keeps its position in the run, but its conditions are read just before + the end of run, where the run's objects are still valid. + """ + return min(timestamp, eor - 1) + +def rate_lookup_time(timestamp, run_start, collisions_end, explicit_timeframe): + """ + The time at which to read the CTP interaction rate. For a timeframe taken from data, which + may lie past the last CTP scaler record, this is clamped to just before the collisions end. + """ + if explicit_timeframe: + return max(run_start, min(timestamp, collisions_end - 1000)) + return timestamp + def exclude_timestamp(ts, orbit, run, filename, global_run_params): """ Checks if timestamp ts (or orbit) falls within a bad data period. @@ -614,7 +641,7 @@ def main(): # Adjust the anchoring point using the ITS ramp-up period ITS_rampup = retrieve_ITS_RampDuration(ccdbreader, run_start) print(f"ITS ramp-up time: {ITS_rampup} ms") - effective_run_start, effective_first_orbit = shift_anchor_past_ITS_rampup(run_start, GLOparams["FirstOrbit"], GLOparams["OrbitsPerTF"], ITS_rampup) + effective_run_start, effective_first_orbit = anchor_origin(run_start, GLOparams["FirstOrbit"], GLOparams["OrbitsPerTF"], ITS_rampup, args.timeframeID != -1) mid_run_timestamp = (effective_run_start + run_end) // 2 # -------- @@ -685,9 +712,10 @@ def main(): timestamp = 0 prod_offset = 0 if args.timeframeID != -1: - # an explicit timeframe is a fixed position in the run, so bound it by the run as recorded; - # if it falls past the collisions the rate below is zero and the job is excluded + # an explicit timeframe comes from data: it exists even past the recorded end of run, + # and its interaction rate is read at the last time the CTP counted collisions timestamp = determine_timestamp_from_timeframeID(effective_run_start, GLOparams["EOR"], args.timeframeID, GLOparams["OrbitsPerTF"]) + timestamp = conditions_timestamp(timestamp, GLOparams["EOR"]) prod_offset = args.timeframeID else: timestamp, prod_offset = determine_timestamp(effective_run_start, run_end, [args.split_id - 1, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) @@ -720,7 +748,8 @@ def main(): ctp_local_rate_raw = None if args.ccdb_IRate == True: - rate, ctp_local_rate_raw = retrieve_MinBias_CTPScaler_Rate(raw_rate_at, timestamp, args.trig_eff, grplhcif.getBunchFilling().getNBunches(), ColSystem, eCM, args.run_number) + rate_time = rate_lookup_time(timestamp, effective_run_start, run_end, args.timeframeID != -1) + rate, ctp_local_rate_raw = retrieve_MinBias_CTPScaler_Rate(raw_rate_at, rate_time, args.trig_eff, grplhcif.getBunchFilling().getNBunches(), ColSystem, eCM, args.run_number) # no collisions here, so nothing to simulate: treat it like any other excluded timestamp if rate == None and ctp_local_rate_raw == 0: diff --git a/MC/bin/tests/test_anchoring_timeframeid.py b/MC/bin/tests/test_anchoring_timeframeid.py new file mode 100644 index 000000000..4c5f758e4 --- /dev/null +++ b/MC/bin/tests/test_anchoring_timeframeid.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Offline test: an explicit timeframe ID is placed from the unshifted run start (MC-DATA embedding).""" +import os +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) + +os.environ.setdefault("O2DPG_ROOT", os.path.dirname(os.path.dirname(os.path.dirname(HERE)))) + +try: + import o2dpg_sim_workflow_anchored as anchored # needs ROOT +except ImportError as exc: # pragma: no cover + anchored = None + +SOR = 1778806732526 +FIRST_ORBIT = 20505888 +ORBITS_PER_TF = 32 +ITS_RAMPUP_MS = 5000 + + +@unittest.skipIf(anchored is None, "o2dpg_sim_workflow_anchored not importable (needs ROOT)") +class TestTimeframeIdAnchor(unittest.TestCase): + + def test_explicit_timeframe_ignores_rampup_shift(self): + start, orbit = anchored.anchor_origin(SOR, FIRST_ORBIT, ORBITS_PER_TF, ITS_RAMPUP_MS, explicit_timeframe=True) + self.assertEqual((start, orbit), (SOR, FIRST_ORBIT)) + # the MC timeframe tf then starts where the data timeframe tf starts + tf = 10 + self.assertEqual(orbit + tf * ORBITS_PER_TF, FIRST_ORBIT + 320) + + def test_sampled_timeframe_keeps_rampup_shift(self): + self.assertEqual(anchored.anchor_origin(SOR, FIRST_ORBIT, ORBITS_PER_TF, ITS_RAMPUP_MS, explicit_timeframe=False), + anchored.shift_anchor_past_ITS_rampup(SOR, FIRST_ORBIT, ORBITS_PER_TF, ITS_RAMPUP_MS)) + + +@unittest.skipIf(anchored is None, "o2dpg_sim_workflow_anchored not importable (needs ROOT)") +class TestEndOfRun(unittest.TestCase): + """Data timeframes can lie a little past the end of run recorded in CCDB.""" + EOR = SOR + 2232696 # run 544653: last data TF starts about 0.6 s after EOR + + def test_explicit_timeframe_past_eor_is_accepted(self): + tf = 784641 + ts = anchored.determine_timestamp_from_timeframeID(SOR, self.EOR, tf, ORBITS_PER_TF) + self.assertGreater(ts, self.EOR) + + def test_rate_lookup_time_clamped_for_explicit_timeframe(self): + collisions_end = self.EOR - 500 + self.assertEqual(anchored.rate_lookup_time(self.EOR + 600, SOR, collisions_end, True), collisions_end - 1000) + self.assertEqual(anchored.rate_lookup_time(SOR + 5000, SOR, collisions_end, True), SOR + 5000) + + def test_conditions_timestamp_clamped_to_run(self): + self.assertEqual(anchored.conditions_timestamp(self.EOR + 68, self.EOR), self.EOR - 1) + self.assertEqual(anchored.conditions_timestamp(self.EOR - 5000, self.EOR), self.EOR - 5000) + + def test_rate_lookup_time_unchanged_for_sampled(self): + self.assertEqual(anchored.rate_lookup_time(self.EOR + 600, SOR, self.EOR, False), self.EOR + 600) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/bin/tests/test_sim_config_embedding.py b/MC/bin/tests/test_sim_config_embedding.py new file mode 100644 index 000000000..87ee44ae8 --- /dev/null +++ b/MC/bin/tests/test_sim_config_embedding.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Offline tests: MC-DATA embedding switches the TPC looper generator and the QED background off.""" +import argparse +import os +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) + +import o2dpg_sim_config as simconfig + + +def args(**kw): + base = dict(gen="pythia8pp", col="PbPb", with_ZDC=False, skipModules=[], data_anchoring="") + base.update(kw) + return argparse.Namespace(**base) + + +class TestLoopersVeto(unittest.TestCase): + + def test_data_anchoring_vetoes_loopers(self): + cfg = simconfig.create_geant_config(args(data_anchoring="ctx.root"), "") + self.assertEqual(cfg["GenTPCLoopers"].get("loopersVeto"), "true") + + def test_ordinary_production_keeps_loopers(self): + cfg = simconfig.create_geant_config(args(), "") + self.assertNotIn("loopersVeto", cfg["GenTPCLoopers"]) + + def test_explicit_key_wins(self): + cfg = simconfig.create_geant_config(args(data_anchoring="ctx.root"), "GenTPCLoopers.loopersVeto=false") + self.assertEqual(cfg["GenTPCLoopers"].get("loopersVeto"), "false") + + +class TestQED(unittest.TestCase): + PB = 1000822080 + + def test_default_on_for_heavy_ions_only(self): + self.assertTrue(simconfig.qed_included(self.PB, self.PB, False, False, "")) + self.assertFalse(simconfig.qed_included(2212, 2212, False, False, "")) + + def test_flags(self): + self.assertTrue(simconfig.qed_included(2212, 2212, True, False, "")) + self.assertFalse(simconfig.qed_included(self.PB, self.PB, False, True, "")) + + def test_data_anchoring_switches_qed_off(self): + self.assertFalse(simconfig.qed_included(self.PB, self.PB, False, False, "ctx.root")) + + def test_with_qed_wins_over_data_anchoring(self): + self.assertTrue(simconfig.qed_included(self.PB, self.PB, True, False, "ctx.root")) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/run/ANCHOR/anchorMC_DataEmbedding.sh b/MC/run/ANCHOR/anchorMC_DataEmbedding.sh index 8ceee6799..01d40fef7 100755 --- a/MC/run/ANCHOR/anchorMC_DataEmbedding.sh +++ b/MC/run/ANCHOR/anchorMC_DataEmbedding.sh @@ -129,6 +129,15 @@ export ALIEN_JDL_ADDTIMESERIESINMC=${ALIEN_JDL_ADDTIMESERIESINMC:-1} [ -z "${ALIEN_JDL_LPMANCHORPRODUCTION}" ] && { echo_error "Set ALIEN_JDL_LPMANCHORPRODUCTION or ANCHORPRODUCTION" ; exit 1 ; } [ -z "${ALIEN_JDL_LPMANCHORYEAR}" ] && { echo_error "Set ALIEN_JDL_LPMANCHORYEAR or ANCHORYEAR" ; exit 1 ; } +# MC-DATA embedding: one timeframe per data timeframe, placed by its ID, not by split +if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ] || [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then + NTIMEFRAMES=${NTIMEFRAMES:-1} + SPLITID=${SPLITID:-1} + PRODSPLIT=${PRODSPLIT:-1} + # the MC timeframes must have the length of the data timeframes + [ "${ALIEN_JDL_MC_ORBITS_PER_TF}" ] && { echo_error "ALIEN_JDL_MC_ORBITS_PER_TF cannot be used for MC-DATA embedding" ; exit 1 ; } +fi + [ -z "${NTIMEFRAMES}" ] && { echo_error "Set NTIMEFRAMES" ; exit 1 ; } [ -z "${SPLITID}" ] && { echo_error "Set SPLITID" ; exit 1 ; } [ -z "${PRODSPLIT}" ] && { echo_error "Set PRODSPLIT" ; exit 1 ; } @@ -339,6 +348,19 @@ else fi PUBLISH_MCPRODINFO_OPTION="" +EMBEDDING_DIR=${O2DPG_ROOT}/MC/run/ANCHOR/dataembedding +EMBEDDING_SELECTION="" +# with a work table, this job treats one row of it: a data AO2D and a set of whole DF folders +if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ]; then + ROWINFO=$(python3 ${EMBEDDING_DIR}/embedding_row.py --table ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} \ + --expect-out embedding_expected_tfs.json --manifest embedding_row.json) + [ $? != 0 ] && { echo_error "Could not determine the embedding row of this job" ; exit 1 ; } + read -r AOD_FROM_TABLE EMBEDDING_DFS <<< "${ROWINFO}" + export ALIEN_JDL_MC_DATA_EMBEDDING_AO2D=${AOD_FROM_TABLE} + EMBEDDING_SELECTION="--dfs ${EMBEDDING_DFS} --expect-tfs embedding_expected_tfs.json" + echo_info "Embedding into ${EMBEDDING_DFS} of ${AOD_FROM_TABLE}" +fi + # let's take the input data AO2D from a JDL variable AOD_DATA_FILE=${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then @@ -346,8 +368,34 @@ if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then NWORKERS=1 # these embedding jobs process only light pp type signals ... the parallelism will come via parallel workflows fi +# a limit would drop planned timeframes; without a table the old default of 8 timeframes stays +if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ]; then + [ "${DATA_EMBEDDING_LIMIT}" ] && { echo_error "DATA_EMBEDDING_LIMIT cannot be used with a work table" ; exit 1 ; } + EMBEDDING_LIMIT_OPTION="" +else + EMBEDDING_LIMIT_OPTION="--limit ${DATA_EMBEDDING_LIMIT:-8}" +fi + # call the python script to extract all collision contexts -python3 ${O2DPG_ROOT}/MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} --limit ${DATA_EMBEDDING_LIMIT:-8} +python3 ${O2DPG_ROOT}/MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} \ + ${EMBEDDING_LIMIT_OPTION} ${EMBEDDING_SELECTION} +CONTEXT_RC=$? +if [ "${CONTEXT_RC}" != "0" ]; then + echo_error "Extraction of the collision contexts failed with ${CONTEXT_RC}" + exit ${CONTEXT_RC} +fi +if ! compgen -G "collission_context_*.root" > /dev/null; then + echo_error "No collision context was extracted from ${AOD_DATA_FILE}" + exit 1 +fi + +# concurrent per-timeframe workflows; each runner gets its share of the CPU and memory of the job +EMBEDDING_PARALLEL=${ALIEN_JDL_MC_DATA_EMBEDDING_PARALLEL:-8} +RUNNER_CPU=$(( ${ALIEN_JDL_CPULIMIT:-8} / EMBEDDING_PARALLEL )) +[ "${RUNNER_CPU}" -lt 1 ] && RUNNER_CPU=1 +# ALICE GRID slots provide 2 GB per core unless told otherwise +RUNNER_MEM=$(( ${ALIEN_JDL_MC_DATA_EMBEDDING_MEMORY_MB:-$(( 2000 * ${ALIEN_JDL_CPULIMIT:-8} ))} / EMBEDDING_PARALLEL )) +echo_info "Running ${EMBEDDING_PARALLEL} timeframe workflows in parallel, each with ${RUNNER_CPU} CPU and ${RUNNER_MEM} MB" parallel_job_count=0 failed_count=0 @@ -364,14 +412,14 @@ for external_context in collission_context_*.root; do echo "Treating timeframe ${anchoring_tf} coming from folder ${df_folder}" # we do it in a separate workspace - workspace="TF_${anchoring_tf}" + workspace="${df_folder}_TF_${anchoring_tf}" mkdir "${workspace}"; cd "${workspace}" # fetch the apass reco anchoring config cp ../*.json . - # we need to adjust the SEED for each job - JOBSEED=$SEED - [ "$JOBSEED" != "-1" ] && let JOBSEED=JOBSEED+anchoring_tf + # seed per timeframe from the job id (SEED, normally ALIEN_PROC_ID), the DF folder and the timeframe + JOBSEED=$(python3 ${EMBEDDING_DIR}/embedding_plan_lib.py seed "${SEED}" "${df_folder}" "${anchoring_tf}") + [ -z "${JOBSEED}" ] && { echo_error "Could not compute the seed of timeframe ${anchoring_tf}" ; exit 1 ; } echo "TF ${anchoring_tf} got seed ${JOBSEED}" # these arguments will be digested by o2dpg_sim_workflow_anchored.py @@ -406,18 +454,29 @@ for external_context in collission_context_*.root; do anchoringLogFile=timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log # query CCDB has changed, w/o "_" - ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> ${anchoringLogFile} - WF_RC="${?}" + # a second attempt covers transient failures (CCDB, slow tools on a busy node) + for attempt in 1 2; do + ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> ${anchoringLogFile} + WF_RC="${?}" + [ "${WF_RC}" == "0" ] && break + echo_info "Workflow creation of timeframe ${anchoring_tf} failed with ${WF_RC} (attempt ${attempt})" + cp ${anchoringLogFile} ${anchoringLogFile}.attempt${attempt} + done if [ "${WF_RC}" != "0" ] ; then - echo_error "Problem during anchor timestamp sampling and workflow creation. Exiting." - exit ${WF_RC} + # count it as a failed timeframe and go on, so that the job still writes its status and logs + echo_error "Problem during anchor timestamp sampling and workflow creation of timeframe ${anchoring_tf} (${df_folder}):" + tail -n 20 ${anchoringLogFile} + ((failed_count++)) + cd .. + continue fi TIMESTAMP=`grep "Determined timestamp to be" ${anchoringLogFile} | awk '//{print $6}'` echo_info "TIMESTAMP IS ${TIMESTAMP}" if [ "${ONLY_WORKFLOW_CREATION}" ]; then - continue # or exit + cd .. + continue fi # check if this job is exluded because it falls inside a bad data-taking period @@ -426,7 +485,9 @@ for external_context in collission_context_*.root; do # we can quit here; there is nothing to do # (apart from maybe creating a fake empty AO2D.root file or the like) echo "Timestamp is excluded from run. Nothing to do here" - continue # or exit 0 + touch EXCLUDED # the final merge reports it as excluded, not as failed + cd .. + continue fi # -- RUN THE MC WORKLOAD TO PRODUCE TARGETS -- @@ -459,13 +520,13 @@ for external_context in collission_context_*.root; do echo_info "Workflow will run with target specification ${targetString}" ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${targetString} -j 1 \ - --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} --dynamic-resources \ + --cpu-limit ${RUNNER_CPU} --mem-limit ${RUNNER_MEM} --optimistic-resources --dynamic-resources --retry-on-failure ${ALIEN_JDL_MC_DATA_EMBEDDING_RETRIES:-2} \ ${ALIEN_O2DPG_FILEGRAPH:+--remove-files-early ${ALIEN_O2DPG_FILEGRAPH}} \ ${ALIEN_O2DPG_ADDITIONAL_WORKFLOW_RUNNER_ARGS} &> pipeline_log & ((parallel_job_count++)) # If limit reached, wait for one job to finish - if ((parallel_job_count >= 8)); then + if ((parallel_job_count >= EMBEDDING_PARALLEL)); then if ! wait -n; then ((failed_count++)) fi @@ -481,10 +542,40 @@ while ((parallel_job_count > 0)); do ((parallel_job_count--)) done -if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then +MERGE_RC=0 +missing_aods=0 +# only the default targets produce an AO2D per timeframe +if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ] && [ -z "${ONLY_WORKFLOW_CREATION}" ] && [ -z "${ALIEN_JDL_O2DPGWORKFLOWTARGET}" ]; then # produce the final merged AO2D - find ./ -maxdepth 2 -mindepth 2 -name "AO2D.root" > aod_inputs.txt - o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name # --merge-by-name restricts merging to folders of same name + # sort by DF folder, then numerically by timeframe (from collission_context_:.root), + # so --merge-by-name sees same-named folders one after another and timeframes stay in time order + : > embedding_status.tsv + : > aod_unsorted.tsv + for ctx in collission_context_*.root; do + name="${ctx#collission_context_}" + df="${name%%:*}" + tf="${name##*:}" + tf="${tf%.root}" + aod="./${df}_TF_${tf}/AO2D.root" + if [ -f "./${df}_TF_${tf}/EXCLUDED" ]; then + printf '%s\t%s\texcluded\n' "${df}" "${tf}" >> embedding_status.tsv + elif [ -f "${aod}" ]; then + printf '%s\t%s\t%s\n' "${df}" "${tf}" "${aod}" >> aod_unsorted.tsv + printf '%s\t%s\tdone\n' "${df}" "${tf}" >> embedding_status.tsv + else + echo_error "No AO2D for timeframe ${tf} of ${df}" + printf '%s\t%s\tfailed\n' "${df}" "${tf}" >> embedding_status.tsv + ((missing_aods++)) + fi + done + sort -t $'\t' -k1,1 -k2,2n aod_unsorted.tsv | cut -f3 > aod_inputs.txt + if [ -s aod_inputs.txt ]; then + # a huge --max-size: a size-triggered flush would make the next folder of the same name collide + o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name --max-size 1000000000000 + MERGE_RC=$? + else + echo_info "No AO2D to merge: all timeframes of this job were excluded or failed" + fi fi # @@ -496,3 +587,9 @@ if [[ -n "$ALIEN_PROC_ID" ]]; then find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T - fi fi + +# a job with a failed timeframe fails as a whole, so that it is resubmitted +if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ] && (( failed_count > 0 || missing_aods > 0 || MERGE_RC != 0 )); then + echo_error "Embedding incomplete: ${failed_count} failed workflows, ${missing_aods} missing AO2Ds, merger exit code ${MERGE_RC}" + exit 1 +fi diff --git a/MC/run/ANCHOR/dataembedding/check_embedding_output.py b/MC/run/ANCHOR/dataembedding/check_embedding_output.py new file mode 100755 index 000000000..72b4afe4d --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/check_embedding_output.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +# +# Merges the outputs of an MC-DATA embedding production per parent data file and checks +# that each merged MC AO2D is correctly linked to its parent: +# - no DF folder appears twice, every planned DF is there and no other one; +# - every DF has a parentFiles entry pointing to the parent; +# - per DF, the MC truth collisions sit at the BCs of the data collisions, and as many. +# +# Examples: +# check_embedding_output.py merge --plan plan_544653 --outputs out_cycle0 out_cycle1 --out merged +# check_embedding_output.py check --plan plan_544653 --mc merged/0000/AO2D.root --parent alien:///.../AO2D.root + +import argparse +import collections +import glob +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "utils")) + +import embedding_plan_lib as lib + + +# ---- pure checks ------------------------------------------------------------------------------- + +def planned_dfs(table, lfn): + """DF folders the plan expects in the MC output of one parent (empty DFs have no row).""" + return sorted((df for row in table["files"][lfn] for df in row["dfs"]), key=lib.df_number) + + +def coverage(planned, produced): + """(missing, duplicated, unexpected) DF folder names, comparing the plan with the MC file's folders.""" + counts = collections.Counter(produced) + missing = sorted((df for df in planned if df not in counts), key=lib.df_number) + duplicated = sorted((df for df, n in counts.items() if n > 1), key=lib.df_number) + unexpected = sorted((df for df in counts if df not in set(planned)), key=lib.df_number) + return missing, duplicated, unexpected + + +def bc_subset(mc_bcs, data_bcs): + return set(mc_bcs) <= set(data_bcs) + + +def count_mismatches(mc_counts, data_counts): + return [f"{df}: {mc_counts.get(df, 0)} MC collisions for {data_counts[df]} data collisions" + for df in sorted(data_counts, key=lib.df_number) if mc_counts.get(df, 0) != data_counts[df]] + + +def mccollision_key(keys): + """The MC collision table among a DF folder's keys (not O2mccollisionlabel).""" + for k in keys: + if re.match(r"^O2mccollision_\d+(;\d+)?$", k): + return k + return None + + +def data_counts(data, planned, excluded): + """DF -> number of data collisions to compare with, leaving out excluded (DF, TF).""" + return {df: sum(len(bcs) for tf, bcs in data.get(df, {}).items() if (df, tf) not in excluded) for df in planned} + + +def excluded_from_status(status_paths): + """(DF, TF) pairs reported as excluded in the given embedding_status.tsv files.""" + out = set() + for status in status_paths: + if os.path.exists(status): + with open(status) as f: + for line in f: + parts = line.split() + if len(parts) == 3 and parts[2] == "excluded": + out.add((parts[0], int(parts[1]))) + return out + + +def excluded_tfs(manifest_paths): + """(DF, TF) pairs a job reported as excluded, from the embedding_status.tsv beside each manifest.""" + return excluded_from_status([os.path.join(os.path.dirname(p), "embedding_status.tsv") for p in manifest_paths]) + + +def missing_aod_is_error(manifest_path): + """A job without AO2D.root is fine only when its status file reports all its timeframes excluded.""" + status = os.path.join(os.path.dirname(manifest_path), "embedding_status.tsv") + if not os.path.exists(status): + return True + with open(status) as f: + states = [line.split()[2] for line in f if len(line.split()) == 3] + return not states or any(st != "excluded" for st in states) + + +def fully_excluded_dfs(table, lfn, excluded): + """Planned DF folders of a parent whose timeframes were all excluded; they have no MC output.""" + out = [] + for row in table["files"][lfn]: + for df, tfs in row["dfs"].items(): + if tfs and all((df, tf) in excluded for tf in tfs): + out.append(df) + return sorted(out, key=lib.df_number) + + +def expected_dfs(table, lfn, excluded): + """DF folders the MC output of a parent must contain.""" + gone = set(fully_excluded_dfs(table, lfn, excluded)) + return [df for df in planned_dfs(table, lfn) if df not in gone] + + +def group_by_parent(manifests): + """ + Parent LFN -> AO2D files of the jobs that embedded into it, ordered by their first DF, + so that --merge-by-name sees the DF folders in order. manifests: [(manifest path, content)]. + """ + groups = collections.defaultdict(list) + for path, m in manifests: + first = min(lib.df_number(df) for df in m["dfs"]) + groups[m["lfn"]].append((first, os.path.abspath(os.path.join(os.path.dirname(path), "AO2D.root")))) + return {lfn: [aod for _, aod in sorted(items)] for lfn, items in groups.items()} + + +# ---- ROOT-based readers ------------------------------------------------------------------------ + +def _open(path): + """TFile.Open that first connects to AliEn for alien:// paths.""" + import ROOT + if path.startswith("alien://") and not ROOT.gGrid: + ROOT.TGrid.Connect("alien://") + f = ROOT.TFile.Open(path) + if not f or f.IsZombie(): + raise OSError(f"Failed to open file {path}") + return f + + +def mc_df_folders(path): + """DF folder names in a file, once per key cycle (a duplicated folder shows up twice).""" + import ROOT + f = _open(path) + names = [k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")] + f.Close() + return names + + +def parent_files(path): + import ROOT + f = _open(path) + m = f.Get("parentFiles") + out = {} + if m: + it = m.MakeIterator() + key = it.Next() + while key: + out[key.GetName()] = m.GetValue(key).GetName() + key = it.Next() + f.Close() + return out + + +def mc_truth_bcs(path): + """DF -> list of globalBCs of the MC truth collisions (read with ROOT: AO2Ds may use zstd).""" + import ROOT + f = _open(path) + out = {} + for df in sorted({k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")}): + d = f.Get(df) + keys = [k.GetName() for k in d.GetListOfKeys()] + mcc = mccollision_key(keys) + bcs = [k for k in keys if k.startswith("O2bc_")] + if mcc is None or not bcs: + out[df] = [] + continue + index = ROOT.RDataFrame(d.Get(mcc)).AsNumpy(["fIndexBCs"])["fIndexBCs"] + gbc = ROOT.RDataFrame(d.Get(bcs[0])).AsNumpy(["fGlobalBC"])["fGlobalBC"] + out[df] = [int(gbc[i]) for i in index] + f.Close() + return out + + +def data_collision_bcs(parent, run): + """DF -> TF -> globalBCs of the triggered data collisions (the ones the contexts were made from).""" + import o2dpg_data_embedding_utils as utils + merged = utils.read_timeframe_structure(parent, run) + out = {} + for key, frame in merged.items(): + out[key.split("@")[0]] = {int(tf): [int(c[0]) for c in cols] + for tf, cols in zip(frame["timeframeID"], frame["position_vectors"])} + return out + + +def check(mc, parent, table, workdir, excluded=frozenset()): + """Returns a list of problems (empty: the MC file is correctly linked to its parent).""" + problems = [] + planned = expected_dfs(table, parent, excluded) + missing, duplicated, unexpected = coverage(planned, mc_df_folders(mc)) + problems += [f"missing DF {df}" for df in missing] + problems += [f"duplicated DF {df}" for df in duplicated] + problems += [f"DF {df} is not in the plan" for df in unexpected] + + links = parent_files(mc) + for df in planned: + if links.get(df) is None: + problems.append(f"{df} has no parentFiles entry") + elif links[df].replace("alien://", "") != parent.replace("alien://", ""): + problems.append(f"{df} points to {links[df]}") + + cwd = os.getcwd() + os.chdir(workdir) # the data tables are copied locally here + try: + data = data_collision_bcs(parent, table["run"]) + finally: + os.chdir(cwd) + mc_bcs = mc_truth_bcs(mc) + for df in planned: + data_bcs = [bc for bcs in data.get(df, {}).values() for bc in bcs] + if not bc_subset(mc_bcs.get(df, []), data_bcs): + problems.append(f"{df}: MC collisions at BCs without a data collision") + problems += count_mismatches({df: len(mc_bcs.get(df, [])) for df in planned}, + data_counts(data, planned, excluded)) + return problems + + +# ---- commands ---------------------------------------------------------------------------------- + +def cmd_merge(args, table): + manifests = [] + for d in args.outputs: + for p in sorted(glob.glob(os.path.join(d, "**", "embedding_row.json"), recursive=True)): + with open(p) as f: + manifests.append((p, json.load(f))) + groups = group_by_parent(manifests) + rc = 0 + for i, parent in enumerate(sorted(groups)): + outdir = os.path.join(args.out, f"{i:04d}") + os.makedirs(outdir, exist_ok=True) + manifests_here = [p for p, m in manifests if m["lfn"] == parent] + aods = [a for a in groups[parent] if os.path.exists(a)] + bad = [p for p in manifests_here + if not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(p)), "AO2D.root")) and missing_aod_is_error(p)] + if bad: + print(f"ERROR: {len(bad)} job outputs without AO2D.root for {parent}") + rc = 1 + if not aods: + if not bad: + print(f"{parent}: all timeframes excluded, nothing to merge") + continue + with open(os.path.join(outdir, "inputs.txt"), "w") as f: + f.write("\n".join(aods) + "\n") + with open(os.path.join(outdir, "parent.txt"), "w") as f: + f.write(parent + "\n") + merge = subprocess.run(["o2-aod-merger", "--input", "inputs.txt", "--output", "AO2D.root", + "--merge-by-name", "--max-size", "1000000000000"], cwd=outdir) + if merge.returncode != 0: + print(f"ERROR: merging for {parent} failed") + rc = 1 + continue + excluded = excluded_tfs([p for p, m in manifests if m["lfn"] == parent]) + problems = check(os.path.join(outdir, "AO2D.root"), parent, table, outdir, excluded) + print(f"{parent} -> {outdir}/AO2D.root: {'OK' if not problems else 'FAILED'}") + for p in problems: + print(f" {p}") + rc |= 1 if problems else 0 + planned_parents = set(table["files"]) + for lfn in sorted(planned_parents - set(groups)): + print(f"ERROR: no job output at all for {lfn}") + rc = 1 + return rc + + +def main(): + parser = argparse.ArgumentParser(description="Merge and check MC-DATA embedding outputs") + sub = parser.add_subparsers(dest="cmd", required=True) + m = sub.add_parser("merge", help="Merge job outputs per parent and check each result") + m.add_argument("--plan", required=True) + m.add_argument("--outputs", nargs="+", required=True, help="Local directories holding the job outputs") + m.add_argument("--out", required=True) + c = sub.add_parser("check", help="Check one MC AO2D against its parent") + c.add_argument("--plan", required=True) + c.add_argument("--mc", required=True) + c.add_argument("--parent", required=True) + c.add_argument("--status", nargs="*", default=[], help="embedding_status.tsv files of the jobs, for excluded timeframes") + args = parser.parse_args() + + with open(os.path.join(args.plan, "embedding_table.json")) as f: + table = json.load(f) + if args.cmd == "merge": + return cmd_merge(args, table) + # the data tables are copied beside a local MC file, or into the current directory for alien:// + workdir = os.getcwd() if args.mc.startswith("alien://") else os.path.dirname(os.path.abspath(args.mc)) + problems = check(args.mc, args.parent, table, workdir, excluded_from_status(args.status)) + for p in problems: + print(p) + print("OK" if not problems else "FAILED") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MC/run/ANCHOR/dataembedding/embedding_plan_lib.py b/MC/run/ANCHOR/dataembedding/embedding_plan_lib.py new file mode 100644 index 000000000..b6bdfa141 --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/embedding_plan_lib.py @@ -0,0 +1,102 @@ +# Pure helpers for planning MC-DATA embedding productions (no ROOT, no GRID). +# +# The work table (embedding_table.json) has the form +# {"run": R, "orbits_per_tf": n, +# "files": {"alien:///.../AO2D.root": [{"dfs": {"DF_123": [tf, ...]}}, ...]}} +# Each entry of a file's list is one job ("row"). A DF folder is never split across rows. + +import hashlib +import math +import random +import re + +_DF_NUMBER = re.compile(r"^DF_(\d+)") + + +def df_number(df_name): + """Numeric part of a DF folder name such as DF_2500123.""" + m = _DF_NUMBER.match(df_name) + if not m: + raise ValueError(f"not a DF folder name: {df_name}") + return int(m.group(1)) + + +def sample_files(lfns, fraction, seed): + """Deterministic sample of ceil(fraction * N) files (at least one), returned sorted.""" + pool = sorted(set(lfns)) + n = min(len(pool), max(1, math.ceil(fraction * len(pool)))) + return sorted(random.Random(seed).sample(pool, n)) + + +def empty_dfs(dfs): + """DF folders without any timeframe to embed (no triggered collision).""" + return sorted((df for df, tfs in dfs.items() if not tfs), key=df_number) + + +def pack_rows(dfs, tfs_per_job): + """ + Pack whole DF folders, in numeric DF order, greedily and contiguously into rows + of at most tfs_per_job timeframes. A DF larger than the budget gets a row of its own; + tfs_per_job <= 0 puts all DFs into one row. Empty DFs get no row. + """ + rows, current, ntf = [], {}, 0 + for df in sorted(dfs, key=df_number): + tfs = sorted(dfs[df]) + if not tfs: + continue + if current and tfs_per_job > 0 and ntf + len(tfs) > tfs_per_job: + rows.append(current) + current, ntf = {}, 0 + current[df] = tfs + ntf += len(tfs) + if current: + rows.append(current) + return rows + + +def embedding_seed(job_id, df, tf): + """ + Seed for one data timeframe of a GRID job: a hash of (job id, DF folder, timeframe) in [1, 2^31). + The job id is normally ALIEN_PROC_ID. Timeframes collide only by chance (about 2 pairs per 10^5). + """ + digest = hashlib.sha1(f"{job_id}:{df}:{tf}".encode()).digest() + return max(1, int.from_bytes(digest[:4], "big") & 0x7fffffff) + + +def flat_rows(table): + """All (lfn, row) pairs: files sorted by LFN, then row index (rows submission mode).""" + return [(lfn, row) for lfn in sorted(table["files"]) for row in table["files"][lfn]] + + +def num_cycles(table): + """Number of masterjobs in cycles submission mode: the largest row count of any file.""" + return max((len(rows) for rows in table["files"].values()), default=0) + + +def files_for_cycle(table, cycle): + """Files that have a row with index `cycle` (cycles submission mode).""" + return sorted(lfn for lfn, rows in table["files"].items() if len(rows) > cycle) + + +def aod_lfns(found): + """ + Per-job data AO2D files from an 'alien.py find' listing, with an alien:// prefix. + Merged copies under an AOD/ directory are left out, so a DF is not planned twice. + """ + out = set() + for path in found: + path = path.strip() + if not path.endswith("/AO2D.root") or "/AOD/" in path: + continue + out.add(path if path.startswith("alien://") else "alien://" + path) + return sorted(out) + + +if __name__ == "__main__": + # embedding_plan_lib.py seed : prints the seed, for the job script + import sys + if len(sys.argv) == 5 and sys.argv[1] == "seed": + print(embedding_seed(sys.argv[2], sys.argv[3], int(sys.argv[4]))) + else: + print("usage: embedding_plan_lib.py seed ", file=sys.stderr) + sys.exit(1) diff --git a/MC/run/ANCHOR/dataembedding/embedding_row.py b/MC/run/ANCHOR/dataembedding/embedding_row.py new file mode 100644 index 000000000..9db9aecbf --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/embedding_row.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# +# Finds the row of the MC-DATA embedding work table that this GRID job has to treat. +# Cycles mode: ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE=c and the job's input file (input.list or wn.xml). +# Rows mode: ALIEN_JDL_MC_DATA_EMBEDDING_ROW=r (1-based, set to #alien_counter# in the JDL). +# Prints " " and writes the expected timeframes and a manifest. + +import argparse +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import embedding_plan_lib as lib + + +def _with_prefix(lfn): + return lfn if lfn.startswith("alien://") else "alien://" + lfn + + +def job_lfn(cwd): + """The data file AliEn assigned to this job: input.list (txt-list) or wn.xml (xml-single).""" + path = os.path.join(cwd, "input.list") + if os.path.exists(path): + with open(path) as f: + for line in f: + if line.strip(): + return _with_prefix(line.strip()) + path = os.path.join(cwd, "wn.xml") + if os.path.exists(path): + with open(path) as f: + m = re.search(r'\blfn="([^"]+)"', f.read()) + if m: + return _with_prefix(m.group(1)) + return None + + +def resolve_row(table, env, lfn): + """Returns (data LFN, {DF: [tf, ...]}) for this job, or raises ValueError.""" + if env.get("ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE", "") != "": + cycle = int(env["ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE"]) + if lfn is None: + raise ValueError("cycles mode, but the job has no input file (input.list / wn.xml)") + rows = table["files"].get(lfn) + if rows is None: + raise ValueError(f"input file {lfn} is not in the work table") + if not 0 <= cycle < len(rows): + raise ValueError(f"cycle {cycle} out of range: {lfn} has {len(rows)} rows") + return lfn, rows[cycle]["dfs"] + if env.get("ALIEN_JDL_MC_DATA_EMBEDDING_ROW", "") != "": + index = int(env["ALIEN_JDL_MC_DATA_EMBEDDING_ROW"]) + flat = lib.flat_rows(table) + if not 1 <= index <= len(flat): + raise ValueError(f"row {index} out of range: the table has {len(flat)} rows") + lfn, row = flat[index - 1] + return lfn, row["dfs"] + raise ValueError("neither ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE nor ALIEN_JDL_MC_DATA_EMBEDDING_ROW is set") + + +def run_cli(argv, env, cwd): + parser = argparse.ArgumentParser(description="Find this job's row of the MC-DATA embedding work table") + parser.add_argument("--table", required=True, help="Work table (embedding_table.json)") + parser.add_argument("--expect-out", required=True, help="Write the planned DF -> timeframes here") + parser.add_argument("--manifest", required=True, help="Write the job manifest here") + args = parser.parse_args(argv) + + with open(os.path.join(cwd, args.table)) as f: + table = json.load(f) + lfn, dfs = resolve_row(table, env, job_lfn(cwd)) + with open(os.path.join(cwd, args.expect_out), "w") as f: + json.dump(dfs, f) + cycle = env.get("ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE", "") + row = env.get("ALIEN_JDL_MC_DATA_EMBEDDING_ROW", "") + with open(os.path.join(cwd, args.manifest), "w") as f: + json.dump({"run": table["run"], "lfn": lfn, + "cycle": int(cycle) if cycle != "" else None, + "row": int(row) if row != "" else None, + "dfs": dfs}, f, indent=1) + return f"{lfn} {','.join(sorted(dfs, key=lib.df_number))}" + + +if __name__ == "__main__": + try: + print(run_cli(sys.argv[1:], os.environ, os.getcwd())) + except (ValueError, OSError, KeyError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) diff --git a/MC/run/ANCHOR/dataembedding/plan_embedding.py b/MC/run/ANCHOR/dataembedding/plan_embedding.py new file mode 100755 index 000000000..14bc2e5d5 --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/plan_embedding.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# +# Plans an MC-DATA embedding production: samples a fraction of a run's data AO2D files, +# reads their BC/collision tables and packs whole DF folders into jobs ("rows"). +# Writes embedding_table.json (the job input), empty_dfs.json and plan_summary.txt. +# +# Example: +# plan_embedding.py --run 544653 --data-path /alice/data/2023/LHC23zzm/544653/apass4 \ +# --fraction 0.002 --seed 1 --tfs-per-job 2 --out plan_544653 + +import argparse +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "utils")) + +import embedding_plan_lib as lib + + +def find_files(data_path): + """All per-job AO2D files below data_path on AliEn.""" + out = subprocess.run(["alien.py", "find", data_path, "AO2D.root"], capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(f"alien.py find failed: {out.stderr.strip()}") + return lib.aod_lfns(out.stdout.splitlines()) + + +def main(): + parser = argparse.ArgumentParser(description="Plan an MC-DATA embedding production") + parser.add_argument("--run", type=int, required=True, help="Run number of the data") + src = parser.add_mutually_exclusive_group(required=True) + src.add_argument("--data-path", help="AliEn directory to search for AO2D.root, e.g. /alice/data/2023/LHC23zzm/544653/apass4") + src.add_argument("--file-list", help="Text file with one data AO2D LFN per line") + parser.add_argument("--fraction", type=float, default=1.0, help="Fraction of the files to embed into (default 1)") + parser.add_argument("--seed", type=int, default=1, help="Seed of the file sampling") + parser.add_argument("--tfs-per-job", type=int, default=0, help="Timeframes per job; a DF is never split (0: one job per file)") + parser.add_argument("--out", required=True, help="Output directory") + args = parser.parse_args() + + if args.data_path: + candidates = find_files(args.data_path) + else: + with open(args.file_list) as f: + candidates = lib.aod_lfns(f.read().splitlines()) + if not candidates: + print("No data AO2D files found") + return 1 + files = lib.sample_files(candidates, args.fraction, args.seed) + print(f"Sampled {len(files)} of {len(candidates)} data AO2D files") + + import o2dpg_data_embedding_utils as utils # needs ROOT and a GRID token + + os.makedirs(args.out, exist_ok=True) + workdir = os.getcwd() + os.chdir(args.out) # the utils write their local copy of the BC/collision tables here + table = {"run": args.run, + "orbits_per_tf": utils.retrieve_Aggregated_RunInfos(args.run)["OrbitsPerTF"], + "tfs_per_job": args.tfs_per_job, + "files": {}} + empty, ntf, nrows, largest = {}, 0, 0, 0 + for lfn in files: + dfs = utils.units_from_file(lfn, args.run) + rows = lib.pack_rows(dfs, args.tfs_per_job) + table["files"][lfn] = [{"dfs": r} for r in rows] + empty[lfn] = lib.empty_dfs(dfs) + ntf += sum(len(t) for t in dfs.values()) + nrows += len(rows) + largest = max([largest] + [sum(len(t) for t in r.values()) for r in rows]) + print(f"{lfn}: {len(dfs)} DFs, {sum(len(t) for t in dfs.values())} TFs, {len(rows)} jobs") + os.chdir(workdir) + + with open(os.path.join(args.out, "embedding_table.json"), "w") as f: + json.dump(table, f, indent=1) + with open(os.path.join(args.out, "empty_dfs.json"), "w") as f: + json.dump(empty, f, indent=1) + summary = (f"run {args.run}, {len(files)} of {len(candidates)} files (fraction {args.fraction}, seed {args.seed})\n" + f"{ntf} timeframes in {nrows} jobs, at most {largest} TFs per job (budget {args.tfs_per_job})\n" + f"cycles mode: {lib.num_cycles(table)} masterjobs; rows mode: 1 masterjob with {nrows} subjobs\n") + with open(os.path.join(args.out, "plan_summary.txt"), "w") as f: + f.write(summary) + print(summary, end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MC/run/ANCHOR/dataembedding/submit_embedding.py b/MC/run/ANCHOR/dataembedding/submit_embedding.py new file mode 100755 index 000000000..f947390fb --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/submit_embedding.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# +# Submits an MC-DATA embedding production planned by plan_embedding.py, via grid_submit.sh. +# +# cycles mode (default): one masterjob per cycle c, split on the collection of the data files +# that have more than c rows; a subjob runs close to its data file and treats row c of it. +# rows mode: one masterjob with one subjob per row of the table (no data locality). +# +# The anchoring variables (run, pass, production tag, generator options, ...) come from +# --anchor-env, a file of 'export ALIEN_JDL_...=...' lines. Without --submit nothing is sent. +# +# Example: +# submit_embedding.py --plan plan_544653 --anchor-env anchor_544653.env \ +# --table-dir /alice/cern.ch/user/s/swenzel/embedding/544653 --jobname emb_544653 \ +# -- --ttl 36000 --packagespec '"VO_ALICE@O2PDPSuite::"' + +import argparse +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import embedding_plan_lib as lib + +# patterns for files a successful job may lack: JAliEn fails a job whose plain output file is missing +OUTPUTS = ["AO2D*.root@disk=2", "embedding_row.json@disk=2", "embedding_status*.tsv@disk=2", + "debug_log_archive.tgz@disk=1", "*.log@disk=1"] + + +def _plain(lfn): + return lfn[len("alien://"):] if lfn.startswith("alien://") else lfn + + +def submissions(table, mode): + """The masterjobs to submit: name, cycle, prodsplit and the collection of data files.""" + if mode == "cycles": + return [{"name": f"cycle{c}", "cycle": c, "prodsplit": None, + "collection": [_plain(lfn) for lfn in lib.files_for_cycle(table, c)]} + for c in range(lib.num_cycles(table))] + if mode == "rows": + return [{"name": "rows", "cycle": None, "prodsplit": len(lib.flat_rows(table)), "collection": None}] + raise ValueError(f"unknown mode {mode}") + + +def job_script(sub, anchor_env): + """The job script of one masterjob: anchoring variables, the row selection, then the embedding.""" + owned = re.search(r"\bALIEN_JDL_MC_DATA_EMBEDDING_(TABLE|CYCLE|ROW|AO2D)\b", anchor_env) + if owned: + raise ValueError(f"the anchor environment must not set {owned.group(0)}; the wrapper sets it") + lines = ["#!/bin/bash", + "#JDL_OUTPUT=" + ",".join(OUTPUTS), + "#JDL_ERROROUTPUT=" + ",".join(o for o in OUTPUTS if not o.startswith("AO2D")), + "", + anchor_env.rstrip("\n"), + "export ALIEN_JDL_MC_DATA_EMBEDDING_TABLE=embedding_table.json"] + if sub["cycle"] is not None: + lines.append(f"export ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE={sub['cycle']}") + else: + lines.append("export ALIEN_JDL_MC_DATA_EMBEDDING_ROW=${ALIEN_JDL_MC_DATA_EMBEDDING_ROW:-${ALIEN_O2DPG_GRIDSUBMIT_SUBJOBID}}") + lines += ["", "${O2DPG_ROOT}/MC/run/ANCHOR/anchorMC_DataEmbedding.sh", ""] + return "\n".join(lines) + + +def grid_submit_args(sub, script, collection_file, table_lfn, jobname, dry, extra): + args = ["--script", script, "--jobname", f"{jobname}_{sub['name']}", "--inputfile", table_lfn] + if sub["collection"] is not None: + args += ["--split-on-collection", collection_file] + else: + args += ["--prodsplit", str(sub["prodsplit"])] + if dry: + args.append("--dry") + return args + extra + + +def main(): + parser = argparse.ArgumentParser(description="Submit a planned MC-DATA embedding production") + parser.add_argument("--plan", required=True, help="Output directory of plan_embedding.py") + parser.add_argument("--anchor-env", required=True, help="File with 'export ALIEN_JDL_...' anchoring lines") + parser.add_argument("--table-dir", required=True, help="AliEn directory to upload embedding_table.json to") + parser.add_argument("--jobname", required=True, help="Job name prefix") + parser.add_argument("--mode", choices=["cycles", "rows"], default="cycles") + parser.add_argument("--submit", action="store_true", help="Really submit (default: grid_submit.sh --dry)") + parser.add_argument("extra", nargs="*", help="After '--': further grid_submit.sh arguments") + args = parser.parse_args() + + with open(os.path.join(args.plan, "embedding_table.json")) as f: + table = json.load(f) + with open(args.anchor_env) as f: + anchor_env = f.read() + table_lfn = args.table_dir.rstrip("/") + "/embedding_table.json" + grid_submit = os.path.join(HERE, "..", "..", "..", "..", "GRID", "utils", "grid_submit.sh") + + if args.submit: + subprocess.run(["alien.py", "mkdir", "-p", args.table_dir], check=True) + subprocess.run(["alien.py", "cp", "-f", "file:" + os.path.join(args.plan, "embedding_table.json"), + "alien:" + table_lfn], check=True) + else: + print(f"[dry] would upload embedding_table.json to {table_lfn}") + + rc = 0 + for sub in submissions(table, args.mode): + script = os.path.abspath(os.path.join(args.plan, f"job_{sub['name']}.sh")) + with open(script, "w") as f: + f.write(job_script(sub, anchor_env)) + os.chmod(script, 0o755) + collection_file = None + if sub["collection"] is not None: + collection_file = os.path.abspath(os.path.join(args.plan, f"collection_{sub['name']}.txt")) + with open(collection_file, "w") as f: + f.write("\n".join(sub["collection"]) + "\n") + cmd = [grid_submit] + grid_submit_args(sub, script, collection_file, table_lfn, args.jobname, + not args.submit, args.extra) + print(f"{sub['name']}: {len(sub['collection']) if sub['collection'] else sub['prodsplit']} subjobs") + print(" ".join(cmd)) + rc |= subprocess.run(cmd, cwd=args.plan).returncode + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/MC/run/ANCHOR/dataembedding/tests/test_check_embedding_output.py b/MC/run/ANCHOR/dataembedding/tests/test_check_embedding_output.py new file mode 100644 index 000000000..72cd64034 --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/tests/test_check_embedding_output.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Offline tests for the pure checks of the MC-DATA embedding merge/validation (no ROOT, no GRID).""" +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import check_embedding_output as chk + +A = "alien:///d/a/AO2D.root" +B = "alien:///d/b/AO2D.root" + + +class TestCoverage(unittest.TestCase): + + def test_coverage_clean(self): + self.assertEqual(chk.coverage(["DF_1", "DF_2"], ["DF_1", "DF_2"]), ([], [], [])) + + def test_coverage_missing_duplicated_unexpected(self): + missing, duplicated, unexpected = chk.coverage(["DF_1", "DF_2", "DF_3"], ["DF_1", "DF_1", "DF_3", "DF_9"]) + self.assertEqual((missing, duplicated, unexpected), (["DF_2"], ["DF_1"], ["DF_9"])) + + def test_empty_df_counts_as_covered(self): + # empty DFs are not in the plan's rows, so they are never expected in the output + planned = chk.planned_dfs({"files": {A: [{"dfs": {"DF_1": [5]}}, {"dfs": {"DF_3": [6]}}]}}, A) + self.assertEqual(planned, ["DF_1", "DF_3"]) + self.assertEqual(chk.coverage(planned, ["DF_1", "DF_3"]), ([], [], [])) + + +class TestBCs(unittest.TestCase): + + def test_bc_subset(self): + self.assertTrue(chk.bc_subset([10, 20], [5, 10, 20, 30])) + self.assertFalse(chk.bc_subset([10, 21], [5, 10, 20, 30])) + + def test_collision_counts(self): + self.assertEqual(chk.count_mismatches({"DF_1": 90, "DF_2": 80}, {"DF_1": 90, "DF_2": 81}), + ["DF_2: 80 MC collisions for 81 data collisions"]) + + +class TestTables(unittest.TestCase): + + def test_mccollision_key_exact(self): + self.assertEqual(chk.mccollision_key(["O2mccollisionlabel_001", "O2bc_001", "O2mccollision_001"]), "O2mccollision_001") + self.assertIsNone(chk.mccollision_key(["O2mccollisionlabel_001"])) + + def test_data_counts_skip_excluded(self): + data = {"DF_1": {10: [1, 2, 3], 11: [4]}, "DF_2": {12: [5]}} + self.assertEqual(chk.data_counts(data, ["DF_1", "DF_2"], {("DF_1", 11)}), {"DF_1": 3, "DF_2": 1}) + + def test_excluded_from_status(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "embedding_status.tsv"), "w") as f: + f.write("DF_1\t10\tdone\nDF_1\t11\texcluded\n") + self.assertEqual(chk.excluded_tfs([os.path.join(d, "embedding_row.json")]), {("DF_1", 11)}) + + +class TestExcluded(unittest.TestCase): + + TABLE = {"files": {A: [{"dfs": {"DF_1": [10, 11], "DF_2": [12]}}]}} + + def test_fully_excluded_dfs(self): + self.assertEqual(chk.fully_excluded_dfs(self.TABLE, A, {("DF_1", 10), ("DF_2", 12)}), ["DF_2"]) + self.assertEqual(chk.fully_excluded_dfs(self.TABLE, A, set()), []) + + def test_expected_dfs_skip_fully_excluded(self): + self.assertEqual(chk.expected_dfs(self.TABLE, A, {("DF_2", 12)}), ["DF_1"]) + + def test_excluded_from_status_files(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "embedding_status.tsv") + with open(p, "w") as f: + f.write("DF_2\t12\texcluded\nDF_1\t10\tdone\n") + self.assertEqual(chk.excluded_from_status([p]), {("DF_2", 12)}) + + +class TestMissingAOD(unittest.TestCase): + + def _status(self, d, text): + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "embedding_status.tsv"), "w") as f: + f.write(text) + return os.path.join(d, "embedding_row.json") + + def test_all_excluded_job_may_lack_aod(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + m = self._status(d, "DF_1\t10\texcluded\nDF_1\t11\texcluded\n") + self.assertFalse(chk.missing_aod_is_error(m)) + + def test_failed_or_unknown_job_without_aod_is_error(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + self.assertTrue(chk.missing_aod_is_error(self._status(os.path.join(d, "a"), "DF_1\t10\texcluded\nDF_1\t11\tfailed\n"))) + self.assertTrue(chk.missing_aod_is_error(os.path.join(d, "b", "embedding_row.json"))) # no status file + + +class TestGrouping(unittest.TestCase): + + def test_group_outputs_by_parent_in_row_order(self): + manifests = [("o/3/embedding_row.json", {"lfn": A, "cycle": 1, "row": None, "dfs": {"DF_3": [6]}}), + ("o/1/embedding_row.json", {"lfn": B, "cycle": 0, "row": None, "dfs": {"DF_9": [1]}}), + ("o/2/embedding_row.json", {"lfn": A, "cycle": 0, "row": None, "dfs": {"DF_1": [5]}})] + groups = chk.group_by_parent(manifests) + cwd = os.getcwd() + self.assertEqual(groups, {A: [os.path.join(cwd, "o/2/AO2D.root"), os.path.join(cwd, "o/3/AO2D.root")], + B: [os.path.join(cwd, "o/1/AO2D.root")]}) + + def test_rows_mode_orders_by_first_df(self): + manifests = [("x/embedding_row.json", {"lfn": A, "cycle": None, "row": 7, "dfs": {"DF_30": [6]}}), + ("y/embedding_row.json", {"lfn": A, "cycle": None, "row": 2, "dfs": {"DF_4": [5]}})] + self.assertEqual(chk.group_by_parent(manifests), {A: [os.path.abspath("y/AO2D.root"), os.path.abspath("x/AO2D.root")]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/run/ANCHOR/dataembedding/tests/test_embedding_plan_lib.py b/MC/run/ANCHOR/dataembedding/tests/test_embedding_plan_lib.py new file mode 100644 index 000000000..10ddd8922 --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/tests/test_embedding_plan_lib.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Offline tests for the MC-DATA embedding work planner (no ROOT, no GRID).""" +import os +import random +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import embedding_plan_lib as lib + +LFNS = [f"alien:///alice/data/2023/LHC23zzm/544653/apass4/{i:04d}/AO2D.root" for i in range(100)] + + +def dfs_with(counts): + """DF name -> list of TF ids; DF numbers increase, TF ids are consecutive.""" + dfs, tf = {}, 1000 + for i, n in enumerate(counts): + dfs[f"DF_{100 + i}"] = list(range(tf, tf + n)) + tf += n + return dfs + + +class TestSampling(unittest.TestCase): + + def test_sample_deterministic_and_size(self): + a = lib.sample_files(LFNS, 0.05, 42) + self.assertEqual(len(a), 5) + self.assertEqual(a, lib.sample_files(list(reversed(LFNS)), 0.05, 42)) + self.assertEqual(a, sorted(a)) + self.assertNotEqual(a, lib.sample_files(LFNS, 0.05, 43)) + + def test_sample_at_least_one(self): + self.assertEqual(len(lib.sample_files(LFNS, 0.0001, 1)), 1) + + def test_sample_fraction_one_takes_all(self): + self.assertEqual(lib.sample_files(LFNS, 1.0, 1), sorted(LFNS)) + + +class TestPacking(unittest.TestCase): + + def test_pack_contiguous(self): + rows = lib.pack_rows(dfs_with([2, 2, 2, 2]), 4) + self.assertEqual([sorted(r) for r in rows], [["DF_100", "DF_101"], ["DF_102", "DF_103"]]) + + def test_oversized_df_own_row(self): + rows = lib.pack_rows(dfs_with([1, 5, 1]), 2) + self.assertEqual([sorted(r) for r in rows], [["DF_100"], ["DF_101"], ["DF_102"]]) + + def test_budget_zero_is_one_row(self): + self.assertEqual(len(lib.pack_rows(dfs_with([3, 3, 3]), 0)), 1) + + def test_pack_numeric_df_order(self): + dfs = {"DF_9": [1], "DF_10": [2], "DF_100": [3]} + rows = lib.pack_rows(dfs, 1) + self.assertEqual([list(r) for r in rows], [["DF_9"], ["DF_10"], ["DF_100"]]) + + def test_pack_order_independent(self): + dfs = dfs_with([1, 3, 2, 2, 1, 4]) + items = list(dfs.items()) + random.Random(7).shuffle(items) + self.assertEqual(lib.pack_rows(dfs, 4), lib.pack_rows(dict(items), 4)) + + def test_rows_cover_each_df_once(self): + dfs = dfs_with([1, 3, 2, 2, 1, 4, 7, 1]) + rows = lib.pack_rows(dfs, 3) + seen = [df for r in rows for df in r] + self.assertEqual(sorted(seen), sorted(dfs)) + self.assertEqual(len(seen), len(set(seen))) + for r in rows: + self.assertEqual({df: dfs[df] for df in r}, r) + + def test_empty_df_gets_no_row(self): + dfs = {"DF_1": [5], "DF_2": [], "DF_3": [6]} + rows = lib.pack_rows(dfs, 0) + self.assertEqual(rows, [{"DF_1": [5], "DF_3": [6]}]) + self.assertEqual(lib.empty_dfs(dfs), ["DF_2"]) + + +class TestSeed(unittest.TestCase): + JOB = 3752251383 # an ALIEN_PROC_ID + + def test_seed_deterministic(self): + self.assertEqual(lib.embedding_seed(self.JOB, "DF_1", 1234), lib.embedding_seed(self.JOB, "DF_1", 1234)) + + def test_seed_depends_on_job_df_and_tf(self): + base = lib.embedding_seed(self.JOB, "DF_1", 1000) + others = [lib.embedding_seed(self.JOB + 1, "DF_1", 1000), # same timeframe in another job + lib.embedding_seed(self.JOB, "DF_2", 1000), # same timeframe in another DF folder + lib.embedding_seed(self.JOB, "DF_1", 1001)] + self.assertNotIn(base, others) + self.assertEqual(len(set(others)), len(others)) + + def test_consecutive_jobs_do_not_shift_onto_each_other(self): + # the old SEED + timeframe gave job N timeframe X+1 the seed of job N+1 timeframe X + self.assertNotEqual(lib.embedding_seed(self.JOB, "DF_1", 1001), lib.embedding_seed(self.JOB + 1, "DF_1", 1000)) + + def test_seed_range(self): + for tf in range(0, 2_000_000, 997): + self.assertTrue(1 <= lib.embedding_seed(self.JOB, "DF_7", tf) < 2**31) + + def test_seed_collisions_rare(self): + # 1e5 timeframes over 2000 consecutive jobs in a 2^31 space: about 2 colliding pairs expected + seeds = [lib.embedding_seed(self.JOB + tf // 50, f"DF_{tf // 50}", tf) for tf in range(100_000)] + self.assertLess(len(seeds) - len(set(seeds)), 20) + + def test_seed_cli(self): + import subprocess + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + out = subprocess.run([sys.executable, os.path.join(here, "embedding_plan_lib.py"), "seed", str(self.JOB), "DF_3", "12"], + capture_output=True, text=True, check=True).stdout.strip() + self.assertEqual(int(out), lib.embedding_seed(self.JOB, "DF_3", 12)) + + +class TestFind(unittest.TestCase): + + def test_aod_lfns_filters_and_prefixes(self): + found = ["/alice/data/2023/LHC23zzm/544653/apass4/0710/o2_ctf_run00544653_orbit0064712416_tf0000150534_epn244/001/AO2D.root", + "/alice/data/2023/LHC23zzm/544653/apass4/AOD/012/AO2D.root", + "/alice/data/2023/LHC23zzm/544653/apass4/0710/o2_ctf_x/001/AO2D.root.bak", + "", + "alien:///alice/data/2023/LHC23zzm/544653/apass4/0711/o2_ctf_y/001/AO2D.root"] + self.assertEqual(lib.aod_lfns(found), + ["alien:///alice/data/2023/LHC23zzm/544653/apass4/0710/o2_ctf_run00544653_orbit0064712416_tf0000150534_epn244/001/AO2D.root", + "alien:///alice/data/2023/LHC23zzm/544653/apass4/0711/o2_ctf_y/001/AO2D.root"]) + + +class TestTable(unittest.TestCase): + + def test_flat_rows_order(self): + table = {"run": 1, "orbits_per_tf": 32, "files": { + "alien:///b/AO2D.root": [{"dfs": {"DF_1": [1]}}], + "alien:///a/AO2D.root": [{"dfs": {"DF_5": [7]}}, {"dfs": {"DF_6": [8]}}]}} + flat = lib.flat_rows(table) + self.assertEqual([(lfn, r["dfs"]) for lfn, r in flat], + [("alien:///a/AO2D.root", {"DF_5": [7]}), + ("alien:///a/AO2D.root", {"DF_6": [8]}), + ("alien:///b/AO2D.root", {"DF_1": [1]})]) + + def test_files_for_cycle(self): + table = {"files": {"x": [{}, {}, {}], "y": [{}], "z": [{}, {}]}} + self.assertEqual(lib.files_for_cycle(table, 0), ["x", "y", "z"]) + self.assertEqual(lib.files_for_cycle(table, 1), ["x", "z"]) + self.assertEqual(lib.files_for_cycle(table, 2), ["x"]) + self.assertEqual(lib.files_for_cycle(table, 3), []) + self.assertEqual(lib.num_cycles(table), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/run/ANCHOR/dataembedding/tests/test_embedding_row.py b/MC/run/ANCHOR/dataembedding/tests/test_embedding_row.py new file mode 100644 index 000000000..37380c83e --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/tests/test_embedding_row.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Offline tests for how an embedding job finds its row of the work table.""" +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import embedding_row as er + +A = "alien:///alice/data/2023/LHC23zzm/544653/apass4/0710/o2_ctf_a/001/AO2D.root" +B = "alien:///alice/data/2023/LHC23zzm/544653/apass4/0711/o2_ctf_b/001/AO2D.root" +TABLE = {"run": 544653, "orbits_per_tf": 32, "files": { + B: [{"dfs": {"DF_20": [7]}}], + A: [{"dfs": {"DF_1": [1, 2], "DF_2": [3]}}, {"dfs": {"DF_3": [4, 5]}}]}} + +WN_XML = """ + + + + + + + +""" + + +class TestJobLfn(unittest.TestCase): + + def test_lfn_from_input_list(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "input.list"), "w") as f: + f.write("\n/alice/data/2023/LHC23zzm/544653/apass4/0710/o2_ctf_a/001/AO2D.root\n") + self.assertEqual(er.job_lfn(d), A) + + def test_lfn_from_wn_xml(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "wn.xml"), "w") as f: + f.write(WN_XML) + self.assertEqual(er.job_lfn(d), A) + + def test_no_input_gives_none(self): + with tempfile.TemporaryDirectory() as d: + self.assertIsNone(er.job_lfn(d)) + + +class TestResolveRow(unittest.TestCase): + + def test_row_by_cycle_and_lfn(self): + self.assertEqual(er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE": "1"}, A), + (A, {"DF_3": [4, 5]})) + + def test_row_by_flat_index(self): + # flat order: files sorted by LFN (A before B), then row index + self.assertEqual(er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_ROW": "3"}, None), (B, {"DF_20": [7]})) + self.assertEqual(er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_ROW": "1"}, None), (A, {"DF_1": [1, 2], "DF_2": [3]})) + + def test_row_out_of_range_raises(self): + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE": "1"}, B) + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_ROW": "4"}, None) + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_ROW": "0"}, None) + + def test_lfn_not_in_table_raises(self): + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE": "0"}, "alien:///other/AO2D.root") + + def test_cycle_without_lfn_raises(self): + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {"ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE": "0"}, None) + + def test_neither_variable_raises(self): + with self.assertRaises(ValueError): + er.resolve_row(TABLE, {}, A) + + +class TestCli(unittest.TestCase): + + def test_expect_tfs_and_manifest_written(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "table.json"), "w") as f: + json.dump(TABLE, f) + with open(os.path.join(d, "input.list"), "w") as f: + f.write(A + "\n") + line = er.run_cli(["--table", "table.json", "--expect-out", "expect.json", "--manifest", "m.json"], + {"ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE": "0"}, d) + self.assertEqual(line, f"{A} DF_1,DF_2") + with open(os.path.join(d, "expect.json")) as f: + self.assertEqual(json.load(f), {"DF_1": [1, 2], "DF_2": [3]}) + with open(os.path.join(d, "m.json")) as f: + m = json.load(f) + self.assertEqual((m["lfn"], m["cycle"], m["row"], m["run"]), (A, 0, None, 544653)) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/run/ANCHOR/dataembedding/tests/test_submit_embedding.py b/MC/run/ANCHOR/dataembedding/tests/test_submit_embedding.py new file mode 100644 index 000000000..0e42a6fcf --- /dev/null +++ b/MC/run/ANCHOR/dataembedding/tests/test_submit_embedding.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Offline tests for the MC-DATA embedding submission wrapper (no GRID).""" +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import submit_embedding as se + +A = "alien:///alice/data/2023/LHC23zzm/544653/apass4/0710/a/001/AO2D.root" +B = "alien:///alice/data/2023/LHC23zzm/544653/apass4/0711/b/001/AO2D.root" +TABLE = {"run": 544653, "files": {A: [{"dfs": {"DF_1": [1]}}, {"dfs": {"DF_2": [2]}}], B: [{"dfs": {"DF_3": [3]}}]}} +ANCHOR_ENV = "export ALIEN_JDL_LPMRUNNUMBER=544653\nexport ALIEN_JDL_LPMANCHORPASSNAME=apass4\n" + + +class TestSubmissions(unittest.TestCase): + + def test_cycles_mode_one_masterjob_per_cycle(self): + subs = se.submissions(TABLE, "cycles") + self.assertEqual([s["name"] for s in subs], ["cycle0", "cycle1"]) + self.assertEqual(subs[0]["collection"], ["/alice/data/2023/LHC23zzm/544653/apass4/0710/a/001/AO2D.root", + "/alice/data/2023/LHC23zzm/544653/apass4/0711/b/001/AO2D.root"]) + self.assertEqual(subs[1]["collection"], ["/alice/data/2023/LHC23zzm/544653/apass4/0710/a/001/AO2D.root"]) + self.assertIsNone(subs[0]["prodsplit"]) + + def test_rows_mode_one_masterjob(self): + subs = se.submissions(TABLE, "rows") + self.assertEqual(len(subs), 1) + self.assertEqual((subs[0]["prodsplit"], subs[0]["collection"]), (3, None)) + + +class TestJobScript(unittest.TestCase): + + def test_cycles_script_exports_cycle_and_table(self): + text = se.job_script({"name": "cycle1", "cycle": 1, "prodsplit": None, "collection": ["x"]}, ANCHOR_ENV) + self.assertIn("export ALIEN_JDL_MC_DATA_EMBEDDING_TABLE=embedding_table.json", text) + self.assertIn("export ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE=1", text) + self.assertNotIn("MC_DATA_EMBEDDING_ROW", text) + self.assertIn("export ALIEN_JDL_LPMRUNNUMBER=544653", text) + self.assertTrue(text.startswith("#!/bin/bash\n")) + self.assertIn("#JDL_OUTPUT=", text) + outputs = text.split("#JDL_OUTPUT=")[1].split("\n")[0].split(",") + for f in ("AO2D*.root@disk=2", "embedding_row.json@disk=2", "embedding_status*.tsv@disk=2"): + self.assertIn(f, outputs) + self.assertIn("anchorMC_DataEmbedding.sh", text) + + def test_outputs_that_may_be_absent_are_patterns(self): + # JAliEn fails a successful job whose plain output file is missing; a pattern may match nothing + outputs = [o.split("@")[0] for o in se.OUTPUTS] + self.assertNotIn("AO2D.root", outputs) + self.assertNotIn("embedding_status.tsv", outputs) + + def test_rows_script_takes_row_from_subjob_id(self): + text = se.job_script({"name": "rows", "cycle": None, "prodsplit": 3, "collection": None}, ANCHOR_ENV) + self.assertIn("export ALIEN_JDL_MC_DATA_EMBEDDING_ROW=${ALIEN_JDL_MC_DATA_EMBEDDING_ROW:-${ALIEN_O2DPG_GRIDSUBMIT_SUBJOBID}}", text) + self.assertNotIn("MC_DATA_EMBEDDING_CYCLE", text) + + def test_anchor_env_may_set_tuning_knobs(self): + env = ANCHOR_ENV + "export ALIEN_JDL_MC_DATA_EMBEDDING_PARALLEL=4\nexport ALIEN_JDL_MC_DATA_EMBEDDING_RETRIES=3\n" + text = se.job_script({"name": "cycle0", "cycle": 0, "prodsplit": None, "collection": ["x"]}, env) + self.assertIn("ALIEN_JDL_MC_DATA_EMBEDDING_PARALLEL=4", text) + + def test_anchor_env_must_not_set_embedding_variables(self): + with self.assertRaises(ValueError): + se.job_script({"name": "cycle0", "cycle": 0, "prodsplit": None, "collection": ["x"]}, + ANCHOR_ENV + "export ALIEN_JDL_MC_DATA_EMBEDDING_CYCLE=5\n") + + +class TestGridSubmitArgs(unittest.TestCase): + + def test_args_for_cycle(self): + args = se.grid_submit_args({"name": "cycle0", "cycle": 0, "prodsplit": None, "collection": ["x"]}, + "job.sh", "list.txt", "/alice/cern.ch/user/s/swenzel/emb/embedding_table.json", "emb_544653", True, ["--ttl", "36000"]) + self.assertEqual(args, ["--script", "job.sh", "--jobname", "emb_544653_cycle0", + "--inputfile", "/alice/cern.ch/user/s/swenzel/emb/embedding_table.json", + "--split-on-collection", "list.txt", "--dry", "--ttl", "36000"]) + + def test_args_for_rows(self): + args = se.grid_submit_args({"name": "rows", "cycle": None, "prodsplit": 3, "collection": None}, + "job.sh", None, "/t/embedding_table.json", "emb", False, []) + self.assertEqual(args, ["--script", "job.sh", "--jobname", "emb_rows", "--inputfile", "/t/embedding_table.json", + "--prodsplit", "3"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/MC/utils/o2dpg_data_embedding_utils.py b/MC/utils/o2dpg_data_embedding_utils.py index b152d991e..148a39768 100644 --- a/MC/utils/o2dpg_data_embedding_utils.py +++ b/MC/utils/o2dpg_data_embedding_utils.py @@ -14,6 +14,7 @@ import re from ROOT import o2 # for CCDB import argparse +import json import sys class lhc_constants: @@ -150,7 +151,8 @@ def find_tree_key(keys, pattern): folders = { k.split("/")[0] : 1 for k in raw_keys if "O2bc_001" in k } folders = [ k for k in folders.keys() ] - folders = folders[:max_folders] + if max_folders is not None: + folders = folders[:max_folders] print ("have ", len(raw_keys), f" in file {filepath}") @@ -178,11 +180,15 @@ def find_tree_key(keys, pattern): # do the actual timeframe structure calculation; we only take collisions with a trigger decision attached triggered = combined[combined["fTriggerMask"] != 0] + folderkey = folder + '@' + filepath + if triggered.empty: + # a DF without triggered collisions has no timeframe to embed + merged[folderkey] = pd.DataFrame({'timeframeID': [], 'position_vectors': []}) + continue timeframe_structure = triggered.groupby('timeframeID').apply( lambda g: list(zip(g['fGlobalBC'], g['fPosX'], g['fPosY'], g['fPosZ'], g['orbit'], g['bc_within_orbit'], g['fCollisionTime'])) ).reset_index(name='position_vectors') - folderkey = folder + '@' + filepath merged[folderkey] = timeframe_structure # data per folder if include_dataframe: merged["data"] = combined @@ -247,11 +253,21 @@ def fetch_bccoll_to_localFile(alien_file, local_filename): return True +def time_ordered_collisions(collisions): + """ + Sorts collisions (globalBC, x, y, z, orbit, bc, collisionTime[ns]) by their time, BC plus + collision time. The digitizers need time-ordered interactions; the AO2D order is by BC only, + and a negative collision time can move a collision before one in an earlier BC. + """ + return sorted(collisions, key=lambda c: c[0] * lhc_constants.LHCBunchSpacingNS + c[6]) + + def convert_to_digicontext(aod_timeframe, df_folder, timeframeID=-1): """ converts AOD collision information from AO2D to collision context which can be used for MC """ + aod_timeframe = time_ordered_collisions(aod_timeframe) # we create the digitization context object digicontext=o2.steer.DigitizationContext() @@ -301,44 +317,126 @@ def convert_to_digicontext(aod_timeframe, df_folder, timeframeID=-1): digicontext.saveToFile(f"collission_context_{df_folder}:{timeframeID}.root") -def process_data_AO2D(file_name, run_number, upper_limit = -1): +def units_from_timeframe_structure(merged): """ - Creates all the collision contexts + DF folder name -> sorted list of timeframe IDs with at least one triggered collision, + from the output of get_timeframe_structure. A DF without such collisions maps to []. """ - timeframe_data = [] + units = {} + for key, frame in merged.items(): + df = key.split('@')[0] + units[df] = sorted(int(tf) for tf in frame['timeframeID']) if len(frame) > 0 else [] + return units - local_filename = "local.root" - fetch_bccoll_to_localFile(file_name, local_filename) - # fetch run_info object +def select_folders(merged, dfs): + """Restricts the get_timeframe_structure output to the given DF folder names.""" + wanted = set(dfs) + return {key: frame for key, frame in merged.items() if key.split('@')[0] in wanted} + + +def expectation_mismatches(units, expected): + """ + Compares the timeframes found per DF with the ones the planner recorded. + Returns a list of human-readable mismatches (empty when they agree). + """ + problems = [] + for df in sorted(expected): + if df not in units: + problems.append(f"{df}: expected {sorted(expected[df])}, found none") + elif sorted(units[df]) != sorted(expected[df]): + problems.append(f"{df}: expected {sorted(expected[df])}, found {sorted(units[df])}") + return problems + + +def read_timeframe_structure(file_name, run_number): + """Fetches the BC and collision tables of a (possibly remote) AO2D and returns the timeframe structure.""" + local_filename = "local.root" + if not fetch_bccoll_to_localFile(file_name, local_filename): + raise RuntimeError(f"Could not fetch BC/collision tables from {file_name}") run_info = retrieve_Aggregated_RunInfos(run_number) - merged = get_timeframe_structure(local_filename, run_info, max_folders=1000) + return get_timeframe_structure(local_filename, run_info, max_folders=None) + + +def units_from_file(file_name, run_number): + """DF folder name -> list of timeframe IDs to embed, for one data AO2D.""" + return units_from_timeframe_structure(read_timeframe_structure(file_name, run_number)) + + +def set_context_interaction_rate(context_file, rate): + """ + Stores the interaction rate (Hz) in a collision context taken from data. The contexts built + from data carry no rate, but the TPC looper generator and the digitizers read it from there. + """ + if rate <= 0: + raise ValueError(f"invalid interaction rate {rate}") + context = o2.steer.DigitizationContext.loadFromFile(context_file) + context.setDigitizerInteractionRate(float(rate)) + context.saveToFile(context_file) + + +def process_data_AO2D(file_name, run_number, upper_limit = -1, dfs = None, expected = None): + """ + Creates the collision contexts, one per (DF folder, timeframe). + dfs restricts this to the given DF folders; expected (DF -> timeframe IDs) is checked first + and a disagreement exits with code 3, so a job never embeds something else than planned. + """ + merged = read_timeframe_structure(file_name, run_number) print ("Got " + str(len(merged)) + " datasets") - timeframe_data.append(merged) + if dfs is not None: + merged = select_folders(merged, dfs) + if expected is not None: + problems = expectation_mismatches(units_from_timeframe_structure(merged), expected) + if problems: + for p in problems: + print (f"ERROR: timeframe structure differs from the plan: {p}") + sys.exit(3) counter = 0 - for d in timeframe_data: - for key in d: - result = d[key] - for index, row in result.iterrows(): - if upper_limit >= 0 and counter >= upper_limit: - break - tf = row['timeframeID'] - cols = row['position_vectors'] - df = key.split('@')[0] # this is the DF folder name - convert_to_digicontext(cols, df, tf) - counter = counter + 1 + for key in merged: + result = merged[key] + for index, row in result.iterrows(): + if upper_limit >= 0 and counter >= upper_limit: + break + tf = row['timeframeID'] + cols = row['position_vectors'] + df = key.split('@')[0] # this is the DF folder name + convert_to_digicontext(cols, df, tf) + counter = counter + 1 def main(): parser = argparse.ArgumentParser(description='Extracts collision contexts from reconstructed AO2D') - parser.add_argument("--run-number", type=int, help="Run number to anchor to", required=True) - parser.add_argument("--aod-file", type=str, help="Data AO2D file (can be on AliEn)", required=True) + parser.add_argument("--run-number", type=int, help="Run number to anchor to") + parser.add_argument("--aod-file", type=str, help="Data AO2D file (can be on AliEn)") parser.add_argument("--limit", type=int, default=-1, help="Upper limit of timeframes to be extracted (-1 is no limit)") + parser.add_argument("--dfs", type=str, default=None, help="Comma-separated DF folders to treat (default: all)") + parser.add_argument("--expect-tfs", type=str, default=None, help="JSON file DF -> timeframe IDs the plan expects; exit 3 on a difference") + parser.add_argument("--list-units", type=str, default=None, help="Only write DF -> timeframe IDs as JSON to this file") + parser.add_argument("--set-interaction-rate", type=float, default=None, help="Only store this interaction rate (Hz) in the contexts given by --context") + parser.add_argument("--context", type=str, nargs="+", default=[], help="Collision context files for --set-interaction-rate") args = parser.parse_args() - process_data_AO2D(args.aod_file, args.run_number, args.limit) + if args.set_interaction_rate is not None: + for c in args.context: + set_context_interaction_rate(c, args.set_interaction_rate) + return 0 + if args.run_number is None or args.aod_file is None: + parser.error("--run-number and --aod-file are required") + + if args.list_units: + with open(args.list_units, "w") as f: + json.dump(units_from_file(args.aod_file, args.run_number), f, indent=1) + return 0 + + dfs = args.dfs.split(",") if args.dfs else None + expected = None + if args.expect_tfs: + with open(args.expect_tfs) as f: + expected = json.load(f) + process_data_AO2D(args.aod_file, args.run_number, args.limit, dfs, expected) + return 0 if __name__ == "__main__": sys.exit(main()) diff --git a/MC/utils/tests/test_data_embedding_utils.py b/MC/utils/tests/test_data_embedding_utils.py new file mode 100644 index 000000000..37b6196f9 --- /dev/null +++ b/MC/utils/tests/test_data_embedding_utils.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Offline tests for the DF selection in o2dpg_data_embedding_utils (needs ROOT + pandas, no GRID).""" +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +try: + import pandas as pd + import o2dpg_data_embedding_utils as utils # needs ROOT +except ImportError: # pragma: no cover + utils = None + + +def structure(tfs_per_folder): + """Mimics get_timeframe_structure: 'DF_x@file' -> DataFrame(timeframeID, position_vectors).""" + merged = {} + for df, tfs in tfs_per_folder.items(): + merged[f"{df}@local.root"] = pd.DataFrame({"timeframeID": tfs, + "position_vectors": [[(0, 0., 0., 0., 0, 0, 0.)]] * len(tfs)}) + return merged + + +@unittest.skipIf(utils is None, "needs ROOT and pandas") +class TestDFSelection(unittest.TestCase): + + def test_units_from_timeframe_structure(self): + units = utils.units_from_timeframe_structure(structure({"DF_7": [31, 30], "DF_5": [12], "DF_9": []})) + self.assertEqual(units, {"DF_5": [12], "DF_7": [30, 31], "DF_9": []}) + + def test_select_rows_keeps_only_requested_dfs(self): + merged = structure({"DF_5": [12], "DF_7": [30, 31], "DF_9": [40]}) + selected = utils.select_folders(merged, ["DF_7", "DF_9"]) + self.assertEqual(sorted(k.split("@")[0] for k in selected), ["DF_7", "DF_9"]) + + def test_missing_requested_df_is_a_mismatch(self): + units = {"DF_5": [12]} + self.assertEqual(utils.expectation_mismatches(units, {"DF_5": [12], "DF_6": [13]}), ["DF_6: expected [13], found none"]) + + def test_different_tfs_are_a_mismatch(self): + units = {"DF_5": [12, 13]} + self.assertEqual(utils.expectation_mismatches(units, {"DF_5": [12]}), ["DF_5: expected [12], found [12, 13]"]) + + def test_matching_expectation_is_clean(self): + self.assertEqual(utils.expectation_mismatches({"DF_5": [12], "DF_6": [1]}, {"DF_5": [12]}), []) + + +@unittest.skipIf(utils is None, "needs ROOT and pandas") +class TestTimeframeStructureFile(unittest.TestCase): + """get_timeframe_structure on a small AO2D-like file written with uproot.""" + + RUN_INFO = {"OrbitReset": 0, "FirstOrbit": 1000, "OrbitsPerTF": 32} + + def write_file(self, path): + import numpy as np + import uproot + first_bc = 1000 * 3564 + with uproot.recreate(path) as f: + # DF_1: two triggered collisions in TF 0 and one untriggered + f["DF_1/O2bc_001"] = {"fGlobalBC": np.array([first_bc + 10, first_bc + 20, first_bc + 30], dtype=np.uint64), + "fTriggerMask": np.array([1, 1, 0], dtype=np.uint64)} + f["DF_1/O2collision_001"] = {"fIndexBCs": np.array([0, 1, 2], dtype=np.int32), + "fPosX": np.zeros(3, dtype=np.float32), "fPosY": np.zeros(3, dtype=np.float32), + "fPosZ": np.zeros(3, dtype=np.float32), "fCollisionTime": np.zeros(3, dtype=np.float32)} + # DF_2: no triggered collision at all + f["DF_2/O2bc_001"] = {"fGlobalBC": np.array([first_bc + 40], dtype=np.uint64), + "fTriggerMask": np.array([0], dtype=np.uint64)} + f["DF_2/O2collision_001"] = {"fIndexBCs": np.array([0], dtype=np.int32), + "fPosX": np.zeros(1, dtype=np.float32), "fPosY": np.zeros(1, dtype=np.float32), + "fPosZ": np.zeros(1, dtype=np.float32), "fCollisionTime": np.zeros(1, dtype=np.float32)} + + def test_empty_df_does_not_crash(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "aod.root") + self.write_file(path) + merged = utils.get_timeframe_structure(path, self.RUN_INFO, max_folders=None) + self.assertEqual(utils.units_from_timeframe_structure(merged), {"DF_1": [0], "DF_2": []}) + + +@unittest.skipIf(utils is None, "needs ROOT and pandas") +class TestTimeOrder(unittest.TestCase): + # (globalBC, x, y, z, orbit, bc_within_orbit, collisionTime ns) + A = (1000, 0., 0., 1., 0, 1000, 0.0) + B = (1001, 0., 0., 2., 0, 1001, -40.0) # 1 BC later, but 40 ns earlier in time + C = (1002, 0., 0., 3., 0, 1002, 3.0) + + def test_time_ordered_collisions(self): + self.assertEqual(utils.time_ordered_collisions([self.A, self.B, self.C]), [self.B, self.A, self.C]) + + def test_order_kept_when_already_ordered(self): + self.assertEqual(utils.time_ordered_collisions([self.A, self.C]), [self.A, self.C]) + + def test_context_records_are_time_ordered(self): + import tempfile + from ROOT import o2 + with tempfile.TemporaryDirectory() as d: + cwd = os.getcwd() + os.chdir(d) + try: + utils.convert_to_digicontext([self.A, self.B, self.C], "DF_1", 7) + ctx = o2.steer.DigitizationContext.loadFromFile("collission_context_DF_1:7.root") + finally: + os.chdir(cwd) + times = [r.getTimeNS() for r in ctx.getEventRecords()] + self.assertEqual(times, sorted(times)) + zs = [v.Z() for v in ctx.getInteractionVertices()] + self.assertEqual(zs, [2., 1., 3.]) + + +@unittest.skipIf(utils is None, "needs ROOT and pandas") +class TestContextInteractionRate(unittest.TestCase): + + def test_set_interaction_rate(self): + import tempfile + import ROOT + from ROOT import o2 + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "collisioncontext.root") + ctx = o2.steer.DigitizationContext() + ctx.saveToFile(path) + self.assertLess(o2.steer.DigitizationContext.loadFromFile(path).getDigitizerInteractionRate(), 0) + utils.set_context_interaction_rate(path, 31234.) + self.assertAlmostEqual(o2.steer.DigitizationContext.loadFromFile(path).getDigitizerInteractionRate(), 31234., places=0) + + def test_rejects_non_positive_rate(self): + with self.assertRaises(ValueError): + utils.set_context_interaction_rate("unused.root", 0) + + +if __name__ == "__main__": + unittest.main()