@@ -129,6 +129,15 @@ export ALIEN_JDL_ADDTIMESERIESINMC=${ALIEN_JDL_ADDTIMESERIESINMC:-1}
129129[ -z " ${ALIEN_JDL_LPMANCHORPRODUCTION} " ] && { echo_error " Set ALIEN_JDL_LPMANCHORPRODUCTION or ANCHORPRODUCTION" ; exit 1 ; }
130130[ -z " ${ALIEN_JDL_LPMANCHORYEAR} " ] && { echo_error " Set ALIEN_JDL_LPMANCHORYEAR or ANCHORYEAR" ; exit 1 ; }
131131
132+ # MC-DATA embedding: one timeframe per data timeframe, placed by its ID, not by split
133+ if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} " ] || [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} " ]; then
134+ NTIMEFRAMES=${NTIMEFRAMES:- 1}
135+ SPLITID=${SPLITID:- 1}
136+ PRODSPLIT=${PRODSPLIT:- 1}
137+ # the MC timeframes must have the length of the data timeframes
138+ [ " ${ALIEN_JDL_MC_ORBITS_PER_TF} " ] && { echo_error " ALIEN_JDL_MC_ORBITS_PER_TF cannot be used for MC-DATA embedding" ; exit 1 ; }
139+ fi
140+
132141[ -z " ${NTIMEFRAMES} " ] && { echo_error " Set NTIMEFRAMES" ; exit 1 ; }
133142[ -z " ${SPLITID} " ] && { echo_error " Set SPLITID" ; exit 1 ; }
134143[ -z " ${PRODSPLIT} " ] && { echo_error " Set PRODSPLIT" ; exit 1 ; }
@@ -339,15 +348,54 @@ else
339348fi
340349PUBLISH_MCPRODINFO_OPTION=" "
341350
351+ EMBEDDING_DIR=${O2DPG_ROOT} /MC/run/ANCHOR/dataembedding
352+ EMBEDDING_SELECTION=" "
353+ # with a work table, this job treats one row of it: a data AO2D and a set of whole DF folders
354+ if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} " ]; then
355+ ROWINFO=$( python3 ${EMBEDDING_DIR} /embedding_row.py --table ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} \
356+ --expect-out embedding_expected_tfs.json --manifest embedding_row.json)
357+ [ $? != 0 ] && { echo_error " Could not determine the embedding row of this job" ; exit 1 ; }
358+ read -r AOD_FROM_TABLE EMBEDDING_DFS <<< " ${ROWINFO}"
359+ export ALIEN_JDL_MC_DATA_EMBEDDING_AO2D=${AOD_FROM_TABLE}
360+ EMBEDDING_SELECTION=" --dfs ${EMBEDDING_DFS} --expect-tfs embedding_expected_tfs.json"
361+ echo_info " Embedding into ${EMBEDDING_DFS} of ${AOD_FROM_TABLE} "
362+ fi
363+
342364# let's take the input data AO2D from a JDL variable
343365AOD_DATA_FILE=${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}
344366if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} " ]; then
345367 NTIMEFRAMES=1
346368 NWORKERS=1 # these embedding jobs process only light pp type signals ... the parallelism will come via parallel workflows
347369fi
348370
371+ # a limit would drop planned timeframes; without a table the old default of 8 timeframes stays
372+ if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} " ]; then
373+ [ " ${DATA_EMBEDDING_LIMIT} " ] && { echo_error " DATA_EMBEDDING_LIMIT cannot be used with a work table" ; exit 1 ; }
374+ EMBEDDING_LIMIT_OPTION=" "
375+ else
376+ EMBEDDING_LIMIT_OPTION=" --limit ${DATA_EMBEDDING_LIMIT:- 8} "
377+ fi
378+
349379# call the python script to extract all collision contexts
350- python3 ${O2DPG_ROOT} /MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} --limit ${DATA_EMBEDDING_LIMIT:- 8}
380+ python3 ${O2DPG_ROOT} /MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} \
381+ ${EMBEDDING_LIMIT_OPTION} ${EMBEDDING_SELECTION}
382+ CONTEXT_RC=$?
383+ if [ " ${CONTEXT_RC} " != " 0" ]; then
384+ echo_error " Extraction of the collision contexts failed with ${CONTEXT_RC} "
385+ exit ${CONTEXT_RC}
386+ fi
387+ if ! compgen -G " collission_context_*.root" > /dev/null; then
388+ echo_error " No collision context was extracted from ${AOD_DATA_FILE} "
389+ exit 1
390+ fi
391+
392+ # concurrent per-timeframe workflows; each runner gets its share of the CPU and memory of the job
393+ EMBEDDING_PARALLEL=${ALIEN_JDL_MC_DATA_EMBEDDING_PARALLEL:- 8}
394+ RUNNER_CPU=$(( ${ALIEN_JDL_CPULIMIT:- 8} / EMBEDDING_PARALLEL ))
395+ [ " ${RUNNER_CPU} " -lt 1 ] && RUNNER_CPU=1
396+ # ALICE GRID slots provide 2 GB per core unless told otherwise
397+ RUNNER_MEM=$(( ${ALIEN_JDL_MC_DATA_EMBEDDING_MEMORY_MB:- $(( 2000 * ${ALIEN_JDL_CPULIMIT:- 8} ))} / EMBEDDING_PARALLEL ))
398+ echo_info " Running ${EMBEDDING_PARALLEL} timeframe workflows in parallel, each with ${RUNNER_CPU} CPU and ${RUNNER_MEM} MB"
351399
352400parallel_job_count=0
353401failed_count=0
@@ -364,14 +412,14 @@ for external_context in collission_context_*.root; do
364412 echo " Treating timeframe ${anchoring_tf} coming from folder ${df_folder} "
365413
366414 # we do it in a separate workspace
367- workspace=" TF_ ${anchoring_tf} "
415+ workspace=" ${df_folder} _TF_ ${anchoring_tf} "
368416 mkdir " ${workspace} " ; cd " ${workspace} "
369417 # fetch the apass reco anchoring config
370418 cp ../* .json .
371419
372- # we need to adjust the SEED for each job
373- JOBSEED=$SEED
374- [ " $ JOBSEED" != " -1 " ] && let JOBSEED=JOBSEED+ anchoring_tf
420+ # seed per timeframe from the job id ( SEED, normally ALIEN_PROC_ID), the DF folder and the timeframe
421+ JOBSEED=$( python3 ${EMBEDDING_DIR} /embedding_plan_lib.py seed " ${ SEED} " " ${df_folder} " " ${anchoring_tf} " )
422+ [ -z " ${ JOBSEED} " ] && { echo_error " Could not compute the seed of timeframe ${ anchoring_tf} " ; exit 1 ; }
375423 echo " TF ${anchoring_tf} got seed ${JOBSEED} "
376424
377425 # these arguments will be digested by o2dpg_sim_workflow_anchored.py
@@ -406,18 +454,29 @@ for external_context in collission_context_*.root; do
406454
407455 anchoringLogFile=timestampsampling_${ALIEN_JDL_LPMRUNNUMBER} .log
408456 # query CCDB has changed, w/o "_"
409- ${O2DPG_ROOT} /MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} & > ${anchoringLogFile}
410- WF_RC=" ${?} "
457+ # a second attempt covers transient failures (CCDB, slow tools on a busy node)
458+ for attempt in 1 2; do
459+ ${O2DPG_ROOT} /MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} & > ${anchoringLogFile}
460+ WF_RC=" ${?} "
461+ [ " ${WF_RC} " == " 0" ] && break
462+ echo_info " Workflow creation of timeframe ${anchoring_tf} failed with ${WF_RC} (attempt ${attempt} )"
463+ cp ${anchoringLogFile} ${anchoringLogFile} .attempt${attempt}
464+ done
411465 if [ " ${WF_RC} " != " 0" ] ; then
412- echo_error " Problem during anchor timestamp sampling and workflow creation. Exiting."
413- exit ${WF_RC}
466+ # count it as a failed timeframe and go on, so that the job still writes its status and logs
467+ echo_error " Problem during anchor timestamp sampling and workflow creation of timeframe ${anchoring_tf} (${df_folder} ):"
468+ tail -n 20 ${anchoringLogFile}
469+ (( failed_count++ ))
470+ cd ..
471+ continue
414472 fi
415473
416474 TIMESTAMP=` grep " Determined timestamp to be" ${anchoringLogFile} | awk ' //{print $6}' `
417475 echo_info " TIMESTAMP IS ${TIMESTAMP} "
418476
419477 if [ " ${ONLY_WORKFLOW_CREATION} " ]; then
420- continue # or exit
478+ cd ..
479+ continue
421480 fi
422481
423482 # check if this job is exluded because it falls inside a bad data-taking period
@@ -426,7 +485,9 @@ for external_context in collission_context_*.root; do
426485 # we can quit here; there is nothing to do
427486 # (apart from maybe creating a fake empty AO2D.root file or the like)
428487 echo " Timestamp is excluded from run. Nothing to do here"
429- continue # or exit 0
488+ touch EXCLUDED # the final merge reports it as excluded, not as failed
489+ cd ..
490+ continue
430491 fi
431492
432493 # -- RUN THE MC WORKLOAD TO PRODUCE TARGETS --
@@ -459,13 +520,13 @@ for external_context in collission_context_*.root; do
459520 echo_info " Workflow will run with target specification ${targetString} "
460521
461522 ${O2DPG_ROOT} /MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${targetString} -j 1 \
462- --cpu-limit ${ALIEN_JDL_CPULIMIT :- 8 } --dynamic-resources \
523+ --cpu-limit ${RUNNER_CPU} --mem-limit ${RUNNER_MEM } --optimistic-resources -- dynamic-resources --retry-on-failure ${ALIEN_JDL_MC_DATA_EMBEDDING_RETRIES :- 2} \
463524 ${ALIEN_O2DPG_FILEGRAPH: +--remove-files-early ${ALIEN_O2DPG_FILEGRAPH} } \
464525 ${ALIEN_O2DPG_ADDITIONAL_WORKFLOW_RUNNER_ARGS} & > pipeline_log &
465526
466527 (( parallel_job_count++ ))
467528 # If limit reached, wait for one job to finish
468- if (( parallel_job_count >= 8 )) ; then
529+ if (( parallel_job_count >= EMBEDDING_PARALLEL )) ; then
469530 if ! wait -n; then
470531 (( failed_count++ ))
471532 fi
@@ -481,10 +542,40 @@ while ((parallel_job_count > 0)); do
481542 (( parallel_job_count-- ))
482543done
483544
484- if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} " ]; then
545+ MERGE_RC=0
546+ missing_aods=0
547+ # only the default targets produce an AO2D per timeframe
548+ if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} " ] && [ -z " ${ONLY_WORKFLOW_CREATION} " ] && [ -z " ${ALIEN_JDL_O2DPGWORKFLOWTARGET} " ]; then
485549 # produce the final merged AO2D
486- find ./ -maxdepth 2 -mindepth 2 -name " AO2D.root" > aod_inputs.txt
487- o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name # --merge-by-name restricts merging to folders of same name
550+ # sort by DF folder, then numerically by timeframe (from collission_context_<DF>:<TF>.root),
551+ # so --merge-by-name sees same-named folders one after another and timeframes stay in time order
552+ : > embedding_status.tsv
553+ : > aod_unsorted.tsv
554+ for ctx in collission_context_* .root; do
555+ name=" ${ctx# collission_context_} "
556+ df=" ${name%%:* } "
557+ tf=" ${name##*: } "
558+ tf=" ${tf% .root} "
559+ aod=" ./${df} _TF_${tf} /AO2D.root"
560+ if [ -f " ./${df} _TF_${tf} /EXCLUDED" ]; then
561+ printf ' %s\t%s\texcluded\n' " ${df} " " ${tf} " >> embedding_status.tsv
562+ elif [ -f " ${aod} " ]; then
563+ printf ' %s\t%s\t%s\n' " ${df} " " ${tf} " " ${aod} " >> aod_unsorted.tsv
564+ printf ' %s\t%s\tdone\n' " ${df} " " ${tf} " >> embedding_status.tsv
565+ else
566+ echo_error " No AO2D for timeframe ${tf} of ${df} "
567+ printf ' %s\t%s\tfailed\n' " ${df} " " ${tf} " >> embedding_status.tsv
568+ (( missing_aods++ ))
569+ fi
570+ done
571+ sort -t $' \t ' -k1,1 -k2,2n aod_unsorted.tsv | cut -f3 > aod_inputs.txt
572+ if [ -s aod_inputs.txt ]; then
573+ # a huge --max-size: a size-triggered flush would make the next folder of the same name collide
574+ o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name --max-size 1000000000000
575+ MERGE_RC=$?
576+ else
577+ echo_info " No AO2D to merge: all timeframes of this job were excluded or failed"
578+ fi
488579fi
489580
490581#
@@ -496,3 +587,9 @@ if [[ -n "$ALIEN_PROC_ID" ]]; then
496587 find ./ \( -name " *.log*" -o -name " *mergerlog*" -o -name " *serverlog*" -o -name " *workerlog*" -o -name " *.root" \) | tar -czvf debug_full_archive.tgz -T -
497588 fi
498589fi
590+
591+ # a job with a failed timeframe fails as a whole, so that it is resubmitted
592+ if [ " ${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D} " ] && (( failed_count > 0 || missing_aods > 0 || MERGE_RC != 0 )) ; then
593+ echo_error " Embedding incomplete: ${failed_count} failed workflows, ${missing_aods} missing AO2Ds, merger exit code ${MERGE_RC} "
594+ exit 1
595+ fi
0 commit comments