Skip to content

Commit a5fbb8e

Browse files
sawenzelclaude
andcommitted
Run a planned set of DF folders per MC-DATA embedding job
This lets anchorMC_DataEmbedding.sh treat one row of a work table and makes the job robust at scale. - With ALIEN_JDL_MC_DATA_EMBEDDING_TABLE the job takes its data file and DF folders from the table. - Each timeframe gets its own seed from the job id (ALIEN_PROC_ID), its DF folder and its timeframe ID. - Failed tasks are retried twice, and workflow creation is tried twice. - Each parallel workflow gets its share of the job's CPU and memory. - The final merge orders inputs by DF folder and timeframe and cannot split a folder on size. - Timeframes are reported done, failed or excluded in embedding_status.tsv; a failed one makes the job exit non-zero, excluded ones do not. - The excluded-timestamp and workflow-creation-only paths no longer nest the workspaces. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 6015805 commit a5fbb8e

1 file changed

Lines changed: 113 additions & 16 deletions

File tree

‎MC/run/ANCHOR/anchorMC_DataEmbedding.sh‎

Lines changed: 113 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,15 @@ export ALIEN_JDL_ADDTIMESERIESINMC=${ALIEN_JDL_ADDTIMESERIESINMC:-1}
129129
[ -z "${ALIEN_JDL_LPMANCHORPRODUCTION}" ] && { echo_error "Set ALIEN_JDL_LPMANCHORPRODUCTION or ANCHORPRODUCTION" ; exit 1 ; }
130130
[ -z "${ALIEN_JDL_LPMANCHORYEAR}" ] && { echo_error "Set ALIEN_JDL_LPMANCHORYEAR or ANCHORYEAR" ; exit 1 ; }
131131

132+
# MC-DATA embedding: one timeframe per data timeframe, placed by its ID, not by split
133+
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ] || [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then
134+
NTIMEFRAMES=${NTIMEFRAMES:-1}
135+
SPLITID=${SPLITID:-1}
136+
PRODSPLIT=${PRODSPLIT:-1}
137+
# the MC timeframes must have the length of the data timeframes
138+
[ "${ALIEN_JDL_MC_ORBITS_PER_TF}" ] && { echo_error "ALIEN_JDL_MC_ORBITS_PER_TF cannot be used for MC-DATA embedding" ; exit 1 ; }
139+
fi
140+
132141
[ -z "${NTIMEFRAMES}" ] && { echo_error "Set NTIMEFRAMES" ; exit 1 ; }
133142
[ -z "${SPLITID}" ] && { echo_error "Set SPLITID" ; exit 1 ; }
134143
[ -z "${PRODSPLIT}" ] && { echo_error "Set PRODSPLIT" ; exit 1 ; }
@@ -339,15 +348,54 @@ else
339348
fi
340349
PUBLISH_MCPRODINFO_OPTION=""
341350

351+
EMBEDDING_DIR=${O2DPG_ROOT}/MC/run/ANCHOR/dataembedding
352+
EMBEDDING_SELECTION=""
353+
# with a work table, this job treats one row of it: a data AO2D and a set of whole DF folders
354+
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ]; then
355+
ROWINFO=$(python3 ${EMBEDDING_DIR}/embedding_row.py --table ${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE} \
356+
--expect-out embedding_expected_tfs.json --manifest embedding_row.json)
357+
[ $? != 0 ] && { echo_error "Could not determine the embedding row of this job" ; exit 1 ; }
358+
read -r AOD_FROM_TABLE EMBEDDING_DFS <<< "${ROWINFO}"
359+
export ALIEN_JDL_MC_DATA_EMBEDDING_AO2D=${AOD_FROM_TABLE}
360+
EMBEDDING_SELECTION="--dfs ${EMBEDDING_DFS} --expect-tfs embedding_expected_tfs.json"
361+
echo_info "Embedding into ${EMBEDDING_DFS} of ${AOD_FROM_TABLE}"
362+
fi
363+
342364
# let's take the input data AO2D from a JDL variable
343365
AOD_DATA_FILE=${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}
344366
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then
345367
NTIMEFRAMES=1
346368
NWORKERS=1 # these embedding jobs process only light pp type signals ... the parallelism will come via parallel workflows
347369
fi
348370

371+
# a limit would drop planned timeframes; without a table the old default of 8 timeframes stays
372+
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_TABLE}" ]; then
373+
[ "${DATA_EMBEDDING_LIMIT}" ] && { echo_error "DATA_EMBEDDING_LIMIT cannot be used with a work table" ; exit 1 ; }
374+
EMBEDDING_LIMIT_OPTION=""
375+
else
376+
EMBEDDING_LIMIT_OPTION="--limit ${DATA_EMBEDDING_LIMIT:-8}"
377+
fi
378+
349379
# call the python script to extract all collision contexts
350-
python3 ${O2DPG_ROOT}/MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} --limit ${DATA_EMBEDDING_LIMIT:-8}
380+
python3 ${O2DPG_ROOT}/MC/utils/o2dpg_data_embedding_utils.py --aod-file ${AOD_DATA_FILE} --run-number ${ALIEN_JDL_LPMRUNNUMBER} \
381+
${EMBEDDING_LIMIT_OPTION} ${EMBEDDING_SELECTION}
382+
CONTEXT_RC=$?
383+
if [ "${CONTEXT_RC}" != "0" ]; then
384+
echo_error "Extraction of the collision contexts failed with ${CONTEXT_RC}"
385+
exit ${CONTEXT_RC}
386+
fi
387+
if ! compgen -G "collission_context_*.root" > /dev/null; then
388+
echo_error "No collision context was extracted from ${AOD_DATA_FILE}"
389+
exit 1
390+
fi
391+
392+
# concurrent per-timeframe workflows; each runner gets its share of the CPU and memory of the job
393+
EMBEDDING_PARALLEL=${ALIEN_JDL_MC_DATA_EMBEDDING_PARALLEL:-8}
394+
RUNNER_CPU=$(( ${ALIEN_JDL_CPULIMIT:-8} / EMBEDDING_PARALLEL ))
395+
[ "${RUNNER_CPU}" -lt 1 ] && RUNNER_CPU=1
396+
# ALICE GRID slots provide 2 GB per core unless told otherwise
397+
RUNNER_MEM=$(( ${ALIEN_JDL_MC_DATA_EMBEDDING_MEMORY_MB:-$(( 2000 * ${ALIEN_JDL_CPULIMIT:-8} ))} / EMBEDDING_PARALLEL ))
398+
echo_info "Running ${EMBEDDING_PARALLEL} timeframe workflows in parallel, each with ${RUNNER_CPU} CPU and ${RUNNER_MEM} MB"
351399

352400
parallel_job_count=0
353401
failed_count=0
@@ -364,14 +412,14 @@ for external_context in collission_context_*.root; do
364412
echo "Treating timeframe ${anchoring_tf} coming from folder ${df_folder}"
365413

366414
# we do it in a separate workspace
367-
workspace="TF_${anchoring_tf}"
415+
workspace="${df_folder}_TF_${anchoring_tf}"
368416
mkdir "${workspace}"; cd "${workspace}"
369417
# fetch the apass reco anchoring config
370418
cp ../*.json .
371419

372-
# we need to adjust the SEED for each job
373-
JOBSEED=$SEED
374-
[ "$JOBSEED" != "-1" ] && let JOBSEED=JOBSEED+anchoring_tf
420+
# seed per timeframe from the job id (SEED, normally ALIEN_PROC_ID), the DF folder and the timeframe
421+
JOBSEED=$(python3 ${EMBEDDING_DIR}/embedding_plan_lib.py seed "${SEED}" "${df_folder}" "${anchoring_tf}")
422+
[ -z "${JOBSEED}" ] && { echo_error "Could not compute the seed of timeframe ${anchoring_tf}" ; exit 1 ; }
375423
echo "TF ${anchoring_tf} got seed ${JOBSEED}"
376424

377425
# these arguments will be digested by o2dpg_sim_workflow_anchored.py
@@ -406,18 +454,29 @@ for external_context in collission_context_*.root; do
406454

407455
anchoringLogFile=timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log
408456
# query CCDB has changed, w/o "_"
409-
${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> ${anchoringLogFile}
410-
WF_RC="${?}"
457+
# a second attempt covers transient failures (CCDB, slow tools on a busy node)
458+
for attempt in 1 2; do
459+
${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> ${anchoringLogFile}
460+
WF_RC="${?}"
461+
[ "${WF_RC}" == "0" ] && break
462+
echo_info "Workflow creation of timeframe ${anchoring_tf} failed with ${WF_RC} (attempt ${attempt})"
463+
cp ${anchoringLogFile} ${anchoringLogFile}.attempt${attempt}
464+
done
411465
if [ "${WF_RC}" != "0" ] ; then
412-
echo_error "Problem during anchor timestamp sampling and workflow creation. Exiting."
413-
exit ${WF_RC}
466+
# count it as a failed timeframe and go on, so that the job still writes its status and logs
467+
echo_error "Problem during anchor timestamp sampling and workflow creation of timeframe ${anchoring_tf} (${df_folder}):"
468+
tail -n 20 ${anchoringLogFile}
469+
((failed_count++))
470+
cd ..
471+
continue
414472
fi
415473

416474
TIMESTAMP=`grep "Determined timestamp to be" ${anchoringLogFile} | awk '//{print $6}'`
417475
echo_info "TIMESTAMP IS ${TIMESTAMP}"
418476

419477
if [ "${ONLY_WORKFLOW_CREATION}" ]; then
420-
continue # or exit
478+
cd ..
479+
continue
421480
fi
422481

423482
# check if this job is exluded because it falls inside a bad data-taking period
@@ -426,7 +485,9 @@ for external_context in collission_context_*.root; do
426485
# we can quit here; there is nothing to do
427486
# (apart from maybe creating a fake empty AO2D.root file or the like)
428487
echo "Timestamp is excluded from run. Nothing to do here"
429-
continue # or exit 0
488+
touch EXCLUDED # the final merge reports it as excluded, not as failed
489+
cd ..
490+
continue
430491
fi
431492

432493
# -- RUN THE MC WORKLOAD TO PRODUCE TARGETS --
@@ -459,13 +520,13 @@ for external_context in collission_context_*.root; do
459520
echo_info "Workflow will run with target specification ${targetString}"
460521

461522
${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${targetString} -j 1 \
462-
--cpu-limit ${ALIEN_JDL_CPULIMIT:-8} --dynamic-resources \
523+
--cpu-limit ${RUNNER_CPU} --mem-limit ${RUNNER_MEM} --optimistic-resources --dynamic-resources --retry-on-failure ${ALIEN_JDL_MC_DATA_EMBEDDING_RETRIES:-2} \
463524
${ALIEN_O2DPG_FILEGRAPH:+--remove-files-early ${ALIEN_O2DPG_FILEGRAPH}} \
464525
${ALIEN_O2DPG_ADDITIONAL_WORKFLOW_RUNNER_ARGS} &> pipeline_log &
465526

466527
((parallel_job_count++))
467528
# If limit reached, wait for one job to finish
468-
if ((parallel_job_count >= 8)); then
529+
if ((parallel_job_count >= EMBEDDING_PARALLEL)); then
469530
if ! wait -n; then
470531
((failed_count++))
471532
fi
@@ -481,10 +542,40 @@ while ((parallel_job_count > 0)); do
481542
((parallel_job_count--))
482543
done
483544

484-
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ]; then
545+
MERGE_RC=0
546+
missing_aods=0
547+
# only the default targets produce an AO2D per timeframe
548+
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ] && [ -z "${ONLY_WORKFLOW_CREATION}" ] && [ -z "${ALIEN_JDL_O2DPGWORKFLOWTARGET}" ]; then
485549
# produce the final merged AO2D
486-
find ./ -maxdepth 2 -mindepth 2 -name "AO2D.root" > aod_inputs.txt
487-
o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name # --merge-by-name restricts merging to folders of same name
550+
# sort by DF folder, then numerically by timeframe (from collission_context_<DF>:<TF>.root),
551+
# so --merge-by-name sees same-named folders one after another and timeframes stay in time order
552+
: > embedding_status.tsv
553+
: > aod_unsorted.tsv
554+
for ctx in collission_context_*.root; do
555+
name="${ctx#collission_context_}"
556+
df="${name%%:*}"
557+
tf="${name##*:}"
558+
tf="${tf%.root}"
559+
aod="./${df}_TF_${tf}/AO2D.root"
560+
if [ -f "./${df}_TF_${tf}/EXCLUDED" ]; then
561+
printf '%s\t%s\texcluded\n' "${df}" "${tf}" >> embedding_status.tsv
562+
elif [ -f "${aod}" ]; then
563+
printf '%s\t%s\t%s\n' "${df}" "${tf}" "${aod}" >> aod_unsorted.tsv
564+
printf '%s\t%s\tdone\n' "${df}" "${tf}" >> embedding_status.tsv
565+
else
566+
echo_error "No AO2D for timeframe ${tf} of ${df}"
567+
printf '%s\t%s\tfailed\n' "${df}" "${tf}" >> embedding_status.tsv
568+
((missing_aods++))
569+
fi
570+
done
571+
sort -t $'\t' -k1,1 -k2,2n aod_unsorted.tsv | cut -f3 > aod_inputs.txt
572+
if [ -s aod_inputs.txt ]; then
573+
# a huge --max-size: a size-triggered flush would make the next folder of the same name collide
574+
o2-aod-merger --input aod_inputs.txt --output AO2D.root --merge-by-name --max-size 1000000000000
575+
MERGE_RC=$?
576+
else
577+
echo_info "No AO2D to merge: all timeframes of this job were excluded or failed"
578+
fi
488579
fi
489580

490581
#
@@ -496,3 +587,9 @@ if [[ -n "$ALIEN_PROC_ID" ]]; then
496587
find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T -
497588
fi
498589
fi
590+
591+
# a job with a failed timeframe fails as a whole, so that it is resubmitted
592+
if [ "${ALIEN_JDL_MC_DATA_EMBEDDING_AO2D}" ] && (( failed_count > 0 || missing_aods > 0 || MERGE_RC != 0 )); then
593+
echo_error "Embedding incomplete: ${failed_count} failed workflows, ${missing_aods} missing AO2Ds, merger exit code ${MERGE_RC}"
594+
exit 1
595+
fi

0 commit comments

Comments
 (0)