Skip to content

Commit 9f615cf

Browse files
sawenzelclaude
andcommitted
Select DF folders and time-order collisions in the embedding utils
This extends the MC-DATA embedding utils for production use and adds unit tests. - --dfs restricts the collision contexts to given DF folders, and --expect-tfs stops with exit code 3 if the timeframes differ from the plan. - --list-units writes DF folder -> timeframe IDs as JSON. - Collisions are ordered by BC time plus collision time; the AO2D order made the ITS/MFT digitizer fail with "deduced ROFrame precedes already processed one". - DF folders without triggered collisions no longer crash, and files with more than 1000 DF folders are no longer truncated. - --set-interaction-rate stores a rate in existing contexts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 3e443ae commit 9f615cf

2 files changed

Lines changed: 255 additions & 24 deletions

File tree

‎MC/utils/o2dpg_data_embedding_utils.py‎

Lines changed: 122 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import re
1515
from ROOT import o2 # for CCDB
1616
import argparse
17+
import json
1718
import sys
1819

1920
class lhc_constants:
@@ -150,7 +151,8 @@ def find_tree_key(keys, pattern):
150151

151152
folders = { k.split("/")[0] : 1 for k in raw_keys if "O2bc_001" in k }
152153
folders = [ k for k in folders.keys() ]
153-
folders = folders[:max_folders]
154+
if max_folders is not None:
155+
folders = folders[:max_folders]
154156

155157
print ("have ", len(raw_keys), f" in file {filepath}")
156158

@@ -178,11 +180,15 @@ def find_tree_key(keys, pattern):
178180

179181
# do the actual timeframe structure calculation; we only take collisions with a trigger decision attached
180182
triggered = combined[combined["fTriggerMask"] != 0]
183+
folderkey = folder + '@' + filepath
184+
if triggered.empty:
185+
# a DF without triggered collisions has no timeframe to embed
186+
merged[folderkey] = pd.DataFrame({'timeframeID': [], 'position_vectors': []})
187+
continue
181188
timeframe_structure = triggered.groupby('timeframeID').apply(
182189
lambda g: list(zip(g['fGlobalBC'], g['fPosX'], g['fPosY'], g['fPosZ'], g['orbit'], g['bc_within_orbit'], g['fCollisionTime']))
183190
).reset_index(name='position_vectors')
184191

185-
folderkey = folder + '@' + filepath
186192
merged[folderkey] = timeframe_structure # data per folder
187193
if include_dataframe:
188194
merged["data"] = combined
@@ -247,11 +253,21 @@ def fetch_bccoll_to_localFile(alien_file, local_filename):
247253
return True
248254

249255

256+
def time_ordered_collisions(collisions):
257+
"""
258+
Sorts collisions (globalBC, x, y, z, orbit, bc, collisionTime[ns]) by their time, BC plus
259+
collision time. The digitizers need time-ordered interactions; the AO2D order is by BC only,
260+
and a negative collision time can move a collision before one in an earlier BC.
261+
"""
262+
return sorted(collisions, key=lambda c: c[0] * lhc_constants.LHCBunchSpacingNS + c[6])
263+
264+
250265
def convert_to_digicontext(aod_timeframe, df_folder, timeframeID=-1):
251266
"""
252267
converts AOD collision information from AO2D to collision context
253268
which can be used for MC
254269
"""
270+
aod_timeframe = time_ordered_collisions(aod_timeframe)
255271
# we create the digitization context object
256272
digicontext=o2.steer.DigitizationContext()
257273

@@ -301,44 +317,126 @@ def convert_to_digicontext(aod_timeframe, df_folder, timeframeID=-1):
301317
digicontext.saveToFile(f"collission_context_{df_folder}:{timeframeID}.root")
302318

303319

304-
def process_data_AO2D(file_name, run_number, upper_limit = -1):
320+
def units_from_timeframe_structure(merged):
305321
"""
306-
Creates all the collision contexts
322+
DF folder name -> sorted list of timeframe IDs with at least one triggered collision,
323+
from the output of get_timeframe_structure. A DF without such collisions maps to [].
307324
"""
308-
timeframe_data = []
325+
units = {}
326+
for key, frame in merged.items():
327+
df = key.split('@')[0]
328+
units[df] = sorted(int(tf) for tf in frame['timeframeID']) if len(frame) > 0 else []
329+
return units
309330

310-
local_filename = "local.root"
311-
fetch_bccoll_to_localFile(file_name, local_filename)
312331

313-
# fetch run_info object
332+
def select_folders(merged, dfs):
333+
"""Restricts the get_timeframe_structure output to the given DF folder names."""
334+
wanted = set(dfs)
335+
return {key: frame for key, frame in merged.items() if key.split('@')[0] in wanted}
336+
337+
338+
def expectation_mismatches(units, expected):
339+
"""
340+
Compares the timeframes found per DF with the ones the planner recorded.
341+
Returns a list of human-readable mismatches (empty when they agree).
342+
"""
343+
problems = []
344+
for df in sorted(expected):
345+
if df not in units:
346+
problems.append(f"{df}: expected {sorted(expected[df])}, found none")
347+
elif sorted(units[df]) != sorted(expected[df]):
348+
problems.append(f"{df}: expected {sorted(expected[df])}, found {sorted(units[df])}")
349+
return problems
350+
351+
352+
def read_timeframe_structure(file_name, run_number):
353+
"""Fetches the BC and collision tables of a (possibly remote) AO2D and returns the timeframe structure."""
354+
local_filename = "local.root"
355+
if not fetch_bccoll_to_localFile(file_name, local_filename):
356+
raise RuntimeError(f"Could not fetch BC/collision tables from {file_name}")
314357
run_info = retrieve_Aggregated_RunInfos(run_number)
315-
merged = get_timeframe_structure(local_filename, run_info, max_folders=1000)
358+
return get_timeframe_structure(local_filename, run_info, max_folders=None)
359+
360+
361+
def units_from_file(file_name, run_number):
362+
"""DF folder name -> list of timeframe IDs to embed, for one data AO2D."""
363+
return units_from_timeframe_structure(read_timeframe_structure(file_name, run_number))
364+
365+
366+
def set_context_interaction_rate(context_file, rate):
367+
"""
368+
Stores the interaction rate (Hz) in a collision context taken from data. The contexts built
369+
from data carry no rate, but the TPC looper generator and the digitizers read it from there.
370+
"""
371+
if rate <= 0:
372+
raise ValueError(f"invalid interaction rate {rate}")
373+
context = o2.steer.DigitizationContext.loadFromFile(context_file)
374+
context.setDigitizerInteractionRate(float(rate))
375+
context.saveToFile(context_file)
376+
377+
378+
def process_data_AO2D(file_name, run_number, upper_limit = -1, dfs = None, expected = None):
379+
"""
380+
Creates the collision contexts, one per (DF folder, timeframe).
381+
dfs restricts this to the given DF folders; expected (DF -> timeframe IDs) is checked first
382+
and a disagreement exits with code 3, so a job never embeds something else than planned.
383+
"""
384+
merged = read_timeframe_structure(file_name, run_number)
316385
print ("Got " + str(len(merged)) + " datasets")
317-
timeframe_data.append(merged)
386+
if dfs is not None:
387+
merged = select_folders(merged, dfs)
388+
if expected is not None:
389+
problems = expectation_mismatches(units_from_timeframe_structure(merged), expected)
390+
if problems:
391+
for p in problems:
392+
print (f"ERROR: timeframe structure differs from the plan: {p}")
393+
sys.exit(3)
318394

319395
counter = 0
320-
for d in timeframe_data:
321-
for key in d:
322-
result = d[key]
323-
for index, row in result.iterrows():
324-
if upper_limit >= 0 and counter >= upper_limit:
325-
break
326-
tf = row['timeframeID']
327-
cols = row['position_vectors']
328-
df = key.split('@')[0] # this is the DF folder name
329-
convert_to_digicontext(cols, df, tf)
330-
counter = counter + 1
396+
for key in merged:
397+
result = merged[key]
398+
for index, row in result.iterrows():
399+
if upper_limit >= 0 and counter >= upper_limit:
400+
break
401+
tf = row['timeframeID']
402+
cols = row['position_vectors']
403+
df = key.split('@')[0] # this is the DF folder name
404+
convert_to_digicontext(cols, df, tf)
405+
counter = counter + 1
331406

332407

333408
def main():
334409
parser = argparse.ArgumentParser(description='Extracts collision contexts from reconstructed AO2D')
335410

336-
parser.add_argument("--run-number", type=int, help="Run number to anchor to", required=True)
337-
parser.add_argument("--aod-file", type=str, help="Data AO2D file (can be on AliEn)", required=True)
411+
parser.add_argument("--run-number", type=int, help="Run number to anchor to")
412+
parser.add_argument("--aod-file", type=str, help="Data AO2D file (can be on AliEn)")
338413
parser.add_argument("--limit", type=int, default=-1, help="Upper limit of timeframes to be extracted (-1 is no limit)")
414+
parser.add_argument("--dfs", type=str, default=None, help="Comma-separated DF folders to treat (default: all)")
415+
parser.add_argument("--expect-tfs", type=str, default=None, help="JSON file DF -> timeframe IDs the plan expects; exit 3 on a difference")
416+
parser.add_argument("--list-units", type=str, default=None, help="Only write DF -> timeframe IDs as JSON to this file")
417+
parser.add_argument("--set-interaction-rate", type=float, default=None, help="Only store this interaction rate (Hz) in the contexts given by --context")
418+
parser.add_argument("--context", type=str, nargs="+", default=[], help="Collision context files for --set-interaction-rate")
339419
args = parser.parse_args()
340420

341-
process_data_AO2D(args.aod_file, args.run_number, args.limit)
421+
if args.set_interaction_rate is not None:
422+
for c in args.context:
423+
set_context_interaction_rate(c, args.set_interaction_rate)
424+
return 0
425+
if args.run_number is None or args.aod_file is None:
426+
parser.error("--run-number and --aod-file are required")
427+
428+
if args.list_units:
429+
with open(args.list_units, "w") as f:
430+
json.dump(units_from_file(args.aod_file, args.run_number), f, indent=1)
431+
return 0
432+
433+
dfs = args.dfs.split(",") if args.dfs else None
434+
expected = None
435+
if args.expect_tfs:
436+
with open(args.expect_tfs) as f:
437+
expected = json.load(f)
438+
process_data_AO2D(args.aod_file, args.run_number, args.limit, dfs, expected)
439+
return 0
342440

343441
if __name__ == "__main__":
344442
sys.exit(main())
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#!/usr/bin/env python3
2+
"""Offline tests for the DF selection in o2dpg_data_embedding_utils (needs ROOT + pandas, no GRID)."""
3+
import os
4+
import sys
5+
import unittest
6+
7+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
8+
9+
try:
10+
import pandas as pd
11+
import o2dpg_data_embedding_utils as utils # needs ROOT
12+
except ImportError: # pragma: no cover
13+
utils = None
14+
15+
16+
def structure(tfs_per_folder):
17+
"""Mimics get_timeframe_structure: 'DF_x@file' -> DataFrame(timeframeID, position_vectors)."""
18+
merged = {}
19+
for df, tfs in tfs_per_folder.items():
20+
merged[f"{df}@local.root"] = pd.DataFrame({"timeframeID": tfs,
21+
"position_vectors": [[(0, 0., 0., 0., 0, 0, 0.)]] * len(tfs)})
22+
return merged
23+
24+
25+
@unittest.skipIf(utils is None, "needs ROOT and pandas")
26+
class TestDFSelection(unittest.TestCase):
27+
28+
def test_units_from_timeframe_structure(self):
29+
units = utils.units_from_timeframe_structure(structure({"DF_7": [31, 30], "DF_5": [12], "DF_9": []}))
30+
self.assertEqual(units, {"DF_5": [12], "DF_7": [30, 31], "DF_9": []})
31+
32+
def test_select_rows_keeps_only_requested_dfs(self):
33+
merged = structure({"DF_5": [12], "DF_7": [30, 31], "DF_9": [40]})
34+
selected = utils.select_folders(merged, ["DF_7", "DF_9"])
35+
self.assertEqual(sorted(k.split("@")[0] for k in selected), ["DF_7", "DF_9"])
36+
37+
def test_missing_requested_df_is_a_mismatch(self):
38+
units = {"DF_5": [12]}
39+
self.assertEqual(utils.expectation_mismatches(units, {"DF_5": [12], "DF_6": [13]}), ["DF_6: expected [13], found none"])
40+
41+
def test_different_tfs_are_a_mismatch(self):
42+
units = {"DF_5": [12, 13]}
43+
self.assertEqual(utils.expectation_mismatches(units, {"DF_5": [12]}), ["DF_5: expected [12], found [12, 13]"])
44+
45+
def test_matching_expectation_is_clean(self):
46+
self.assertEqual(utils.expectation_mismatches({"DF_5": [12], "DF_6": [1]}, {"DF_5": [12]}), [])
47+
48+
49+
@unittest.skipIf(utils is None, "needs ROOT and pandas")
50+
class TestTimeframeStructureFile(unittest.TestCase):
51+
"""get_timeframe_structure on a small AO2D-like file written with uproot."""
52+
53+
RUN_INFO = {"OrbitReset": 0, "FirstOrbit": 1000, "OrbitsPerTF": 32}
54+
55+
def write_file(self, path):
56+
import numpy as np
57+
import uproot
58+
first_bc = 1000 * 3564
59+
with uproot.recreate(path) as f:
60+
# DF_1: two triggered collisions in TF 0 and one untriggered
61+
f["DF_1/O2bc_001"] = {"fGlobalBC": np.array([first_bc + 10, first_bc + 20, first_bc + 30], dtype=np.uint64),
62+
"fTriggerMask": np.array([1, 1, 0], dtype=np.uint64)}
63+
f["DF_1/O2collision_001"] = {"fIndexBCs": np.array([0, 1, 2], dtype=np.int32),
64+
"fPosX": np.zeros(3, dtype=np.float32), "fPosY": np.zeros(3, dtype=np.float32),
65+
"fPosZ": np.zeros(3, dtype=np.float32), "fCollisionTime": np.zeros(3, dtype=np.float32)}
66+
# DF_2: no triggered collision at all
67+
f["DF_2/O2bc_001"] = {"fGlobalBC": np.array([first_bc + 40], dtype=np.uint64),
68+
"fTriggerMask": np.array([0], dtype=np.uint64)}
69+
f["DF_2/O2collision_001"] = {"fIndexBCs": np.array([0], dtype=np.int32),
70+
"fPosX": np.zeros(1, dtype=np.float32), "fPosY": np.zeros(1, dtype=np.float32),
71+
"fPosZ": np.zeros(1, dtype=np.float32), "fCollisionTime": np.zeros(1, dtype=np.float32)}
72+
73+
def test_empty_df_does_not_crash(self):
74+
import tempfile
75+
with tempfile.TemporaryDirectory() as d:
76+
path = os.path.join(d, "aod.root")
77+
self.write_file(path)
78+
merged = utils.get_timeframe_structure(path, self.RUN_INFO, max_folders=None)
79+
self.assertEqual(utils.units_from_timeframe_structure(merged), {"DF_1": [0], "DF_2": []})
80+
81+
82+
@unittest.skipIf(utils is None, "needs ROOT and pandas")
83+
class TestTimeOrder(unittest.TestCase):
84+
# (globalBC, x, y, z, orbit, bc_within_orbit, collisionTime ns)
85+
A = (1000, 0., 0., 1., 0, 1000, 0.0)
86+
B = (1001, 0., 0., 2., 0, 1001, -40.0) # 1 BC later, but 40 ns earlier in time
87+
C = (1002, 0., 0., 3., 0, 1002, 3.0)
88+
89+
def test_time_ordered_collisions(self):
90+
self.assertEqual(utils.time_ordered_collisions([self.A, self.B, self.C]), [self.B, self.A, self.C])
91+
92+
def test_order_kept_when_already_ordered(self):
93+
self.assertEqual(utils.time_ordered_collisions([self.A, self.C]), [self.A, self.C])
94+
95+
def test_context_records_are_time_ordered(self):
96+
import tempfile
97+
from ROOT import o2
98+
with tempfile.TemporaryDirectory() as d:
99+
cwd = os.getcwd()
100+
os.chdir(d)
101+
try:
102+
utils.convert_to_digicontext([self.A, self.B, self.C], "DF_1", 7)
103+
ctx = o2.steer.DigitizationContext.loadFromFile("collission_context_DF_1:7.root")
104+
finally:
105+
os.chdir(cwd)
106+
times = [r.getTimeNS() for r in ctx.getEventRecords()]
107+
self.assertEqual(times, sorted(times))
108+
zs = [v.Z() for v in ctx.getInteractionVertices()]
109+
self.assertEqual(zs, [2., 1., 3.])
110+
111+
112+
@unittest.skipIf(utils is None, "needs ROOT and pandas")
113+
class TestContextInteractionRate(unittest.TestCase):
114+
115+
def test_set_interaction_rate(self):
116+
import tempfile
117+
import ROOT
118+
from ROOT import o2
119+
with tempfile.TemporaryDirectory() as d:
120+
path = os.path.join(d, "collisioncontext.root")
121+
ctx = o2.steer.DigitizationContext()
122+
ctx.saveToFile(path)
123+
self.assertLess(o2.steer.DigitizationContext.loadFromFile(path).getDigitizerInteractionRate(), 0)
124+
utils.set_context_interaction_rate(path, 31234.)
125+
self.assertAlmostEqual(o2.steer.DigitizationContext.loadFromFile(path).getDigitizerInteractionRate(), 31234., places=0)
126+
127+
def test_rejects_non_positive_rate(self):
128+
with self.assertRaises(ValueError):
129+
utils.set_context_interaction_rate("unused.root", 0)
130+
131+
132+
if __name__ == "__main__":
133+
unittest.main()

0 commit comments

Comments
 (0)