Skip to content

Commit 6015805

Browse files
sawenzelclaude
andcommitted
Add tools to plan, submit and check MC-DATA embedding productions
This provides new tools in MC/run/ANCHOR/dataembedding to run MC-DATA embedding as a production. - plan_embedding.py samples a fraction of a run's data AO2Ds and packs whole DF folders into jobs (embedding_table.json); a DF folder is never split. - submit_embedding.py submits the table via grid_submit.sh: one split-on-collection masterjob per cycle, so jobs run close to their data, or one production split. - check_embedding_output.py merges per parent file and checks the DF folders, the parentFiles links and that MC collisions sit at the data collision BCs; excluded timeframes are taken into account. - embedding_row.py finds a job's row from its input file and cycle, or from a row index. - Seeds are a hash of the job id (ALIEN_PROC_ID), DF folder and timeframe; distinct timeframes share a seed only by chance. plan_embedding.py --run 544653 --data-path /alice/data/2023/LHC23zzm/544653/apass4 \ --fraction 0.01 --seed 1 --tfs-per-job 200 --out plan submit_embedding.py --plan plan --anchor-env anchor.env --table-dir <alien dir> --jobname emb --submit check_embedding_output.py check --plan plan --mc <MC AO2D> --parent <data AO2D> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 9f615cf commit 6015805

9 files changed

Lines changed: 1161 additions & 0 deletions
Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Merges the outputs of an MC-DATA embedding production per parent data file and checks
4+
# that each merged MC AO2D is correctly linked to its parent:
5+
# - no DF folder appears twice, every planned DF is there and no other one;
6+
# - every DF has a parentFiles entry pointing to the parent;
7+
# - per DF, the MC truth collisions sit at the BCs of the data collisions, and as many.
8+
#
9+
# Examples:
10+
# check_embedding_output.py merge --plan plan_544653 --outputs out_cycle0 out_cycle1 --out merged
11+
# check_embedding_output.py check --plan plan_544653 --mc merged/0000/AO2D.root --parent alien:///.../AO2D.root
12+
13+
import argparse
14+
import collections
15+
import glob
16+
import json
17+
import os
18+
import re
19+
import subprocess
20+
import sys
21+
22+
HERE = os.path.dirname(os.path.abspath(__file__))
23+
sys.path.insert(0, HERE)
24+
sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "utils"))
25+
26+
import embedding_plan_lib as lib
27+
28+
29+
# ---- pure checks -------------------------------------------------------------------------------
30+
31+
def planned_dfs(table, lfn):
32+
"""DF folders the plan expects in the MC output of one parent (empty DFs have no row)."""
33+
return sorted((df for row in table["files"][lfn] for df in row["dfs"]), key=lib.df_number)
34+
35+
36+
def coverage(planned, produced):
37+
"""(missing, duplicated, unexpected) DF folder names, comparing the plan with the MC file's folders."""
38+
counts = collections.Counter(produced)
39+
missing = sorted((df for df in planned if df not in counts), key=lib.df_number)
40+
duplicated = sorted((df for df, n in counts.items() if n > 1), key=lib.df_number)
41+
unexpected = sorted((df for df in counts if df not in set(planned)), key=lib.df_number)
42+
return missing, duplicated, unexpected
43+
44+
45+
def bc_subset(mc_bcs, data_bcs):
46+
return set(mc_bcs) <= set(data_bcs)
47+
48+
49+
def count_mismatches(mc_counts, data_counts):
50+
return [f"{df}: {mc_counts.get(df, 0)} MC collisions for {data_counts[df]} data collisions"
51+
for df in sorted(data_counts, key=lib.df_number) if mc_counts.get(df, 0) != data_counts[df]]
52+
53+
54+
def mccollision_key(keys):
55+
"""The MC collision table among a DF folder's keys (not O2mccollisionlabel)."""
56+
for k in keys:
57+
if re.match(r"^O2mccollision_\d+(;\d+)?$", k):
58+
return k
59+
return None
60+
61+
62+
def data_counts(data, planned, excluded):
63+
"""DF -> number of data collisions to compare with, leaving out excluded (DF, TF)."""
64+
return {df: sum(len(bcs) for tf, bcs in data.get(df, {}).items() if (df, tf) not in excluded) for df in planned}
65+
66+
67+
def excluded_from_status(status_paths):
68+
"""(DF, TF) pairs reported as excluded in the given embedding_status.tsv files."""
69+
out = set()
70+
for status in status_paths:
71+
if os.path.exists(status):
72+
with open(status) as f:
73+
for line in f:
74+
parts = line.split()
75+
if len(parts) == 3 and parts[2] == "excluded":
76+
out.add((parts[0], int(parts[1])))
77+
return out
78+
79+
80+
def excluded_tfs(manifest_paths):
81+
"""(DF, TF) pairs a job reported as excluded, from the embedding_status.tsv beside each manifest."""
82+
return excluded_from_status([os.path.join(os.path.dirname(p), "embedding_status.tsv") for p in manifest_paths])
83+
84+
85+
def missing_aod_is_error(manifest_path):
86+
"""A job without AO2D.root is fine only when its status file reports all its timeframes excluded."""
87+
status = os.path.join(os.path.dirname(manifest_path), "embedding_status.tsv")
88+
if not os.path.exists(status):
89+
return True
90+
with open(status) as f:
91+
states = [line.split()[2] for line in f if len(line.split()) == 3]
92+
return not states or any(st != "excluded" for st in states)
93+
94+
95+
def fully_excluded_dfs(table, lfn, excluded):
96+
"""Planned DF folders of a parent whose timeframes were all excluded; they have no MC output."""
97+
out = []
98+
for row in table["files"][lfn]:
99+
for df, tfs in row["dfs"].items():
100+
if tfs and all((df, tf) in excluded for tf in tfs):
101+
out.append(df)
102+
return sorted(out, key=lib.df_number)
103+
104+
105+
def expected_dfs(table, lfn, excluded):
106+
"""DF folders the MC output of a parent must contain."""
107+
gone = set(fully_excluded_dfs(table, lfn, excluded))
108+
return [df for df in planned_dfs(table, lfn) if df not in gone]
109+
110+
111+
def group_by_parent(manifests):
112+
"""
113+
Parent LFN -> AO2D files of the jobs that embedded into it, ordered by their first DF,
114+
so that --merge-by-name sees the DF folders in order. manifests: [(manifest path, content)].
115+
"""
116+
groups = collections.defaultdict(list)
117+
for path, m in manifests:
118+
first = min(lib.df_number(df) for df in m["dfs"])
119+
groups[m["lfn"]].append((first, os.path.abspath(os.path.join(os.path.dirname(path), "AO2D.root"))))
120+
return {lfn: [aod for _, aod in sorted(items)] for lfn, items in groups.items()}
121+
122+
123+
# ---- ROOT-based readers ------------------------------------------------------------------------
124+
125+
def _open(path):
126+
"""TFile.Open that first connects to AliEn for alien:// paths."""
127+
import ROOT
128+
if path.startswith("alien://") and not ROOT.gGrid:
129+
ROOT.TGrid.Connect("alien://")
130+
f = ROOT.TFile.Open(path)
131+
if not f or f.IsZombie():
132+
raise OSError(f"Failed to open file {path}")
133+
return f
134+
135+
136+
def mc_df_folders(path):
137+
"""DF folder names in a file, once per key cycle (a duplicated folder shows up twice)."""
138+
import ROOT
139+
f = _open(path)
140+
names = [k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")]
141+
f.Close()
142+
return names
143+
144+
145+
def parent_files(path):
146+
import ROOT
147+
f = _open(path)
148+
m = f.Get("parentFiles")
149+
out = {}
150+
if m:
151+
it = m.MakeIterator()
152+
key = it.Next()
153+
while key:
154+
out[key.GetName()] = m.GetValue(key).GetName()
155+
key = it.Next()
156+
f.Close()
157+
return out
158+
159+
160+
def mc_truth_bcs(path):
161+
"""DF -> list of globalBCs of the MC truth collisions (read with ROOT: AO2Ds may use zstd)."""
162+
import ROOT
163+
f = _open(path)
164+
out = {}
165+
for df in sorted({k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")}):
166+
d = f.Get(df)
167+
keys = [k.GetName() for k in d.GetListOfKeys()]
168+
mcc = mccollision_key(keys)
169+
bcs = [k for k in keys if k.startswith("O2bc_")]
170+
if mcc is None or not bcs:
171+
out[df] = []
172+
continue
173+
index = ROOT.RDataFrame(d.Get(mcc)).AsNumpy(["fIndexBCs"])["fIndexBCs"]
174+
gbc = ROOT.RDataFrame(d.Get(bcs[0])).AsNumpy(["fGlobalBC"])["fGlobalBC"]
175+
out[df] = [int(gbc[i]) for i in index]
176+
f.Close()
177+
return out
178+
179+
180+
def data_collision_bcs(parent, run):
181+
"""DF -> TF -> globalBCs of the triggered data collisions (the ones the contexts were made from)."""
182+
import o2dpg_data_embedding_utils as utils
183+
merged = utils.read_timeframe_structure(parent, run)
184+
out = {}
185+
for key, frame in merged.items():
186+
out[key.split("@")[0]] = {int(tf): [int(c[0]) for c in cols]
187+
for tf, cols in zip(frame["timeframeID"], frame["position_vectors"])}
188+
return out
189+
190+
191+
def check(mc, parent, table, workdir, excluded=frozenset()):
192+
"""Returns a list of problems (empty: the MC file is correctly linked to its parent)."""
193+
problems = []
194+
planned = expected_dfs(table, parent, excluded)
195+
missing, duplicated, unexpected = coverage(planned, mc_df_folders(mc))
196+
problems += [f"missing DF {df}" for df in missing]
197+
problems += [f"duplicated DF {df}" for df in duplicated]
198+
problems += [f"DF {df} is not in the plan" for df in unexpected]
199+
200+
links = parent_files(mc)
201+
for df in planned:
202+
if links.get(df) is None:
203+
problems.append(f"{df} has no parentFiles entry")
204+
elif links[df].replace("alien://", "") != parent.replace("alien://", ""):
205+
problems.append(f"{df} points to {links[df]}")
206+
207+
cwd = os.getcwd()
208+
os.chdir(workdir) # the data tables are copied locally here
209+
try:
210+
data = data_collision_bcs(parent, table["run"])
211+
finally:
212+
os.chdir(cwd)
213+
mc_bcs = mc_truth_bcs(mc)
214+
for df in planned:
215+
data_bcs = [bc for bcs in data.get(df, {}).values() for bc in bcs]
216+
if not bc_subset(mc_bcs.get(df, []), data_bcs):
217+
problems.append(f"{df}: MC collisions at BCs without a data collision")
218+
problems += count_mismatches({df: len(mc_bcs.get(df, [])) for df in planned},
219+
data_counts(data, planned, excluded))
220+
return problems
221+
222+
223+
# ---- commands ----------------------------------------------------------------------------------
224+
225+
def cmd_merge(args, table):
226+
manifests = []
227+
for d in args.outputs:
228+
for p in sorted(glob.glob(os.path.join(d, "**", "embedding_row.json"), recursive=True)):
229+
with open(p) as f:
230+
manifests.append((p, json.load(f)))
231+
groups = group_by_parent(manifests)
232+
rc = 0
233+
for i, parent in enumerate(sorted(groups)):
234+
outdir = os.path.join(args.out, f"{i:04d}")
235+
os.makedirs(outdir, exist_ok=True)
236+
manifests_here = [p for p, m in manifests if m["lfn"] == parent]
237+
aods = [a for a in groups[parent] if os.path.exists(a)]
238+
bad = [p for p in manifests_here
239+
if not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(p)), "AO2D.root")) and missing_aod_is_error(p)]
240+
if bad:
241+
print(f"ERROR: {len(bad)} job outputs without AO2D.root for {parent}")
242+
rc = 1
243+
if not aods:
244+
if not bad:
245+
print(f"{parent}: all timeframes excluded, nothing to merge")
246+
continue
247+
with open(os.path.join(outdir, "inputs.txt"), "w") as f:
248+
f.write("\n".join(aods) + "\n")
249+
with open(os.path.join(outdir, "parent.txt"), "w") as f:
250+
f.write(parent + "\n")
251+
merge = subprocess.run(["o2-aod-merger", "--input", "inputs.txt", "--output", "AO2D.root",
252+
"--merge-by-name", "--max-size", "1000000000000"], cwd=outdir)
253+
if merge.returncode != 0:
254+
print(f"ERROR: merging for {parent} failed")
255+
rc = 1
256+
continue
257+
excluded = excluded_tfs([p for p, m in manifests if m["lfn"] == parent])
258+
problems = check(os.path.join(outdir, "AO2D.root"), parent, table, outdir, excluded)
259+
print(f"{parent} -> {outdir}/AO2D.root: {'OK' if not problems else 'FAILED'}")
260+
for p in problems:
261+
print(f" {p}")
262+
rc |= 1 if problems else 0
263+
planned_parents = set(table["files"])
264+
for lfn in sorted(planned_parents - set(groups)):
265+
print(f"ERROR: no job output at all for {lfn}")
266+
rc = 1
267+
return rc
268+
269+
270+
def main():
271+
parser = argparse.ArgumentParser(description="Merge and check MC-DATA embedding outputs")
272+
sub = parser.add_subparsers(dest="cmd", required=True)
273+
m = sub.add_parser("merge", help="Merge job outputs per parent and check each result")
274+
m.add_argument("--plan", required=True)
275+
m.add_argument("--outputs", nargs="+", required=True, help="Local directories holding the job outputs")
276+
m.add_argument("--out", required=True)
277+
c = sub.add_parser("check", help="Check one MC AO2D against its parent")
278+
c.add_argument("--plan", required=True)
279+
c.add_argument("--mc", required=True)
280+
c.add_argument("--parent", required=True)
281+
c.add_argument("--status", nargs="*", default=[], help="embedding_status.tsv files of the jobs, for excluded timeframes")
282+
args = parser.parse_args()
283+
284+
with open(os.path.join(args.plan, "embedding_table.json")) as f:
285+
table = json.load(f)
286+
if args.cmd == "merge":
287+
return cmd_merge(args, table)
288+
# the data tables are copied beside a local MC file, or into the current directory for alien://
289+
workdir = os.getcwd() if args.mc.startswith("alien://") else os.path.dirname(os.path.abspath(args.mc))
290+
problems = check(args.mc, args.parent, table, workdir, excluded_from_status(args.status))
291+
for p in problems:
292+
print(p)
293+
print("OK" if not problems else "FAILED")
294+
return 1 if problems else 0
295+
296+
297+
if __name__ == "__main__":
298+
sys.exit(main())

0 commit comments

Comments
 (0)