|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# |
| 3 | +# Merges the outputs of an MC-DATA embedding production per parent data file and checks |
| 4 | +# that each merged MC AO2D is correctly linked to its parent: |
| 5 | +# - no DF folder appears twice, every planned DF is there and no other one; |
| 6 | +# - every DF has a parentFiles entry pointing to the parent; |
| 7 | +# - per DF, the MC truth collisions sit at the BCs of the data collisions, and as many. |
| 8 | +# |
| 9 | +# Examples: |
| 10 | +# check_embedding_output.py merge --plan plan_544653 --outputs out_cycle0 out_cycle1 --out merged |
| 11 | +# check_embedding_output.py check --plan plan_544653 --mc merged/0000/AO2D.root --parent alien:///.../AO2D.root |
| 12 | + |
| 13 | +import argparse |
| 14 | +import collections |
| 15 | +import glob |
| 16 | +import json |
| 17 | +import os |
| 18 | +import re |
| 19 | +import subprocess |
| 20 | +import sys |
| 21 | + |
| 22 | +HERE = os.path.dirname(os.path.abspath(__file__)) |
| 23 | +sys.path.insert(0, HERE) |
| 24 | +sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "utils")) |
| 25 | + |
| 26 | +import embedding_plan_lib as lib |
| 27 | + |
| 28 | + |
| 29 | +# ---- pure checks ------------------------------------------------------------------------------- |
| 30 | + |
| 31 | +def planned_dfs(table, lfn): |
| 32 | + """DF folders the plan expects in the MC output of one parent (empty DFs have no row).""" |
| 33 | + return sorted((df for row in table["files"][lfn] for df in row["dfs"]), key=lib.df_number) |
| 34 | + |
| 35 | + |
| 36 | +def coverage(planned, produced): |
| 37 | + """(missing, duplicated, unexpected) DF folder names, comparing the plan with the MC file's folders.""" |
| 38 | + counts = collections.Counter(produced) |
| 39 | + missing = sorted((df for df in planned if df not in counts), key=lib.df_number) |
| 40 | + duplicated = sorted((df for df, n in counts.items() if n > 1), key=lib.df_number) |
| 41 | + unexpected = sorted((df for df in counts if df not in set(planned)), key=lib.df_number) |
| 42 | + return missing, duplicated, unexpected |
| 43 | + |
| 44 | + |
| 45 | +def bc_subset(mc_bcs, data_bcs): |
| 46 | + return set(mc_bcs) <= set(data_bcs) |
| 47 | + |
| 48 | + |
| 49 | +def count_mismatches(mc_counts, data_counts): |
| 50 | + return [f"{df}: {mc_counts.get(df, 0)} MC collisions for {data_counts[df]} data collisions" |
| 51 | + for df in sorted(data_counts, key=lib.df_number) if mc_counts.get(df, 0) != data_counts[df]] |
| 52 | + |
| 53 | + |
| 54 | +def mccollision_key(keys): |
| 55 | + """The MC collision table among a DF folder's keys (not O2mccollisionlabel).""" |
| 56 | + for k in keys: |
| 57 | + if re.match(r"^O2mccollision_\d+(;\d+)?$", k): |
| 58 | + return k |
| 59 | + return None |
| 60 | + |
| 61 | + |
| 62 | +def data_counts(data, planned, excluded): |
| 63 | + """DF -> number of data collisions to compare with, leaving out excluded (DF, TF).""" |
| 64 | + return {df: sum(len(bcs) for tf, bcs in data.get(df, {}).items() if (df, tf) not in excluded) for df in planned} |
| 65 | + |
| 66 | + |
| 67 | +def excluded_from_status(status_paths): |
| 68 | + """(DF, TF) pairs reported as excluded in the given embedding_status.tsv files.""" |
| 69 | + out = set() |
| 70 | + for status in status_paths: |
| 71 | + if os.path.exists(status): |
| 72 | + with open(status) as f: |
| 73 | + for line in f: |
| 74 | + parts = line.split() |
| 75 | + if len(parts) == 3 and parts[2] == "excluded": |
| 76 | + out.add((parts[0], int(parts[1]))) |
| 77 | + return out |
| 78 | + |
| 79 | + |
| 80 | +def excluded_tfs(manifest_paths): |
| 81 | + """(DF, TF) pairs a job reported as excluded, from the embedding_status.tsv beside each manifest.""" |
| 82 | + return excluded_from_status([os.path.join(os.path.dirname(p), "embedding_status.tsv") for p in manifest_paths]) |
| 83 | + |
| 84 | + |
| 85 | +def missing_aod_is_error(manifest_path): |
| 86 | + """A job without AO2D.root is fine only when its status file reports all its timeframes excluded.""" |
| 87 | + status = os.path.join(os.path.dirname(manifest_path), "embedding_status.tsv") |
| 88 | + if not os.path.exists(status): |
| 89 | + return True |
| 90 | + with open(status) as f: |
| 91 | + states = [line.split()[2] for line in f if len(line.split()) == 3] |
| 92 | + return not states or any(st != "excluded" for st in states) |
| 93 | + |
| 94 | + |
| 95 | +def fully_excluded_dfs(table, lfn, excluded): |
| 96 | + """Planned DF folders of a parent whose timeframes were all excluded; they have no MC output.""" |
| 97 | + out = [] |
| 98 | + for row in table["files"][lfn]: |
| 99 | + for df, tfs in row["dfs"].items(): |
| 100 | + if tfs and all((df, tf) in excluded for tf in tfs): |
| 101 | + out.append(df) |
| 102 | + return sorted(out, key=lib.df_number) |
| 103 | + |
| 104 | + |
| 105 | +def expected_dfs(table, lfn, excluded): |
| 106 | + """DF folders the MC output of a parent must contain.""" |
| 107 | + gone = set(fully_excluded_dfs(table, lfn, excluded)) |
| 108 | + return [df for df in planned_dfs(table, lfn) if df not in gone] |
| 109 | + |
| 110 | + |
| 111 | +def group_by_parent(manifests): |
| 112 | + """ |
| 113 | + Parent LFN -> AO2D files of the jobs that embedded into it, ordered by their first DF, |
| 114 | + so that --merge-by-name sees the DF folders in order. manifests: [(manifest path, content)]. |
| 115 | + """ |
| 116 | + groups = collections.defaultdict(list) |
| 117 | + for path, m in manifests: |
| 118 | + first = min(lib.df_number(df) for df in m["dfs"]) |
| 119 | + groups[m["lfn"]].append((first, os.path.abspath(os.path.join(os.path.dirname(path), "AO2D.root")))) |
| 120 | + return {lfn: [aod for _, aod in sorted(items)] for lfn, items in groups.items()} |
| 121 | + |
| 122 | + |
| 123 | +# ---- ROOT-based readers ------------------------------------------------------------------------ |
| 124 | + |
| 125 | +def _open(path): |
| 126 | + """TFile.Open that first connects to AliEn for alien:// paths.""" |
| 127 | + import ROOT |
| 128 | + if path.startswith("alien://") and not ROOT.gGrid: |
| 129 | + ROOT.TGrid.Connect("alien://") |
| 130 | + f = ROOT.TFile.Open(path) |
| 131 | + if not f or f.IsZombie(): |
| 132 | + raise OSError(f"Failed to open file {path}") |
| 133 | + return f |
| 134 | + |
| 135 | + |
| 136 | +def mc_df_folders(path): |
| 137 | + """DF folder names in a file, once per key cycle (a duplicated folder shows up twice).""" |
| 138 | + import ROOT |
| 139 | + f = _open(path) |
| 140 | + names = [k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")] |
| 141 | + f.Close() |
| 142 | + return names |
| 143 | + |
| 144 | + |
| 145 | +def parent_files(path): |
| 146 | + import ROOT |
| 147 | + f = _open(path) |
| 148 | + m = f.Get("parentFiles") |
| 149 | + out = {} |
| 150 | + if m: |
| 151 | + it = m.MakeIterator() |
| 152 | + key = it.Next() |
| 153 | + while key: |
| 154 | + out[key.GetName()] = m.GetValue(key).GetName() |
| 155 | + key = it.Next() |
| 156 | + f.Close() |
| 157 | + return out |
| 158 | + |
| 159 | + |
| 160 | +def mc_truth_bcs(path): |
| 161 | + """DF -> list of globalBCs of the MC truth collisions (read with ROOT: AO2Ds may use zstd).""" |
| 162 | + import ROOT |
| 163 | + f = _open(path) |
| 164 | + out = {} |
| 165 | + for df in sorted({k.GetName() for k in f.GetListOfKeys() if k.GetName().startswith("DF_")}): |
| 166 | + d = f.Get(df) |
| 167 | + keys = [k.GetName() for k in d.GetListOfKeys()] |
| 168 | + mcc = mccollision_key(keys) |
| 169 | + bcs = [k for k in keys if k.startswith("O2bc_")] |
| 170 | + if mcc is None or not bcs: |
| 171 | + out[df] = [] |
| 172 | + continue |
| 173 | + index = ROOT.RDataFrame(d.Get(mcc)).AsNumpy(["fIndexBCs"])["fIndexBCs"] |
| 174 | + gbc = ROOT.RDataFrame(d.Get(bcs[0])).AsNumpy(["fGlobalBC"])["fGlobalBC"] |
| 175 | + out[df] = [int(gbc[i]) for i in index] |
| 176 | + f.Close() |
| 177 | + return out |
| 178 | + |
| 179 | + |
| 180 | +def data_collision_bcs(parent, run): |
| 181 | + """DF -> TF -> globalBCs of the triggered data collisions (the ones the contexts were made from).""" |
| 182 | + import o2dpg_data_embedding_utils as utils |
| 183 | + merged = utils.read_timeframe_structure(parent, run) |
| 184 | + out = {} |
| 185 | + for key, frame in merged.items(): |
| 186 | + out[key.split("@")[0]] = {int(tf): [int(c[0]) for c in cols] |
| 187 | + for tf, cols in zip(frame["timeframeID"], frame["position_vectors"])} |
| 188 | + return out |
| 189 | + |
| 190 | + |
| 191 | +def check(mc, parent, table, workdir, excluded=frozenset()): |
| 192 | + """Returns a list of problems (empty: the MC file is correctly linked to its parent).""" |
| 193 | + problems = [] |
| 194 | + planned = expected_dfs(table, parent, excluded) |
| 195 | + missing, duplicated, unexpected = coverage(planned, mc_df_folders(mc)) |
| 196 | + problems += [f"missing DF {df}" for df in missing] |
| 197 | + problems += [f"duplicated DF {df}" for df in duplicated] |
| 198 | + problems += [f"DF {df} is not in the plan" for df in unexpected] |
| 199 | + |
| 200 | + links = parent_files(mc) |
| 201 | + for df in planned: |
| 202 | + if links.get(df) is None: |
| 203 | + problems.append(f"{df} has no parentFiles entry") |
| 204 | + elif links[df].replace("alien://", "") != parent.replace("alien://", ""): |
| 205 | + problems.append(f"{df} points to {links[df]}") |
| 206 | + |
| 207 | + cwd = os.getcwd() |
| 208 | + os.chdir(workdir) # the data tables are copied locally here |
| 209 | + try: |
| 210 | + data = data_collision_bcs(parent, table["run"]) |
| 211 | + finally: |
| 212 | + os.chdir(cwd) |
| 213 | + mc_bcs = mc_truth_bcs(mc) |
| 214 | + for df in planned: |
| 215 | + data_bcs = [bc for bcs in data.get(df, {}).values() for bc in bcs] |
| 216 | + if not bc_subset(mc_bcs.get(df, []), data_bcs): |
| 217 | + problems.append(f"{df}: MC collisions at BCs without a data collision") |
| 218 | + problems += count_mismatches({df: len(mc_bcs.get(df, [])) for df in planned}, |
| 219 | + data_counts(data, planned, excluded)) |
| 220 | + return problems |
| 221 | + |
| 222 | + |
| 223 | +# ---- commands ---------------------------------------------------------------------------------- |
| 224 | + |
| 225 | +def cmd_merge(args, table): |
| 226 | + manifests = [] |
| 227 | + for d in args.outputs: |
| 228 | + for p in sorted(glob.glob(os.path.join(d, "**", "embedding_row.json"), recursive=True)): |
| 229 | + with open(p) as f: |
| 230 | + manifests.append((p, json.load(f))) |
| 231 | + groups = group_by_parent(manifests) |
| 232 | + rc = 0 |
| 233 | + for i, parent in enumerate(sorted(groups)): |
| 234 | + outdir = os.path.join(args.out, f"{i:04d}") |
| 235 | + os.makedirs(outdir, exist_ok=True) |
| 236 | + manifests_here = [p for p, m in manifests if m["lfn"] == parent] |
| 237 | + aods = [a for a in groups[parent] if os.path.exists(a)] |
| 238 | + bad = [p for p in manifests_here |
| 239 | + if not os.path.exists(os.path.join(os.path.dirname(os.path.abspath(p)), "AO2D.root")) and missing_aod_is_error(p)] |
| 240 | + if bad: |
| 241 | + print(f"ERROR: {len(bad)} job outputs without AO2D.root for {parent}") |
| 242 | + rc = 1 |
| 243 | + if not aods: |
| 244 | + if not bad: |
| 245 | + print(f"{parent}: all timeframes excluded, nothing to merge") |
| 246 | + continue |
| 247 | + with open(os.path.join(outdir, "inputs.txt"), "w") as f: |
| 248 | + f.write("\n".join(aods) + "\n") |
| 249 | + with open(os.path.join(outdir, "parent.txt"), "w") as f: |
| 250 | + f.write(parent + "\n") |
| 251 | + merge = subprocess.run(["o2-aod-merger", "--input", "inputs.txt", "--output", "AO2D.root", |
| 252 | + "--merge-by-name", "--max-size", "1000000000000"], cwd=outdir) |
| 253 | + if merge.returncode != 0: |
| 254 | + print(f"ERROR: merging for {parent} failed") |
| 255 | + rc = 1 |
| 256 | + continue |
| 257 | + excluded = excluded_tfs([p for p, m in manifests if m["lfn"] == parent]) |
| 258 | + problems = check(os.path.join(outdir, "AO2D.root"), parent, table, outdir, excluded) |
| 259 | + print(f"{parent} -> {outdir}/AO2D.root: {'OK' if not problems else 'FAILED'}") |
| 260 | + for p in problems: |
| 261 | + print(f" {p}") |
| 262 | + rc |= 1 if problems else 0 |
| 263 | + planned_parents = set(table["files"]) |
| 264 | + for lfn in sorted(planned_parents - set(groups)): |
| 265 | + print(f"ERROR: no job output at all for {lfn}") |
| 266 | + rc = 1 |
| 267 | + return rc |
| 268 | + |
| 269 | + |
| 270 | +def main(): |
| 271 | + parser = argparse.ArgumentParser(description="Merge and check MC-DATA embedding outputs") |
| 272 | + sub = parser.add_subparsers(dest="cmd", required=True) |
| 273 | + m = sub.add_parser("merge", help="Merge job outputs per parent and check each result") |
| 274 | + m.add_argument("--plan", required=True) |
| 275 | + m.add_argument("--outputs", nargs="+", required=True, help="Local directories holding the job outputs") |
| 276 | + m.add_argument("--out", required=True) |
| 277 | + c = sub.add_parser("check", help="Check one MC AO2D against its parent") |
| 278 | + c.add_argument("--plan", required=True) |
| 279 | + c.add_argument("--mc", required=True) |
| 280 | + c.add_argument("--parent", required=True) |
| 281 | + c.add_argument("--status", nargs="*", default=[], help="embedding_status.tsv files of the jobs, for excluded timeframes") |
| 282 | + args = parser.parse_args() |
| 283 | + |
| 284 | + with open(os.path.join(args.plan, "embedding_table.json")) as f: |
| 285 | + table = json.load(f) |
| 286 | + if args.cmd == "merge": |
| 287 | + return cmd_merge(args, table) |
| 288 | + # the data tables are copied beside a local MC file, or into the current directory for alien:// |
| 289 | + workdir = os.getcwd() if args.mc.startswith("alien://") else os.path.dirname(os.path.abspath(args.mc)) |
| 290 | + problems = check(args.mc, args.parent, table, workdir, excluded_from_status(args.status)) |
| 291 | + for p in problems: |
| 292 | + print(p) |
| 293 | + print("OK" if not problems else "FAILED") |
| 294 | + return 1 if problems else 0 |
| 295 | + |
| 296 | + |
| 297 | +if __name__ == "__main__": |
| 298 | + sys.exit(main()) |
0 commit comments