From ab4cee6e68283fbcc8bd7b558f8271d565e7e879 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 12 Jan 2026 20:31:54 -0500 Subject: [PATCH 1/4] hipo-diff: do all banks, add sorting --- .../main/java/org/jlab/utils/HipoDiff.java | 296 +++++++++++------- 1 file changed, 188 insertions(+), 108 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index e0341362dd..b5dbbe8e4e 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -1,20 +1,186 @@ package org.jlab.utils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import org.jlab.jnp.hipo4.io.HipoReader; import org.jlab.jnp.hipo4.data.Bank; import org.jlab.jnp.hipo4.data.Event; import org.jlab.jnp.hipo4.data.Schema; -import java.util.HashMap; +import org.jlab.jnp.hipo4.data.SchemaFactory; import org.jlab.utils.options.OptionParser; public class HipoDiff { + /** + * Bank sortable by any integer columns. + * + */ + static class SortedBank extends Bank { + SortedBank(Schema s) { super(s); } + /** + * @param index the bank column indices to sort on + * @return the sorted row indices + */ + int[] getSorted(int... index) { + int[] rows = new int[getRows()]; + for (int row = 0; row < rows.length; row++) rows[row] = row; + // bubble sort: + for (int i = 0; i < rows.length - 1; i++) { + for (int j = 0; j < rows.length - i - 1; j++) { + for (int idx : index) { + if (idx >= this.getSchema().getElements()) break; + int x1 = getInt(idx, rows[j]); + int x2 = getInt(idx, rows[j + 1]); + if (x1 > x2) { + int tmp = rows[j]; + rows[j] = rows[j + 1]; + rows[j + 1] = tmp; + break; + } + else if (x1 < x2) break; + } + } + } + return rows; + } + } + + static int nrow = 0; + static int nevent = -1; + static int nentry = 0; + static int nbadevent = 0; + static int nbadrow = 0; + static int nbadentry = 0; + static int[] sortIndex = null; + + static int nmax; + static double tolerance; + static boolean verboseMode; + static boolean quietMode; + static Bank runConfigBank; + + static ArrayList banksA = new ArrayList<>(); + static ArrayList banksB = new ArrayList<>(); + static HashMap> badEntries = new HashMap<>(); + + public static void compare(HipoReader a, HipoReader b) { + Event eventA = new Event(); + Event eventB = new Event(); + while (a.hasNext() && b.hasNext() && (nmax < 1 || nevent < nmax)) { + if (++nevent % 5000 == 0) System.out.println("Analyzed " + nevent + " events"); + a.nextEvent(eventA); + b.nextEvent(eventB); + eventA.read(runConfigBank); + compare(eventA, eventB); + } + System.out.println("\n Analyzed " + nevent + " with " + nbadevent + " bad banks"); + System.out.println(nbadrow + "/" + nrow + " mismatched rows"); + System.out.println(nbadentry + "/" + nentry + " mismatched entry"); + for (String name : badEntries.keySet()) { + System.out.println(name + " " + badEntries.get(name)); + } + System.exit(nbadevent + nbadrow + nbadentry); + } + + public static void compare(Event a, Event b) { + for (int i = 0; i < banksA.size(); i++) { + a.read(banksA.get(i)); + b.read(banksB.get(i)); + compare(banksA.get(i), banksB.get(i)); + } + } + + public static void compare(SortedBank a, SortedBank b) { + if (a.getRows() != b.getRows()) { + System.out.println("========================= Different number of rows:"); + runConfigBank.show(); + a.show(); + b.show(); + nbadevent++; + System.out.println("========================="); + return; + } + int[] rowsA = sortIndex == null ? null : a.getSorted(sortIndex); + int[] rowsB = sortIndex == null ? null : b.getSorted(sortIndex); + for (int row = 0; row < a.getRows(); row++) { + boolean mismatch = false; + nrow++; + int rowA = sortIndex == null ? row : rowsA[row]; + int rowB = sortIndex == null ? row : rowsB[row]; + for (int j = 0; j < a.getSchema().getElements(); j++) { + final int type = a.getSchema().getType(j); + final String name = a.getSchema().getElementName(j); + int element = -1; + String values = ""; + nentry++; + switch (type) { + case 1: + if (a.getByte(name, rowA) != b.getByte(name, rowB)) { + element = j; + values += a.getByte(name, rowA) + "/" + b.getByte(name, rowB); + } + break; + case 2: + if (a.getShort(name, rowA) != b.getShort(name, rowB)) { + element = j; + values += a.getShort(name, rowA) + "/" + b.getShort(name, rowB); + } + break; + case 3: + if (a.getInt(name, rowA) != b.getInt(name, rowB)) { + element = j; + values += a.getInt(name, rowA) + "/" + b.getInt(name, rowB); + } + break; + case 4: + if ((!Double.isNaN(a.getFloat(name, rowA)) || !Double.isNaN(b.getFloat(name, rowB))) + && (!Double.isInfinite(a.getFloat(name, rowA)) || !Double.isInfinite(b.getFloat(name, rowB))) + && Math.abs(a.getFloat(name, rowA) - b.getFloat(name, rowB)) > tolerance) { + element = j; + values += a.getFloat(name, rowA) + "/" + b.getFloat(name, rowB); + } + break; + } + if (element >= 0) { + if (verboseMode) { + System.out.println("Bank.show " + a.getSchema().getName()); + a.show(); + b.show(); + } + if (!quietMode) { + System.out.println(a.getSchema().getName() + " mismatch at event " + runConfigBank.getInt("event", 0) + + " in row " + row + " for variable " + name + " with values " + values); + } + mismatch = true; + nbadentry++; + String bankName = a.getSchema().getName(); + String elementName = a.getSchema().getElementName(element); + if (!badEntries.containsKey(bankName)) { + badEntries.put(bankName, new HashMap<>()); + } + Map m = badEntries.get(bankName); + if (!m.containsKey(elementName)) { + m.put(elementName, 0); + } + m.put(elementName, m.get(elementName) + 1); + } + } + if (mismatch) { + nbadrow++; + } + } + } + public static void main(String args[]) { OptionParser op = new OptionParser("hipo-diff"); op.addOption("-t", "0.00001", "absolute tolerance for comparisons"); op.addOption("-n", "-1", "number of events"); - op.addRequired("-b", "name of bank to diff"); + op.addOption("-q", null, "quiet mode"); + op.addOption("-Q", null, "verbose mode"); + op.addOption("-b", null, "name of bank to diff"); + op.addOption("-s", null, "sort on column index"); op.setRequiresInputList(true); op.parse(args); if (op.getInputList().size() != 2) { @@ -23,120 +189,34 @@ public static void main(String args[]) { System.exit(1); } - final String bankName = op.getOption("-b").stringValue(); - final double tolerance = op.getOption("-t").doubleValue(); - final int nmax = op.getOption("-n").intValue(); + if (op.getOption("-s").stringValue() != null) { + String[] stmp = op.getOption("-s").stringValue().split(","); + sortIndex = new int[stmp.length]; + for (int i = 0; i < stmp.length; i++) sortIndex[i] = Integer.parseInt(stmp[i]); + } + verboseMode = op.getOption("-Q").stringValue() != null; + quietMode = op.getOption("-q").stringValue() != null; + nmax = op.getOption("-n").intValue(); + tolerance = op.getOption("-t").doubleValue(); HipoReader readerA = new HipoReader(); HipoReader readerB = new HipoReader(); readerA.open(op.getInputList().get(0)); readerB.open(op.getInputList().get(1)); + SchemaFactory sf = readerA.getSchemaFactory(); + runConfigBank = new Bank(sf.getSchema("RUN::config")); - Schema schema = readerA.getSchemaFactory().getSchema(bankName); - Bank bankA = new Bank(schema); - Bank bankB = new Bank(schema); - - Bank runConfigBank = new Bank(readerA.getSchemaFactory().getSchema("RUN::config")); - Event event = new Event(); - - int nevent = -1; - int nrow = 0; - int nentry = 0; - int nbadevent = 0; - int nbadrow = 0; - int nbadentry = 0; - HashMap badEntries = new HashMap<>(); - - while (readerA.hasNext() && readerB.hasNext() && (nmax<1 || nevent tolerance) { - element = j; - values += bankA.getFloat(name, i) + "/" + bankB.getFloat(name, i); - } - break; - } - if (element >= 0) { - System.out.println("mismatch at event " + runConfigBank.getInt("event", 0) - + " in row " + i - + " for variable " + name - + " with values " + values); - mismatch = true; - nbadentry++; - if (badEntries.containsKey(schema.getElementName(element))) { - int nbad = badEntries.get(schema.getElementName(element)) + 1; - badEntries.replace(schema.getElementName(element), nbad); - } else { - badEntries.put(schema.getElementName(element), 1); - } - } - } - if (mismatch) { - nbadrow++; - } - } + if (op.getOption("-b").stringValue() == null) { + for (Schema s : sf.getSchemaList()) { + banksA.add(new SortedBank(s)); + banksB.add(new SortedBank(s)); } - } - System.out.println("\n Analyzed " + nevent + " with " + nbadevent + " bad banks"); - System.out.println(nbadrow + "/" + nrow + " mismatched rows"); - System.out.println(nbadentry + "/" + nentry + " mismatched entry"); - for (String name : badEntries.keySet()) { - System.out.println(name + " " + badEntries.get(name)); + } else { + banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); } - if (nbadevent + nbadrow + nbadentry > 0) { - System.exit(7); - } + compare(readerA, readerB); } + } From 327699a9cf4a6909e2bc7ec5595981d954f6f3f1 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 12 Jan 2026 20:38:25 -0500 Subject: [PATCH 2/4] cleanup --- .../main/java/org/jlab/utils/HipoDiff.java | 156 +++++++++--------- 1 file changed, 77 insertions(+), 79 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index b5dbbe8e4e..cb17ad2b05 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; import org.jlab.jnp.hipo4.io.HipoReader; import org.jlab.jnp.hipo4.data.Bank; import org.jlab.jnp.hipo4.data.Event; @@ -12,40 +11,6 @@ public class HipoDiff { - /** - * Bank sortable by any integer columns. - * - */ - static class SortedBank extends Bank { - SortedBank(Schema s) { super(s); } - /** - * @param index the bank column indices to sort on - * @return the sorted row indices - */ - int[] getSorted(int... index) { - int[] rows = new int[getRows()]; - for (int row = 0; row < rows.length; row++) rows[row] = row; - // bubble sort: - for (int i = 0; i < rows.length - 1; i++) { - for (int j = 0; j < rows.length - i - 1; j++) { - for (int idx : index) { - if (idx >= this.getSchema().getElements()) break; - int x1 = getInt(idx, rows[j]); - int x2 = getInt(idx, rows[j + 1]); - if (x1 > x2) { - int tmp = rows[j]; - rows[j] = rows[j + 1]; - rows[j + 1] = tmp; - break; - } - else if (x1 < x2) break; - } - } - } - return rows; - } - } - static int nrow = 0; static int nevent = -1; static int nentry = 0; @@ -64,6 +29,53 @@ int[] getSorted(int... index) { static ArrayList banksB = new ArrayList<>(); static HashMap> badEntries = new HashMap<>(); + public static void main(String args[]) { + + OptionParser op = new OptionParser("hipo-diff"); + op.addOption("-t", "0.00001", "absolute tolerance for comparisons"); + op.addOption("-n", "-1", "number of events"); + op.addOption("-q", null, "quiet mode"); + op.addOption("-Q", null, "verbose mode"); + op.addOption("-b", null, "name of bank to diff"); + op.addOption("-s", null, "sort on column index"); + op.setRequiresInputList(true); + op.parse(args); + if (op.getInputList().size() != 2) { + System.out.println(op.getUsageString()); + System.out.println("ERROR: Exactly 2 input files are required."); + System.exit(1); + } + + if (op.getOption("-s").stringValue() != null) { + String[] stmp = op.getOption("-s").stringValue().split(","); + sortIndex = new int[stmp.length]; + for (int i = 0; i < stmp.length; i++) sortIndex[i] = Integer.parseInt(stmp[i]); + } + verboseMode = op.getOption("-Q").stringValue() != null; + quietMode = op.getOption("-q").stringValue() != null; + nmax = op.getOption("-n").intValue(); + tolerance = op.getOption("-t").doubleValue(); + + HipoReader readerA = new HipoReader(); + HipoReader readerB = new HipoReader(); + readerA.open(op.getInputList().get(0)); + readerB.open(op.getInputList().get(1)); + SchemaFactory sf = readerA.getSchemaFactory(); + runConfigBank = new Bank(sf.getSchema("RUN::config")); + + if (op.getOption("-b").stringValue() == null) { + for (Schema s : sf.getSchemaList()) { + banksA.add(new SortedBank(s)); + banksB.add(new SortedBank(s)); + } + } else { + banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + } + + compare(readerA, readerB); + } + public static void compare(HipoReader a, HipoReader b) { Event eventA = new Event(); Event eventB = new Event(); @@ -159,7 +171,7 @@ public static void compare(SortedBank a, SortedBank b) { if (!badEntries.containsKey(bankName)) { badEntries.put(bankName, new HashMap<>()); } - Map m = badEntries.get(bankName); + HashMap m = badEntries.get(bankName); if (!m.containsKey(elementName)) { m.put(elementName, 0); } @@ -172,51 +184,37 @@ public static void compare(SortedBank a, SortedBank b) { } } - public static void main(String args[]) { - - OptionParser op = new OptionParser("hipo-diff"); - op.addOption("-t", "0.00001", "absolute tolerance for comparisons"); - op.addOption("-n", "-1", "number of events"); - op.addOption("-q", null, "quiet mode"); - op.addOption("-Q", null, "verbose mode"); - op.addOption("-b", null, "name of bank to diff"); - op.addOption("-s", null, "sort on column index"); - op.setRequiresInputList(true); - op.parse(args); - if (op.getInputList().size() != 2) { - System.out.println(op.getUsageString()); - System.out.println("ERROR: Exactly 2 input files are required."); - System.exit(1); - } - - if (op.getOption("-s").stringValue() != null) { - String[] stmp = op.getOption("-s").stringValue().split(","); - sortIndex = new int[stmp.length]; - for (int i = 0; i < stmp.length; i++) sortIndex[i] = Integer.parseInt(stmp[i]); - } - verboseMode = op.getOption("-Q").stringValue() != null; - quietMode = op.getOption("-q").stringValue() != null; - nmax = op.getOption("-n").intValue(); - tolerance = op.getOption("-t").doubleValue(); - - HipoReader readerA = new HipoReader(); - HipoReader readerB = new HipoReader(); - readerA.open(op.getInputList().get(0)); - readerB.open(op.getInputList().get(1)); - SchemaFactory sf = readerA.getSchemaFactory(); - runConfigBank = new Bank(sf.getSchema("RUN::config")); - - if (op.getOption("-b").stringValue() == null) { - for (Schema s : sf.getSchemaList()) { - banksA.add(new SortedBank(s)); - banksB.add(new SortedBank(s)); + /** + * Bank sortable by any integer columns. + */ + static class SortedBank extends Bank { + SortedBank(Schema s) { super(s); } + /** + * @param index the bank column indices to sort on + * @return the sorted row indices + */ + int[] getSorted(int... index) { + int[] rows = new int[getRows()]; + for (int row = 0; row < rows.length; row++) rows[row] = row; + // bubble sort: + for (int i = 0; i < rows.length - 1; i++) { + for (int j = 0; j < rows.length - i - 1; j++) { + for (int idx : index) { + if (idx >= this.getSchema().getElements()) break; + int x1 = getInt(idx, rows[j]); + int x2 = getInt(idx, rows[j + 1]); + if (x1 > x2) { + int tmp = rows[j]; + rows[j] = rows[j + 1]; + rows[j + 1] = tmp; + break; + } + else if (x1 < x2) break; + } + } } - } else { - banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); - banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + return rows; } - - compare(readerA, readerB); } } From 35d7400c8b41c7c1fc2f53452c76d26c57b7e75a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 12 Jan 2026 20:50:19 -0500 Subject: [PATCH 3/4] cleanup --- .../clas-io/src/main/java/org/jlab/utils/HipoDiff.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index cb17ad2b05..7308e9fb96 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -30,7 +30,6 @@ public class HipoDiff { static HashMap> badEntries = new HashMap<>(); public static void main(String args[]) { - OptionParser op = new OptionParser("hipo-diff"); op.addOption("-t", "0.00001", "absolute tolerance for comparisons"); op.addOption("-n", "-1", "number of events"); @@ -45,7 +44,6 @@ public static void main(String args[]) { System.out.println("ERROR: Exactly 2 input files are required."); System.exit(1); } - if (op.getOption("-s").stringValue() != null) { String[] stmp = op.getOption("-s").stringValue().split(","); sortIndex = new int[stmp.length]; @@ -216,5 +214,4 @@ int[] getSorted(int... index) { return rows; } } - } From b9c1fe40f751a839780b35ee885c6560394359f9 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 12 Jan 2026 20:51:40 -0500 Subject: [PATCH 4/4] cleanup --- .../main/java/org/jlab/utils/HipoDiff.java | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 7308e9fb96..52f82c4e2e 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -11,6 +11,39 @@ public class HipoDiff { + /** + * Bank sortable by any integer columns. + */ + static class SortedBank extends Bank { + SortedBank(Schema s) { super(s); } + /** + * @param index the bank column indices to sort on + * @return the sorted row indices + */ + int[] getSorted(int... index) { + int[] rows = new int[getRows()]; + for (int row = 0; row < rows.length; row++) rows[row] = row; + // bubble sort: + for (int i = 0; i < rows.length - 1; i++) { + for (int j = 0; j < rows.length - i - 1; j++) { + for (int idx : index) { + if (idx >= this.getSchema().getElements()) break; + int x1 = getInt(idx, rows[j]); + int x2 = getInt(idx, rows[j + 1]); + if (x1 > x2) { + int tmp = rows[j]; + rows[j] = rows[j + 1]; + rows[j + 1] = tmp; + break; + } + else if (x1 < x2) break; + } + } + } + return rows; + } + } + static int nrow = 0; static int nevent = -1; static int nentry = 0; @@ -181,37 +214,4 @@ public static void compare(SortedBank a, SortedBank b) { } } } - - /** - * Bank sortable by any integer columns. - */ - static class SortedBank extends Bank { - SortedBank(Schema s) { super(s); } - /** - * @param index the bank column indices to sort on - * @return the sorted row indices - */ - int[] getSorted(int... index) { - int[] rows = new int[getRows()]; - for (int row = 0; row < rows.length; row++) rows[row] = row; - // bubble sort: - for (int i = 0; i < rows.length - 1; i++) { - for (int j = 0; j < rows.length - i - 1; j++) { - for (int idx : index) { - if (idx >= this.getSchema().getElements()) break; - int x1 = getInt(idx, rows[j]); - int x2 = getInt(idx, rows[j + 1]); - if (x1 > x2) { - int tmp = rows[j]; - rows[j] = rows[j + 1]; - rows[j + 1] = tmp; - break; - } - else if (x1 < x2) break; - } - } - } - return rows; - } - } }