From 0fd2f4689a54d5247835a57d402524273a2b8441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Sun, 24 May 2026 12:57:21 +0000 Subject: [PATCH 1/7] Update persistent types for peak and sensor numbers --- invisible_cities/evm/nh5.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/invisible_cities/evm/nh5.py b/invisible_cities/evm/nh5.py index 0006158651..d8d4bca387 100644 --- a/invisible_cities/evm/nh5.py +++ b/invisible_cities/evm/nh5.py @@ -90,7 +90,7 @@ class S12(tb.IsDescription): time and energy of the peak. """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number + peak = tb. UInt16Col(pos=2) # peak number time = tb.Float32Col(pos=3) # time in ns bwidth = tb.Float32Col(pos=4) # bin width in ns ene = tb.Float32Col(pos=5) # energy in pes @@ -104,7 +104,7 @@ class S12Pmt(tb.IsDescription): time and energy of the peak. """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number + peak = tb. UInt16Col(pos=2) # peak number npmt = tb. UInt8Col(pos=3) # pmt number (in order of IC db 26/8/2017: equal to SensorID) ene = tb.Float32Col(pos=5) # energy in pes @@ -117,8 +117,8 @@ class S2Si(tb.IsDescription): only energies are stored (times are defined in S2) """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number - nsipm = tb. Int16Col(pos=3) # sipm number + peak = tb. UInt16Col(pos=2) # peak number + nsipm = tb. UInt16Col(pos=3) # sipm number ene = tb.Float32Col(pos=5) # energy in pes From 4e5c4c9523cb58d94ace517c774b9175f3dce0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Sun, 24 May 2026 13:01:30 +0000 Subject: [PATCH 2/7] Add protection for integer overflow in storage --- invisible_cities/evm/event_model.py | 10 ++++++---- invisible_cities/io/pmaps_io.py | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/invisible_cities/evm/event_model.py b/invisible_cities/evm/event_model.py index 10011705e7..5d9439f0d7 100644 --- a/invisible_cities/evm/event_model.py +++ b/invisible_cities/evm/event_model.py @@ -290,10 +290,12 @@ def __init__(self, event_number, event_time, hits=None): def store(self, table): row = table.row + + overflow_protection = np.iinfo(np.uint16).max for hit in self.hits: row["event" ] = self.event row["time" ] = self.time - row["npeak" ] = hit .npeak + row["npeak" ] = min(hit.npeak, overflow_protection) row["Xpeak" ] = hit .Xpeak row["Ypeak" ] = hit .Ypeak row["X" ] = hit .X @@ -362,7 +364,7 @@ def fill_defaults(self): def store(self, table): row = table.row - dummy = np.iinfo(np.uint16).max + overflow_protection = np.iinfo(np.uint16).max s1_peaks = range(int(self.nS1)) if self.nS1 else [0] s2_peaks = range(int(self.nS2)) if self.nS2 else [0] self.fill_defaults() @@ -371,8 +373,8 @@ def store(self, table): for j in s2_peaks: row["event" ] = self.event row["time" ] = self.time - row["s1_peak"] = i if self.nS1 else dummy - row["s2_peak"] = j if self.nS2 else dummy + row["s1_peak"] = min(i, overflow_protection) if self.nS1 else overflow_protection + row["s2_peak"] = min(j, overflow_protection) if self.nS2 else overflow_protection row["nS1" ] = self.nS1 row["nS2" ] = self.nS2 diff --git a/invisible_cities/io/pmaps_io.py b/invisible_cities/io/pmaps_io.py index 5a5ff9a166..21ca5b10b0 100644 --- a/invisible_cities/io/pmaps_io.py +++ b/invisible_cities/io/pmaps_io.py @@ -20,9 +20,10 @@ def store_peak(pmt_table, pmti_table, si_table, pmt_row = pmt_table.row pmti_row = pmti_table.row + overflow_protection = np.iinfo(np.uint16).max for i, t in enumerate(peak.times): pmt_row['event' ] = event_number - pmt_row['peak' ] = peak_number + pmt_row['peak' ] = min(peak_number, overflow_protection) pmt_row['time' ] = t pmt_row['bwidth'] = peak.bin_widths[i] pmt_row['ene' ] = peak.pmts.sum_over_sensors[i] @@ -31,7 +32,7 @@ def store_peak(pmt_table, pmti_table, si_table, for pmt_id in peak.pmts.ids: for e in peak.pmts.waveform(pmt_id): pmti_row['event'] = event_number - pmti_row['peak' ] = peak_number + pmti_row['peak' ] = min(peak_number, overflow_protection) pmti_row['npmt' ] = pmt_id pmti_row['ene' ] = e pmti_row.append() @@ -42,7 +43,7 @@ def store_peak(pmt_table, pmti_table, si_table, for sipm_id in peak.sipms.ids: for q in peak.sipms.waveform(sipm_id): si_row['event'] = event_number - si_row['peak' ] = peak_number + si_row['peak' ] = min(peak_number, overflow_protection) si_row['nsipm'] = sipm_id si_row['ene' ] = q si_row.append() From eb81f9ba0194436474e64de274be2b46c0e09d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Sun, 24 May 2026 13:16:56 +0000 Subject: [PATCH 3/7] Update test files --- .../Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 | 4 ++-- .../Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 | 4 ++-- .../test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 index c0fd9aabb1..cbe4771e0f 100644 --- a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8242917d8c5c53658f278b74e9f7c27a19e6c98d85b4cf0c56bb4a9072588bff -size 121698 +oid sha256:5fbb2acfdf5c0f05e420dbf2929f6fee646d15eb6b85f8561edf8cf65d4e9bac +size 189182 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 index e643b0a5c7..80f9393fa3 100644 --- a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e884d9db77e821f5175762df2cb9c378de74404e13aa860a339d5edb44044fa3 -size 129484 +oid sha256:660e20774878c834ed080c9075c38bcf9675d7cf74be5f524ffc48a8df0cfffe +size 120025 diff --git a/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 b/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 index 8aed94f464..cba9e2a5a8 100644 --- a/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 +++ b/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4bb3cbcb78833133174e30a93a60215dfe8143451f98ebc1364ba946686b796 -size 87877 +oid sha256:f94bb55a59ed898d6928bfdb54383b5297936e68b63a181ce8852b82cfd0d279 +size 129035 From 3a0e1b8a8a691e86b854f4e231809868c75ca83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Sun, 24 May 2026 13:17:24 +0000 Subject: [PATCH 4/7] Update fixture --- invisible_cities/io/pmaps_io_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/invisible_cities/io/pmaps_io_test.py b/invisible_cities/io/pmaps_io_test.py index 2d40a2120e..56d34b8b09 100644 --- a/invisible_cities/io/pmaps_io_test.py +++ b/invisible_cities/io/pmaps_io_test.py @@ -83,28 +83,28 @@ def two_pmaps_dfs(two_pmaps_evm): s1 = pd.DataFrame(dict( event = s1_data.evt_numbers , time = s1_data.times - , peak = s1_data.peak_numbers.astype(np.uint8) + , peak = s1_data.peak_numbers.astype(np.uint16) , bwidth = s1_data.bwidths , ene = s1_data.enes )) s2 = pd.DataFrame(dict( event = s2_data.evt_numbers , time = s2_data.times - , peak = s2_data.peak_numbers.astype(np.uint8) + , peak = s2_data.peak_numbers.astype(np.uint16) , bwidth = s2_data.bwidths , ene = s2_data.enes )) si = pd.DataFrame(dict( event = s2_data.evt_numbers_sipm - , peak = s2_data.peak_numbers_sipm.astype(np.uint8) - , nsipm = s2_data.nsipms.astype(np.int16) + , peak = s2_data.peak_numbers_sipm.astype(np.uint16) + , nsipm = s2_data.nsipms.astype(np.uint16) , ene = s2_data.enes_sipm )) s2pmt = pd.DataFrame(dict( event = s2_data.evt_numbers_pmt - , peak = s2_data.peak_numbers_pmt.astype(np.uint8) + , peak = s2_data.peak_numbers_pmt.astype(np.uint16) , npmt = s2_data.npmts.astype(np.uint8) , ene = s2_data.enes_pmt )) s1pmt = pd.DataFrame(dict( event = s1_data.evt_numbers_pmt - , peak = s1_data.peak_numbers_pmt.astype(np.uint8) + , peak = s1_data.peak_numbers_pmt.astype(np.uint16) , npmt = s1_data.npmts.astype(np.uint8) , ene = s1_data.enes_pmt )) From 6d6e40f29f5f7d1ee4ee7733d5fab568a305ddda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Mon, 8 Jun 2026 10:46:27 +0000 Subject: [PATCH 5/7] Add `overflow_protection` helper --- invisible_cities/core/core_functions.py | 12 ++++++++++++ invisible_cities/core/core_functions_test.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/invisible_cities/core/core_functions.py b/invisible_cities/core/core_functions.py index 7265748147..d2ace560a4 100644 --- a/invisible_cities/core/core_functions.py +++ b/invisible_cities/core/core_functions.py @@ -4,11 +4,13 @@ """ import time from contextlib import contextmanager +from warnings import warn import numpy as np from typing import Sequence from typing import Tuple +from typing import Union from ..types.symbols import NormMode @@ -356,3 +358,13 @@ def fix_random_seed(seed): yield finally: np.random.set_state(state) + + +def overflow_protection( value: Union[int, float] + , upper: Union[int, float] + , origin: str + ): + if value > upper: + warn(f"Overflow detected at {origin}, clipping the value to {upper}", UserWarning) + return upper + return value diff --git a/invisible_cities/core/core_functions_test.py b/invisible_cities/core/core_functions_test.py index 129cd935f5..6aad71608b 100644 --- a/invisible_cities/core/core_functions_test.py +++ b/invisible_cities/core/core_functions_test.py @@ -9,6 +9,7 @@ from pytest import approx from pytest import mark from pytest import raises +from pytest import warns from flaky import flaky from hypothesis import given @@ -412,3 +413,22 @@ def test_fix_random_seed_resets(): assert not np.isclose(value1, value2) assert np.isclose(value1, value3) assert not np.isclose(value2, value3) + + +@mark.parametrize(" value upper".split(), + (( 5, 10), + ( 10, 10), + (2.5, 3.0))) +def test_overflow_protection_keeps_values_not_above_limit(value, upper): + assert core.overflow_protection(value, upper, "test") == value + + +def test_overflow_protection_clips_and_warns(): + upper = 10 + origin = "test table" + match = f"Overflow detected at {origin}, clipping the value to {upper}" + + with warns(UserWarning, match=match): + got = core.overflow_protection(11, upper, origin) + + assert got == upper From 6fe04122da67a53c0856626925147e3fe6604c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Mon, 8 Jun 2026 10:55:40 +0000 Subject: [PATCH 6/7] Warn about integer overflows in PMap storage --- invisible_cities/io/pmaps_io.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/invisible_cities/io/pmaps_io.py b/invisible_cities/io/pmaps_io.py index 21ca5b10b0..d7c475c25c 100644 --- a/invisible_cities/io/pmaps_io.py +++ b/invisible_cities/io/pmaps_io.py @@ -12,6 +12,7 @@ from .. evm .pmaps import S2 from .. evm .pmaps import PMap from .. evm import nh5 as table_formats +from .. core.core_functions import overflow_protection from .. core.tbl_functions import filters as tbl_filters @@ -20,10 +21,10 @@ def store_peak(pmt_table, pmti_table, si_table, pmt_row = pmt_table.row pmti_row = pmti_table.row - overflow_protection = np.iinfo(np.uint16).max + u16max = np.iinfo(np.uint16).max for i, t in enumerate(peak.times): pmt_row['event' ] = event_number - pmt_row['peak' ] = min(peak_number, overflow_protection) + pmt_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") pmt_row['time' ] = t pmt_row['bwidth'] = peak.bin_widths[i] pmt_row['ene' ] = peak.pmts.sum_over_sensors[i] @@ -32,7 +33,7 @@ def store_peak(pmt_table, pmti_table, si_table, for pmt_id in peak.pmts.ids: for e in peak.pmts.waveform(pmt_id): pmti_row['event'] = event_number - pmti_row['peak' ] = min(peak_number, overflow_protection) + pmti_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") pmti_row['npmt' ] = pmt_id pmti_row['ene' ] = e pmti_row.append() @@ -43,7 +44,7 @@ def store_peak(pmt_table, pmti_table, si_table, for sipm_id in peak.sipms.ids: for q in peak.sipms.waveform(sipm_id): si_row['event'] = event_number - si_row['peak' ] = min(peak_number, overflow_protection) + si_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") si_row['nsipm'] = sipm_id si_row['ene' ] = q si_row.append() From cdb5292978d26435138526ac5c82290596bc0200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Mart=C3=ADnez=20Lema?= Date: Mon, 8 Jun 2026 10:56:21 +0000 Subject: [PATCH 7/7] Warn about integer overflows in Event Model --- invisible_cities/evm/event_model.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/invisible_cities/evm/event_model.py b/invisible_cities/evm/event_model.py index 5d9439f0d7..67297d4c64 100644 --- a/invisible_cities/evm/event_model.py +++ b/invisible_cities/evm/event_model.py @@ -8,6 +8,7 @@ from .. types.ic_types import xy from .. types.symbols import HitEnergy from .. core import system_of_units as units +from .. core.core_functions import overflow_protection from typing import List from typing import Tuple @@ -291,11 +292,11 @@ def __init__(self, event_number, event_time, hits=None): def store(self, table): row = table.row - overflow_protection = np.iinfo(np.uint16).max + u16max = np.iinfo(np.uint16).max for hit in self.hits: row["event" ] = self.event row["time" ] = self.time - row["npeak" ] = min(hit.npeak, overflow_protection) + row["npeak" ] = overflow_protection(hit.npeak, u16max, "HitCollection::store (npeak)") row["Xpeak" ] = hit .Xpeak row["Ypeak" ] = hit .Ypeak row["X" ] = hit .X @@ -364,7 +365,7 @@ def fill_defaults(self): def store(self, table): row = table.row - overflow_protection = np.iinfo(np.uint16).max + u16max = np.iinfo(np.uint16).max s1_peaks = range(int(self.nS1)) if self.nS1 else [0] s2_peaks = range(int(self.nS2)) if self.nS2 else [0] self.fill_defaults() @@ -373,8 +374,8 @@ def store(self, table): for j in s2_peaks: row["event" ] = self.event row["time" ] = self.time - row["s1_peak"] = min(i, overflow_protection) if self.nS1 else overflow_protection - row["s2_peak"] = min(j, overflow_protection) if self.nS2 else overflow_protection + row["s1_peak"] = overflow_protection(i, u16max, "KrEvent::store (s1_peak)") if self.nS1 else u16max + row["s2_peak"] = overflow_protection(j, u16max, "KrEvent::store (s2_peak)") if self.nS2 else u16max row["nS1" ] = self.nS1 row["nS2" ] = self.nS2