diff --git a/invisible_cities/core/core_functions.py b/invisible_cities/core/core_functions.py index 726574814..d2ace560a 100644 --- a/invisible_cities/core/core_functions.py +++ b/invisible_cities/core/core_functions.py @@ -4,11 +4,13 @@ """ import time from contextlib import contextmanager +from warnings import warn import numpy as np from typing import Sequence from typing import Tuple +from typing import Union from ..types.symbols import NormMode @@ -356,3 +358,13 @@ def fix_random_seed(seed): yield finally: np.random.set_state(state) + + +def overflow_protection( value: Union[int, float] + , upper: Union[int, float] + , origin: str + ): + if value > upper: + warn(f"Overflow detected at {origin}, clipping the value to {upper}", UserWarning) + return upper + return value diff --git a/invisible_cities/core/core_functions_test.py b/invisible_cities/core/core_functions_test.py index 129cd935f..6aad71608 100644 --- a/invisible_cities/core/core_functions_test.py +++ b/invisible_cities/core/core_functions_test.py @@ -9,6 +9,7 @@ from pytest import approx from pytest import mark from pytest import raises +from pytest import warns from flaky import flaky from hypothesis import given @@ -412,3 +413,22 @@ def test_fix_random_seed_resets(): assert not np.isclose(value1, value2) assert np.isclose(value1, value3) assert not np.isclose(value2, value3) + + +@mark.parametrize(" value upper".split(), + (( 5, 10), + ( 10, 10), + (2.5, 3.0))) +def test_overflow_protection_keeps_values_not_above_limit(value, upper): + assert core.overflow_protection(value, upper, "test") == value + + +def test_overflow_protection_clips_and_warns(): + upper = 10 + origin = "test table" + match = f"Overflow detected at {origin}, clipping the value to {upper}" + + with warns(UserWarning, match=match): + got = core.overflow_protection(11, upper, origin) + + assert got == upper diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 index c0fd9aabb..cbe4771e0 100644 --- a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.NEWMC.PMP.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8242917d8c5c53658f278b74e9f7c27a19e6c98d85b4cf0c56bb4a9072588bff -size 121698 +oid sha256:5fbb2acfdf5c0f05e420dbf2929f6fee646d15eb6b85f8561edf8cf65d4e9bac +size 189182 diff --git a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 index e643b0a5c..80f9393fa 100644 --- a/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 +++ b/invisible_cities/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_3evts.hypathia.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e884d9db77e821f5175762df2cb9c378de74404e13aa860a339d5edb44044fa3 -size 129484 +oid sha256:660e20774878c834ed080c9075c38bcf9675d7cf74be5f524ffc48a8df0cfffe +size 120025 diff --git a/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 b/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 index 8aed94f46..cba9e2a5a 100644 --- a/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 +++ b/invisible_cities/database/test_data/run_2983_pmaps_2evts_sfreq_5ns_200ns.h5 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4bb3cbcb78833133174e30a93a60215dfe8143451f98ebc1364ba946686b796 -size 87877 +oid sha256:f94bb55a59ed898d6928bfdb54383b5297936e68b63a181ce8852b82cfd0d279 +size 129035 diff --git a/invisible_cities/evm/event_model.py b/invisible_cities/evm/event_model.py index 10011705e..67297d4c6 100644 --- a/invisible_cities/evm/event_model.py +++ b/invisible_cities/evm/event_model.py @@ -8,6 +8,7 @@ from .. types.ic_types import xy from .. types.symbols import HitEnergy from .. core import system_of_units as units +from .. core.core_functions import overflow_protection from typing import List from typing import Tuple @@ -290,10 +291,12 @@ def __init__(self, event_number, event_time, hits=None): def store(self, table): row = table.row + + u16max = np.iinfo(np.uint16).max for hit in self.hits: row["event" ] = self.event row["time" ] = self.time - row["npeak" ] = hit .npeak + row["npeak" ] = overflow_protection(hit.npeak, u16max, "HitCollection::store (npeak)") row["Xpeak" ] = hit .Xpeak row["Ypeak" ] = hit .Ypeak row["X" ] = hit .X @@ -362,7 +365,7 @@ def fill_defaults(self): def store(self, table): row = table.row - dummy = np.iinfo(np.uint16).max + u16max = np.iinfo(np.uint16).max s1_peaks = range(int(self.nS1)) if self.nS1 else [0] s2_peaks = range(int(self.nS2)) if self.nS2 else [0] self.fill_defaults() @@ -371,8 +374,8 @@ def store(self, table): for j in s2_peaks: row["event" ] = self.event row["time" ] = self.time - row["s1_peak"] = i if self.nS1 else dummy - row["s2_peak"] = j if self.nS2 else dummy + row["s1_peak"] = overflow_protection(i, u16max, "KrEvent::store (s1_peak)") if self.nS1 else u16max + row["s2_peak"] = overflow_protection(j, u16max, "KrEvent::store (s2_peak)") if self.nS2 else u16max row["nS1" ] = self.nS1 row["nS2" ] = self.nS2 diff --git a/invisible_cities/evm/nh5.py b/invisible_cities/evm/nh5.py index 000615865..d8d4bca38 100644 --- a/invisible_cities/evm/nh5.py +++ b/invisible_cities/evm/nh5.py @@ -90,7 +90,7 @@ class S12(tb.IsDescription): time and energy of the peak. """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number + peak = tb. UInt16Col(pos=2) # peak number time = tb.Float32Col(pos=3) # time in ns bwidth = tb.Float32Col(pos=4) # bin width in ns ene = tb.Float32Col(pos=5) # energy in pes @@ -104,7 +104,7 @@ class S12Pmt(tb.IsDescription): time and energy of the peak. """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number + peak = tb. UInt16Col(pos=2) # peak number npmt = tb. UInt8Col(pos=3) # pmt number (in order of IC db 26/8/2017: equal to SensorID) ene = tb.Float32Col(pos=5) # energy in pes @@ -117,8 +117,8 @@ class S2Si(tb.IsDescription): only energies are stored (times are defined in S2) """ event = tb. Int64Col(pos=0) - peak = tb. UInt8Col(pos=2) # peak number - nsipm = tb. Int16Col(pos=3) # sipm number + peak = tb. UInt16Col(pos=2) # peak number + nsipm = tb. UInt16Col(pos=3) # sipm number ene = tb.Float32Col(pos=5) # energy in pes diff --git a/invisible_cities/io/pmaps_io.py b/invisible_cities/io/pmaps_io.py index 5a5ff9a16..d7c475c25 100644 --- a/invisible_cities/io/pmaps_io.py +++ b/invisible_cities/io/pmaps_io.py @@ -12,6 +12,7 @@ from .. evm .pmaps import S2 from .. evm .pmaps import PMap from .. evm import nh5 as table_formats +from .. core.core_functions import overflow_protection from .. core.tbl_functions import filters as tbl_filters @@ -20,9 +21,10 @@ def store_peak(pmt_table, pmti_table, si_table, pmt_row = pmt_table.row pmti_row = pmti_table.row + u16max = np.iinfo(np.uint16).max for i, t in enumerate(peak.times): pmt_row['event' ] = event_number - pmt_row['peak' ] = peak_number + pmt_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") pmt_row['time' ] = t pmt_row['bwidth'] = peak.bin_widths[i] pmt_row['ene' ] = peak.pmts.sum_over_sensors[i] @@ -31,7 +33,7 @@ def store_peak(pmt_table, pmti_table, si_table, for pmt_id in peak.pmts.ids: for e in peak.pmts.waveform(pmt_id): pmti_row['event'] = event_number - pmti_row['peak' ] = peak_number + pmti_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") pmti_row['npmt' ] = pmt_id pmti_row['ene' ] = e pmti_row.append() @@ -42,7 +44,7 @@ def store_peak(pmt_table, pmti_table, si_table, for sipm_id in peak.sipms.ids: for q in peak.sipms.waveform(sipm_id): si_row['event'] = event_number - si_row['peak' ] = peak_number + si_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)") si_row['nsipm'] = sipm_id si_row['ene' ] = q si_row.append() diff --git a/invisible_cities/io/pmaps_io_test.py b/invisible_cities/io/pmaps_io_test.py index 2d40a2120..56d34b8b0 100644 --- a/invisible_cities/io/pmaps_io_test.py +++ b/invisible_cities/io/pmaps_io_test.py @@ -83,28 +83,28 @@ def two_pmaps_dfs(two_pmaps_evm): s1 = pd.DataFrame(dict( event = s1_data.evt_numbers , time = s1_data.times - , peak = s1_data.peak_numbers.astype(np.uint8) + , peak = s1_data.peak_numbers.astype(np.uint16) , bwidth = s1_data.bwidths , ene = s1_data.enes )) s2 = pd.DataFrame(dict( event = s2_data.evt_numbers , time = s2_data.times - , peak = s2_data.peak_numbers.astype(np.uint8) + , peak = s2_data.peak_numbers.astype(np.uint16) , bwidth = s2_data.bwidths , ene = s2_data.enes )) si = pd.DataFrame(dict( event = s2_data.evt_numbers_sipm - , peak = s2_data.peak_numbers_sipm.astype(np.uint8) - , nsipm = s2_data.nsipms.astype(np.int16) + , peak = s2_data.peak_numbers_sipm.astype(np.uint16) + , nsipm = s2_data.nsipms.astype(np.uint16) , ene = s2_data.enes_sipm )) s2pmt = pd.DataFrame(dict( event = s2_data.evt_numbers_pmt - , peak = s2_data.peak_numbers_pmt.astype(np.uint8) + , peak = s2_data.peak_numbers_pmt.astype(np.uint16) , npmt = s2_data.npmts.astype(np.uint8) , ene = s2_data.enes_pmt )) s1pmt = pd.DataFrame(dict( event = s1_data.evt_numbers_pmt - , peak = s1_data.peak_numbers_pmt.astype(np.uint8) + , peak = s1_data.peak_numbers_pmt.astype(np.uint16) , npmt = s1_data.npmts.astype(np.uint8) , ene = s1_data.enes_pmt ))