Update size of some persistent data types

invisible_cities/core/core_functions.py

@@ -4,11 +4,13 @@
 """
 import time
 from contextlib import contextmanager
+from warnings    import warn
 
 import numpy as np
 
 from typing import Sequence
 from typing import Tuple
+from typing import Union
 
 from ..types.symbols import NormMode
 
@@ -356,3 +358,13 @@ def fix_random_seed(seed):
         yield
     finally:
         np.random.set_state(state)
+
+
+def overflow_protection( value: Union[int, float]
+                       , upper: Union[int, float]
+                       , origin: str
+                       ):
+    if value > upper:
+        warn(f"Overflow detected at {origin}, clipping the value to {upper}", UserWarning)
+        return upper
+    return value

invisible_cities/core/core_functions_test.py

@@ -9,6 +9,7 @@
 from pytest import approx
 from pytest import mark
 from pytest import raises
+from pytest import warns
 
 from flaky                  import flaky
 from hypothesis             import given
@@ -412,3 +413,22 @@ def test_fix_random_seed_resets():
     assert not np.isclose(value1, value2)
     assert     np.isclose(value1, value3)
     assert not np.isclose(value2, value3)
+
+
+@mark.parametrize(" value upper".split(),
+                  ((  5,  10),
+                   ( 10,  10),
+                   (2.5, 3.0)))
+def test_overflow_protection_keeps_values_not_above_limit(value, upper):
+    assert core.overflow_protection(value, upper, "test") == value
+
+
+def test_overflow_protection_clips_and_warns():
+    upper  = 10
+    origin = "test table"
+    match  = f"Overflow detected at {origin}, clipping the value to {upper}"
+
+    with warns(UserWarning, match=match):
+        got = core.overflow_protection(11, upper, origin)
+
+    assert got == upper

invisible_cities/evm/event_model.py

@@ -8,6 +8,7 @@
 from .. types.ic_types import xy
 from .. types.symbols  import HitEnergy
 from .. core           import system_of_units as units
+from .. core.core_functions import overflow_protection
 
 from typing import List
 from typing import Tuple
@@ -290,10 +291,12 @@ def __init__(self, event_number, event_time, hits=None):
 
     def store(self, table):
         row = table.row
+
+        u16max = np.iinfo(np.uint16).max
         for hit in self.hits:
             row["event"   ] = self.event
             row["time"    ] = self.time
-            row["npeak"   ] = hit .npeak
+            row["npeak"   ] = overflow_protection(hit.npeak, u16max, "HitCollection::store (npeak)")
             row["Xpeak"   ] = hit .Xpeak
             row["Ypeak"   ] = hit .Ypeak
             row["X"       ] = hit .X
@@ -362,7 +365,7 @@ def fill_defaults(self):
     def store(self, table):
         row = table.row
 
-        dummy    = np.iinfo(np.uint16).max
+        u16max = np.iinfo(np.uint16).max
         s1_peaks = range(int(self.nS1)) if self.nS1 else [0]
         s2_peaks = range(int(self.nS2)) if self.nS2 else [0]
         self.fill_defaults()
@@ -371,8 +374,8 @@ def store(self, table):
             for j in s2_peaks:
                 row["event"  ] = self.event
                 row["time"   ] = self.time
-                row["s1_peak"] = i if self.nS1 else dummy
-                row["s2_peak"] = j if self.nS2 else dummy
+                row["s1_peak"] = overflow_protection(i, u16max, "KrEvent::store (s1_peak)") if self.nS1 else u16max
+                row["s2_peak"] = overflow_protection(j, u16max, "KrEvent::store (s2_peak)") if self.nS2 else u16max
                 row["nS1"    ] = self.nS1
                 row["nS2"    ] = self.nS2
 

invisible_cities/evm/nh5.py

@@ -90,7 +90,7 @@ class S12(tb.IsDescription):
     time and energy of the peak.
     """
     event  = tb.  Int64Col(pos=0)
-    peak   = tb.  UInt8Col(pos=2) # peak number
+    peak   = tb. UInt16Col(pos=2) # peak number
     time   = tb.Float32Col(pos=3) # time in ns
     bwidth = tb.Float32Col(pos=4) # bin width in ns
     ene    = tb.Float32Col(pos=5) # energy in pes
@@ -104,7 +104,7 @@ class S12Pmt(tb.IsDescription):
     time and energy of the peak.
     """
     event  = tb.  Int64Col(pos=0)
-    peak   = tb.  UInt8Col(pos=2) # peak number
+    peak   = tb. UInt16Col(pos=2) # peak number
     npmt   = tb.  UInt8Col(pos=3) # pmt number (in order of IC db 26/8/2017: equal to SensorID)
     ene    = tb.Float32Col(pos=5) # energy in pes
 
@@ -117,8 +117,8 @@ class S2Si(tb.IsDescription):
     only energies are stored (times are defined in S2)
     """
     event = tb.  Int64Col(pos=0)
-    peak  = tb.  UInt8Col(pos=2) # peak number
-    nsipm = tb.  Int16Col(pos=3) # sipm number
+    peak  = tb. UInt16Col(pos=2) # peak number
+    nsipm = tb. UInt16Col(pos=3) # sipm number
     ene   = tb.Float32Col(pos=5) # energy in pes
 
 

invisible_cities/io/pmaps_io.py

@@ -12,6 +12,7 @@
 from .. evm .pmaps         import S2
 from .. evm .pmaps         import PMap
 from .. evm                import nh5     as table_formats
+from .. core.core_functions import overflow_protection
 from .. core.tbl_functions import filters as tbl_filters
 
 
@@ -20,9 +21,10 @@ def store_peak(pmt_table, pmti_table, si_table,
     pmt_row  =  pmt_table.row
     pmti_row = pmti_table.row
 
+    u16max = np.iinfo(np.uint16).max
     for i, t in enumerate(peak.times):
         pmt_row['event' ] = event_number
-        pmt_row['peak'  ] = peak_number
+        pmt_row['peak'  ] = overflow_protection(peak_number, u16max, "store_peak (peak)")
         pmt_row['time'  ] = t
         pmt_row['bwidth'] = peak.bin_widths[i]
         pmt_row['ene'   ] = peak.pmts.sum_over_sensors[i]
@@ -31,7 +33,7 @@ def store_peak(pmt_table, pmti_table, si_table,
     for pmt_id in peak.pmts.ids:
         for e in peak.pmts.waveform(pmt_id):
             pmti_row['event'] = event_number
-            pmti_row['peak' ] =  peak_number
+            pmti_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)")
             pmti_row['npmt' ] = pmt_id
             pmti_row['ene'  ] = e
             pmti_row.append()
@@ -42,7 +44,7 @@ def store_peak(pmt_table, pmti_table, si_table,
     for sipm_id in peak.sipms.ids:
         for q in peak.sipms.waveform(sipm_id):
             si_row['event'] = event_number
-            si_row['peak' ] =  peak_number
+            si_row['peak' ] = overflow_protection(peak_number, u16max, "store_peak (peak)")
             si_row['nsipm'] = sipm_id
             si_row['ene'  ] = q
             si_row.append()

invisible_cities/io/pmaps_io_test.py

@@ -83,28 +83,28 @@ def two_pmaps_dfs(two_pmaps_evm):
 
     s1 = pd.DataFrame(dict( event  = s1_data.evt_numbers
                           , time   = s1_data.times
-                          , peak   = s1_data.peak_numbers.astype(np.uint8)
+                          , peak   = s1_data.peak_numbers.astype(np.uint16)
                           , bwidth = s1_data.bwidths
                           , ene    = s1_data.enes
                           ))
     s2 = pd.DataFrame(dict( event  = s2_data.evt_numbers
                           , time   = s2_data.times
-                          , peak   = s2_data.peak_numbers.astype(np.uint8)
+                          , peak   = s2_data.peak_numbers.astype(np.uint16)
                           , bwidth = s2_data.bwidths
                           , ene    = s2_data.enes
                           ))
     si = pd.DataFrame(dict( event  = s2_data.evt_numbers_sipm
-                          , peak   = s2_data.peak_numbers_sipm.astype(np.uint8)
-                          , nsipm  = s2_data.nsipms.astype(np.int16)
+                          , peak   = s2_data.peak_numbers_sipm.astype(np.uint16)
+                          , nsipm  = s2_data.nsipms.astype(np.uint16)
                           , ene    = s2_data.enes_sipm
                           ))
     s2pmt = pd.DataFrame(dict( event = s2_data.evt_numbers_pmt
-                             , peak  = s2_data.peak_numbers_pmt.astype(np.uint8)
+                             , peak  = s2_data.peak_numbers_pmt.astype(np.uint16)
                              , npmt  = s2_data.npmts.astype(np.uint8)
                              , ene   = s2_data.enes_pmt
                              ))
     s1pmt = pd.DataFrame(dict( event = s1_data.evt_numbers_pmt
-                             , peak  = s1_data.peak_numbers_pmt.astype(np.uint8)
+                             , peak  = s1_data.peak_numbers_pmt.astype(np.uint16)
                              , npmt  = s1_data.npmts.astype(np.uint8)
                              , ene   = s1_data.enes_pmt
                              ))