Update get_history in HAHistory to take a copy of the history of an entity to avoid race conditions

apps/predbat/fetch.py

@@ -577,6 +577,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0,
                     clean_increment=increment,
                     accumulate=import_today,
                     required_unit=required_unit,
+                    can_modify_history=True, # history is not accessed after this point, so minute_data can freely modify it
                 )
             else:
                 if history is None:
@@ -639,6 +640,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling
                     accumulate=load_minutes,
                     required_unit=required_unit,
                     interpolate=interpolate,
+                    can_modify_history=True, # history is not accessed after this point, so minute_data can freely modify it
                 )
             else:
                 if history is None:
@@ -855,6 +857,7 @@ def fetch_sensor_data(self, save=True):
                 divide_by=1.0,
                 scale=1.0,
                 required_unit="kWh",
+                can_modify_history=True, # soc_kwh_data is not accessed after this point, so minute_data can freely modify it
             )
 
         # Fetch sensor data for cars, e.g. car plan, car energy, car sessions etc.

apps/predbat/ha.py

@@ -26,6 +26,7 @@
 import traceback
 import threading
 import time
+import copy
 from utils import str2time
 from const import TIME_FORMAT_HA, TIMEOUT, TIME_FORMAT_HA_TZ
 from component_base import ComponentBase
@@ -100,6 +101,16 @@ def get_history(self, entity_id, days=30, tracked=True):
                     self.update_entity(entity_id, history_data)
                 result = [history_data]
 
+        if result is not None:
+
+            # Returning result itself is not thread-safe, as it introduces a race condition -- it can be accessed
+            # during calls to update_entity, where it may e.g. appear to be empty during the sort().
+            # Hence, return a copy, while holding the history_lock to ensure result isn't modified during the copy.
+
+            with self.history_lock:
+                result = copy.deepcopy(result)
+
+
         return result
 
     def prune_history(self, now):

apps/predbat/utils.py

@@ -315,6 +315,7 @@ def minute_data(
     max_increment=MAX_INCREMENT,
     interpolate=False,
     debug=False,
+    can_modify_history=False,
 ):
     """
     Turns data from HA into a hash of data indexed by minute with the data being the value
@@ -335,7 +336,8 @@ def minute_data(
     if not history:
         return mdata, io_adjusted
 
-    history = copy.deepcopy(history)  # Copy to avoid modifying original history
+    if not can_modify_history:
+        history = copy.deepcopy(history)  # Copy to avoid modifying original history
 
     # Glitch filter, cleans glitches in the data and removes bad values, only for incrementing data
     if clean_increment and backwards: