diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 7809232d0..baa1c7fb7 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -577,6 +577,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, clean_increment=increment, accumulate=import_today, required_unit=required_unit, + can_modify_history=True, # history is not accessed after this point, so minute_data can freely modify it ) else: if history is None: @@ -639,6 +640,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling accumulate=load_minutes, required_unit=required_unit, interpolate=interpolate, + can_modify_history=True, # history is not accessed after this point, so minute_data can freely modify it ) else: if history is None: @@ -855,6 +857,7 @@ def fetch_sensor_data(self, save=True): divide_by=1.0, scale=1.0, required_unit="kWh", + can_modify_history=True, # soc_kwh_data is not accessed after this point, so minute_data can freely modify it ) # Fetch sensor data for cars, e.g. car plan, car energy, car sessions etc. diff --git a/apps/predbat/ha.py b/apps/predbat/ha.py index e418bbbc4..417d625bd 100644 --- a/apps/predbat/ha.py +++ b/apps/predbat/ha.py @@ -26,6 +26,7 @@ import traceback import threading import time +import copy from utils import str2time from const import TIME_FORMAT_HA, TIMEOUT, TIME_FORMAT_HA_TZ from component_base import ComponentBase @@ -100,6 +101,16 @@ def get_history(self, entity_id, days=30, tracked=True): self.update_entity(entity_id, history_data) result = [history_data] + if result is not None: + + # Returning result itself is not thread-safe, as it introduces a race condition -- it can be accessed + # during calls to update_entity, where it may e.g. appear to be empty during the sort(). + # Hence, return a copy, while holding the history_lock to ensure result isn't modified during the copy. + + with self.history_lock: + result = copy.deepcopy(result) + + return result def prune_history(self, now): diff --git a/apps/predbat/utils.py b/apps/predbat/utils.py index 409a4b109..6e7bed1f3 100644 --- a/apps/predbat/utils.py +++ b/apps/predbat/utils.py @@ -315,6 +315,7 @@ def minute_data( max_increment=MAX_INCREMENT, interpolate=False, debug=False, + can_modify_history=False, ): """ Turns data from HA into a hash of data indexed by minute with the data being the value @@ -335,7 +336,8 @@ def minute_data( if not history: return mdata, io_adjusted - history = copy.deepcopy(history) # Copy to avoid modifying original history + if not can_modify_history: + history = copy.deepcopy(history) # Copy to avoid modifying original history # Glitch filter, cleans glitches in the data and removes bad values, only for incrementing data if clean_increment and backwards: