From 1d896dc4e750cc6819f87b13aeecc1c0c31856f5 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Wed, 27 May 2026 20:21:29 +0200 Subject: [PATCH 01/14] Prevent conditional autoencoder feature being dropped during preprocessing by the `ColumnSelector` or `LowUniqueValueFilter` by adding a protected_features parameter to the fit method. --- .../data_preprocessing/column_selector.py | 33 ++++++- .../data_preprocessing/data_preprocessor.py | 4 +- .../low_unique_value_filter.py | 35 ++++++- energy_fault_detector/fault_detector.py | 9 +- .../test_data_preprocessor.py | 95 +++++++++++++++++++ tests/test_fault_detector.py | 42 ++++++++ 6 files changed, 210 insertions(+), 8 deletions(-) diff --git a/energy_fault_detector/data_preprocessing/column_selector.py b/energy_fault_detector/data_preprocessing/column_selector.py index 3fbf2de..ba70a8a 100644 --- a/energy_fault_detector/data_preprocessing/column_selector.py +++ b/energy_fault_detector/data_preprocessing/column_selector.py @@ -1,4 +1,5 @@ from typing import Optional, List +import logging import numpy as np import pandas as pd @@ -6,6 +7,8 @@ from energy_fault_detector.core.data_transformer import DataTransformer +logger = logging.getLogger('energy_fault_detector') + class ColumnSelector(DataTransformer): """Class for selecting columns, using the provided list of features to exclude/drop and the fraction of NaNs. @@ -40,33 +43,57 @@ def __init__( # pylint: disable=attribute-defined-outside-init # noinspection PyAttributeOutsideInit - def fit(self, x: pd.DataFrame, y: Optional[np.array] = None) -> 'ColumnSelector': + def fit(self, x: pd.DataFrame, y: Optional[np.array] = None, + protected_features: Optional[List[str]] = None) -> 'ColumnSelector': """Find columns to keep for training Args: x: data to filter based on NaN fractions y: target variable, currently unused. + protected_features: list of feature names that should never be dropped (e.g., conditional features for + autoencoders). Warnings will be issued if these features would have been dropped otherwise. """ self.feature_names_in_ = x.columns.to_list() self.n_features_in_ = len(x.columns) + protected_features = protected_features or [] + protected_lower = [f.lower() for f in protected_features] + # If features_to_select is provided - ignore upper/lower case if self.features_to_select is not None: select_lower = [f.lower() for f in self.features_to_select] keep_cols = [col for col in x.columns if col.lower() in select_lower] x_transformed = x[keep_cols] else: - # drop features to exclude - ignore upper/lower case + # drop features to exclude - ignore upper/lower case, but protect protected_features to_drop = [col for col in x.columns if col.lower() in [excluded_feature.lower() for excluded_feature in self.features_to_exclude] ] + + # Warn about protected features in exclusion list + protected_in_exclusion = [col for col in to_drop if col.lower() in protected_lower] + for col in protected_in_exclusion: + logger.warning(f"Feature '{col}' is in features_to_exclude but is used as a conditional feature. " + f"Keeping it anyway.") + to_drop.remove(col) + x_transformed = x.drop(to_drop, axis=1, errors='ignore') # drop columns which have more than max_nan_frac_per_col relative NaN frequency empty_percentage = x_transformed.isnull().mean(axis=0) empty_cols = empty_percentage[empty_percentage >= self.max_nan_frac_per_col].index - x_transformed = x_transformed.drop(empty_cols, axis=1) + + # Protect features from NaN-based dropping + protected_empty_cols = [col for col in empty_cols if col.lower() in protected_lower] + for col in protected_empty_cols: + nan_frac = empty_percentage[col] + logger.warning(f"Feature '{col}' has {nan_frac*100:.1f}% NaN values (exceeds threshold of " + f"{self.max_nan_frac_per_col*100:.1f}%), but is used as a conditional feature. " + f"Keeping it anyway.") + + empty_cols_to_drop = [col for col in empty_cols if col.lower() not in protected_lower] + x_transformed = x_transformed.drop(empty_cols_to_drop, axis=1) # select relevant numeric columns and set attribute for transform self.feature_names_out_ = x_transformed.columns.to_list() diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py index 22b2b53..990ddf7 100644 --- a/energy_fault_detector/data_preprocessing/data_preprocessor.py +++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py @@ -161,11 +161,13 @@ def fit_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: Args: x: Input DataFrame. + **kwargs: Parameters to pass to the fit method of each step. + Use step_name__parameter format (e.g., column_selector__protected_features=['feature1']). Returns: Transformed DataFrame with the same index as input. """ - super().fit(X=x) + super().fit(X=x, **kwargs) return self.transform(x) def _find_step_by_type(self, types: Tuple[type, ...]) -> Tuple[Optional[str], Optional[object]]: diff --git a/energy_fault_detector/data_preprocessing/low_unique_value_filter.py b/energy_fault_detector/data_preprocessing/low_unique_value_filter.py index 3502634..d2c311a 100644 --- a/energy_fault_detector/data_preprocessing/low_unique_value_filter.py +++ b/energy_fault_detector/data_preprocessing/low_unique_value_filter.py @@ -1,4 +1,5 @@ from typing import Optional, List +import logging import numpy as np import pandas as pd @@ -6,6 +7,8 @@ from energy_fault_detector.core import DataTransformer +logger = logging.getLogger('energy_fault_detector') + class LowUniqueValueFilter(DataTransformer): """Removes features with low unique values or high fraction of zeroes. @@ -33,7 +36,8 @@ def __init__(self, min_unique_value_count: int = 2, max_col_zero_frac: float = 1 # pylint: disable=attribute-defined-outside-init # noinspection PyAttributeOutsideInit - def fit(self, x: pd.DataFrame, y: Optional[np.array] = None) -> 'LowUniqueValueFilter': + def fit(self, x: pd.DataFrame, y: Optional[np.array] = None, + protected_features: Optional[List[str]] = None) -> 'LowUniqueValueFilter': """Fit the LowUniqueValueFilter to the data. This method evaluates the features based on the number of unique values and the fraction of zeroes, and @@ -42,6 +46,8 @@ def fit(self, x: pd.DataFrame, y: Optional[np.array] = None) -> 'LowUniqueValueF Args: x (pd.DataFrame): The input data with features. y (Optional[np.array]): The target data (not used). + protected_features: list of feature names that should never be dropped (e.g., conditional features for + autoencoders). Warnings will be issued if these features would have been dropped otherwise. Returns: LowUniqueValueFilter: The fitted filter instance. @@ -50,14 +56,37 @@ def fit(self, x: pd.DataFrame, y: Optional[np.array] = None) -> 'LowUniqueValueF self.feature_names_in_ = x.columns.to_list() self.n_features_in_ = len(x.columns) + protected_features = protected_features or [] + protected_lower = [f.lower() for f in protected_features] + original_columns = x.columns counts = x.nunique() low_unique_count = counts[counts < self.min_unique_value_count].index - x = x.drop(low_unique_count, axis=1) + + # Protect features from low unique value dropping + protected_low_unique = [col for col in low_unique_count if col.lower() in protected_lower] + for col in protected_low_unique: + unique_count = counts[col] + logger.warning(f"Feature '{col}' has only {unique_count} unique value(s) (below threshold of " + f"{self.min_unique_value_count}), but is used as a conditional feature. " + f"Keeping it anyway.") + + low_unique_to_drop = [col for col in low_unique_count if col.lower() not in protected_lower] + x = x.drop(low_unique_to_drop, axis=1) zero_pct_per_column = (x == 0).mean(axis=0) columns_to_drop = zero_pct_per_column[zero_pct_per_column > self.max_col_zero_frac].index - x = x.drop(columns_to_drop, axis=1) + + # Protect features from high zero fraction dropping + protected_high_zeros = [col for col in columns_to_drop if col.lower() in protected_lower] + for col in protected_high_zeros: + zero_frac = zero_pct_per_column[col] + logger.warning(f"Feature '{col}' has {zero_frac*100:.1f}% zeros (exceeds threshold of " + f"{self.max_col_zero_frac*100:.1f}%), but is used as a conditional feature. " + f"Keeping it anyway.") + + high_zero_to_drop = [col for col in columns_to_drop if col.lower() not in protected_lower] + x = x.drop(high_zero_to_drop, axis=1) self.columns_dropped_ = [col for col in original_columns if col not in x.columns] self.feature_names_out_ = x.columns.to_list() diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index b862a8a..e31630d 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -75,7 +75,14 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri x_normal = x[y.values] # filter normal before data prep if fit_preprocessor: logger.info('Fit preprocessor pipeline.') - self.data_preprocessor.fit(x_normal) + # Pass conditional features to protect them from being dropped + protected_features = self.autoencoder.conditional_features if self.autoencoder.is_conditional else [] + # Build fit params for pipeline steps that support protected_features + fit_params = {} + for step_name in self.data_preprocessor.named_steps.keys(): + if 'column_selector' in step_name or 'low_unique_value_filter' in step_name: + fit_params[f'{step_name}__protected_features'] = protected_features + self.data_preprocessor.fit(x_normal, **fit_params) x_prepped = self.data_preprocessor.transform(x_normal) diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py index 683c9a6..10eab79 100644 --- a/tests/data_preprocessing/test_data_preprocessor.py +++ b/tests/data_preprocessing/test_data_preprocessor.py @@ -320,3 +320,98 @@ def test_fit_transform_with_timestamp(self): self.assertIn('minute_of_hour_sine', transformed.columns) self.assertIn('minute_of_hour_cosine', transformed.columns) self.assertIn('is_weekend', transformed.columns) + + +class TestDataPreprocessorProtectedFeatures(TestCase): + """Test that protected features (e.g., conditional features for autoencoders) are never dropped.""" + + def setUp(self) -> None: + length = 10 + time_index = pd.date_range(start='1/1/2021', end='10/1/2021', periods=length) + # Create test data where 'conditional_feature' would normally be dropped + data = { + 'normal_feature': list(range(length)), + 'conditional_feature': [1] * length, # constant - would be dropped by LowUniqueValueFilter + 'high_nan_feature': [None] * 8 + [1, 2], # 80% NaN - would be dropped by ColumnSelector + 'another_feature': list(range(length)), + } + self.test_data = pd.DataFrame(index=time_index, data=data) + + def test_protected_feature_not_dropped_by_low_unique_value_filter(self): + """Test that a constant conditional feature is protected from LowUniqueValueFilter.""" + preprocessor = DataPreprocessor( + steps=[ + {'name': 'low_unique_value_filter', + 'params': {'min_unique_value_count': 2}}, + ] + ) + + # Fit without protection - conditional_feature should be dropped + preprocessor.fit(self.test_data) + transformed = preprocessor.transform(self.test_data) + self.assertNotIn('conditional_feature', transformed.columns) + + # Fit WITH protection - conditional_feature should be kept + preprocessor_protected = DataPreprocessor( + steps=[ + {'name': 'low_unique_value_filter', + 'params': {'min_unique_value_count': 2}}, + ] + ) + fit_params = {'low_unique_value_filter__protected_features': ['conditional_feature']} + preprocessor_protected.fit(self.test_data, **fit_params) + transformed_protected = preprocessor_protected.transform(self.test_data) + self.assertIn('conditional_feature', transformed_protected.columns, + "Protected feature should be kept despite being constant") + + def test_protected_feature_not_dropped_by_column_selector(self): + """Test that a high-NaN conditional feature is protected from ColumnSelector.""" + preprocessor = DataPreprocessor( + steps=[ + {'name': 'column_selector', + 'params': {'max_nan_frac_per_col': 0.5}}, # 50% threshold + ] + ) + + # Fit without protection - high_nan_feature should be dropped (80% NaN > 50%) + preprocessor.fit(self.test_data) + transformed = preprocessor.transform(self.test_data) + self.assertNotIn('high_nan_feature', transformed.columns) + + # Fit WITH protection - high_nan_feature should be kept + preprocessor_protected = DataPreprocessor( + steps=[ + {'name': 'column_selector', + 'params': {'max_nan_frac_per_col': 0.5}}, + ] + ) + fit_params = {'column_selector__protected_features': ['high_nan_feature']} + preprocessor_protected.fit(self.test_data, **fit_params) + transformed_protected = preprocessor_protected.transform(self.test_data) + self.assertIn('high_nan_feature', transformed_protected.columns, + "Protected feature should be kept despite high NaN percentage") + + def test_protected_features_with_both_filters(self): + """Test that protected features work with both ColumnSelector and LowUniqueValueFilter.""" + preprocessor = DataPreprocessor( + steps=[ + {'name': 'column_selector', + 'params': {'max_nan_frac_per_col': 0.5}}, + {'name': 'low_unique_value_filter', + 'params': {'min_unique_value_count': 2}}, + ] + ) + + # Protect both problematic features + fit_params = { + 'column_selector__protected_features': ['high_nan_feature', 'conditional_feature'], + 'low_unique_value_filter__protected_features': ['high_nan_feature', 'conditional_feature'] + } + preprocessor.fit(self.test_data, **fit_params) + transformed = preprocessor.transform(self.test_data) + + # Both protected features should be present + self.assertIn('conditional_feature', transformed.columns, + "Constant protected feature should be kept") + self.assertIn('high_nan_feature', transformed.columns, + "High-NaN protected feature should be kept") diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index 2065df0..9d983c8 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -483,3 +483,45 @@ def test_calls_predict_when_reconstruction_is_none(self): with patch.object(ae, 'predict', return_value=self.reconstruction) as mock_predict: result = ae.get_reconstruction_error(self.sensor_data) mock_predict.assert_called_once() + + +class TestFaultDetectorConditionalFeatureProtection(unittest.TestCase): + """Test that FaultDetector protects conditional features from being dropped during preprocessing.""" + + def setUp(self) -> None: + self.config_path = os.path.join(PROJECT_ROOT, 'tests/test_data/test_conditional_ae_config.yaml') + self.conf = Config(self.config_path) + self.test_dir = tempfile.mkdtemp() + + # Create sensor data where conditional feature is constant (would normally be dropped) + np.random.seed(42) + length = 100 + self.sensor_data = pd.DataFrame({ + 'feature_a': [180] * length, # Constant conditional feature + 'feature_b': np.random.random(size=length), + 'feature_c': np.random.random(size=length), + 'feature_d': np.random.random(size=length), + }) + self.normal_index = pd.Series([True] * 80 + [False] * 20) + + def tearDown(self) -> None: + shutil.rmtree(self.test_dir) + + def test_conditional_features_protected_from_dropping(self): + """Test that conditional features specified in the autoencoder are protected during fit.""" + # The config specifies ConditionalAE with 'feature_a' as conditional feature + # It also has LowUniqueValueFilter which would normally drop constant features + fault_detector = FaultDetector(config=self.conf, model_directory=self.test_dir) + + # Fit the model - this should NOT drop the conditional feature despite it being constant + fault_detector.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) + + # Check that the conditional feature is still present after preprocessing + feature_names = fault_detector.data_preprocessor.get_feature_names_out() + self.assertIn('feature_a', feature_names, + "Conditional feature 'feature_a' should be protected from dropping") + + # Verify we can predict with data containing the conditional feature + result = fault_detector.predict(sensor_data=self.sensor_data) + self.assertIsNotNone(result) + self.assertEqual(len(result.predicted_anomalies), len(self.sensor_data)) From 23de58cf2c15b000a6600d7f638a7efde423b953 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Wed, 27 May 2026 20:30:32 +0200 Subject: [PATCH 02/14] Raise an error if conditional features where dropped after DataPreprocessor fitting. --- .../data_preprocessing/data_preprocessor.py | 40 +++++++++++++++++++ energy_fault_detector/fault_detector.py | 17 ++++++-- .../test_data_preprocessor.py | 21 ++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py index 990ddf7..4c54ed2 100644 --- a/energy_fault_detector/data_preprocessing/data_preprocessor.py +++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py @@ -155,6 +155,46 @@ def transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: x_ = super().transform(X=x.copy()) return pd.DataFrame(data=x_, columns=self.get_feature_names_out(), index=x.index) + # pylint: disable=arguments-renamed + def fit(self, X: pd.DataFrame, y=None, **fit_params): + """Fit all transformers in the pipeline. + + Args: + X: Input DataFrame. + y: Target variable (optional). + **fit_params: Parameters to pass to the fit method of each step. + Use step_name__parameter format (e.g., column_selector__protected_features=['feature1']). + Special parameter 'protected_features' can be used to validate that these features + are present in the output after fitting. + + Returns: + self + + Raises: + ValueError: If protected_features are specified but not all are present in output. + """ + # Extract protected features for validation (if provided) + protected_features = fit_params.pop('protected_features', None) + + # Call parent fit + result = super().fit(X=X, y=y, **fit_params) + + # Validate that protected features are in the output + if protected_features: + output_features = self.get_feature_names_out() + missing_features = [f for f in protected_features if f not in output_features] + if missing_features: + raise ValueError( + f"Protected features were dropped by the preprocessing pipeline: {missing_features}. " + f"This may be caused by AngleTransformer or CounterDiffTransformer modifying these features. " + f"Please ensure protected features (e.g., conditional features for autoencoders) are not " + f"transformed by these steps or consider the transformed version of these features (e.g. " + f"_sin, _cos, _rate, _diff) as " + f"conditional features." + ) + + return result + # pylint: disable=arguments-renamed def fit_transform(self, x: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: """Fit and transform in one step. diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index e31630d..cc931ee 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -66,22 +66,33 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri raise ValueError('There are duplicated indices in the input dataframe `sensor_data` and/or in the ' '`normal_index`, please check your input data.') + # Get conditional features early to protect them from transformations + protected_features = self.autoencoder.conditional_features if self.autoencoder.is_conditional else [] + if self.config.data_clipping: logger.debug('Clip data before scaling.') - data_clipper = DataClipper(**self.config.data_clipping_params) + # Don't clip conditional features + clipper_params = self.config.data_clipping_params.copy() + if protected_features: + # Add conditional features to exclusion list + existing_exclusions = clipper_params.get('features_to_exclude', []) + clipper_params['features_to_exclude'] = list(set(existing_exclusions + protected_features)) + logger.debug(f'Excluding conditional features from clipping: {protected_features}') + data_clipper = DataClipper(**clipper_params) data_clipper.fit(x=x) x = data_clipper.transform(x) x_normal = x[y.values] # filter normal before data prep if fit_preprocessor: logger.info('Fit preprocessor pipeline.') - # Pass conditional features to protect them from being dropped - protected_features = self.autoencoder.conditional_features if self.autoencoder.is_conditional else [] # Build fit params for pipeline steps that support protected_features fit_params = {} for step_name in self.data_preprocessor.named_steps.keys(): if 'column_selector' in step_name or 'low_unique_value_filter' in step_name: fit_params[f'{step_name}__protected_features'] = protected_features + # Add global validation that protected features are in the output + if protected_features: + fit_params['protected_features'] = protected_features self.data_preprocessor.fit(x_normal, **fit_params) x_prepped = self.data_preprocessor.transform(x_normal) diff --git a/tests/data_preprocessing/test_data_preprocessor.py b/tests/data_preprocessing/test_data_preprocessor.py index 10eab79..a4a1567 100644 --- a/tests/data_preprocessing/test_data_preprocessor.py +++ b/tests/data_preprocessing/test_data_preprocessor.py @@ -415,3 +415,24 @@ def test_protected_features_with_both_filters(self): "Constant protected feature should be kept") self.assertIn('high_nan_feature', transformed.columns, "High-NaN protected feature should be kept") + + def test_validation_fails_when_protected_feature_missing_from_output(self): + """Test that fit raises ValueError if a protected feature is missing from output.""" + from energy_fault_detector.data_preprocessing.angle_transformer import AngleTransformer + + # Create a pipeline that transforms a protected feature + preprocessor = DataPreprocessor( + steps=[ + {'name': 'angle_transformer', + 'params': {'angles': ['conditional_feature']}}, # Transforms conditional_feature + ] + ) + + # This should raise an error because 'conditional_feature' will be transformed to + # 'conditional_feature_sin' and 'conditional_feature_cos' + with self.assertRaises(ValueError) as context: + preprocessor.fit(self.test_data, protected_features=['conditional_feature']) + + self.assertIn('Protected features were dropped', str(context.exception)) + self.assertIn('conditional_feature', str(context.exception)) + self.assertIn('AngleTransformer or CounterDiffTransformer', str(context.exception)) From 5c014c35ff9d6bb929033c2813f841798eb44eb1 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Wed, 27 May 2026 21:43:45 +0200 Subject: [PATCH 03/14] Fix mocked autoencoder conditional features --- tests/test_fault_detector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index 9d983c8..b17f99b 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -184,6 +184,8 @@ def test_train(self): mock_data_preprocessor.transform.side_effect = [self.sensor_data[self.normal_index], self.sensor_data] mock_autoencoder.get_reconstruction_error.side_effect = [self.recon_error, self.recon_error, self.recon_error] + mock_autoencoder.conditional_features = [] + mock_autoencoder.is_conditional = False mock_score.transform.side_effect = [pd.Series([0.1, 0.2, 0.15])] * 2 results = fault_detector.fit(sensor_data=self.sensor_data, @@ -225,6 +227,7 @@ def test_tune(self): mock_data_preprocessor.transform.side_effect = [self.sensor_data[self.normal_index], self.sensor_data] mock_autoencoder.get_reconstruction_error.side_effect = [self.recon_error] * 9 + mock_autoencoder.is_conditional = False mock_score.transform.side_effect = [pd.Series([0.1, 0.2, 0.15])] * 3 mock_data_preprocessor.transform.side_effect = [self.sensor_data[self.normal_index], self.sensor_data] From 6f85313622fb802320f47e604ad638daa18502d2 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:30:27 +0200 Subject: [PATCH 04/14] Update the .gitignore file --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 8c9cbd2..2283add 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ __pycache__/ .ipynb_checkpoints/ .vscode +# env +.venv/ +.venv*/ + # Coverage etc. .coverage htmlcov/ @@ -32,6 +36,12 @@ test_report.xml *.pkl *.attrs *.model +*.encoder +*.decoder +*.keras +notebooks/PreDist/models/ +notebooks/PreDist/predist_data/ +*.csv # logging *.log From 03518d78832b3feebcf05ddec9a328c82d9cf807 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:32:57 +0200 Subject: [PATCH 05/14] Bugfix model tuning: Do not pass the (stale) EarlyStopping callback from fitting and do not clear session in FaultDetector.tune after loading the model. --- energy_fault_detector/core/autoencoder.py | 5 +++-- energy_fault_detector/fault_detector.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/energy_fault_detector/core/autoencoder.py b/energy_fault_detector/core/autoencoder.py index 138e12d..049a2c9 100644 --- a/energy_fault_detector/core/autoencoder.py +++ b/energy_fault_detector/core/autoencoder.py @@ -220,7 +220,7 @@ def fit(self, x: DataType, x_val: DataType = None, **kwargs) -> "Autoencoder": # ensure verbose default kwargs.setdefault("verbose", self.verbose) - self._fit_internal(x, x_val, epochs=self.epochs, callbacks=self.callbacks, **kwargs) + self._fit_internal(x, x_val, epochs=self.epochs, callbacks=callbacks, **kwargs) return self def _fit_internal(self, x: DataType, x_val: DataType, epochs: int, callbacks: List[Callback], **kwargs) -> None: @@ -282,11 +282,12 @@ def tune( self.compile_model(learning_rate) # sets new learning rate kwargs.setdefault("verbose", self.verbose) + callbacks = kwargs.pop('callbacks', []) self._fit_internal( x, x_val, epochs=tune_epochs + self.epochs_completed, - callbacks=self.callbacks, initial_epoch=self.epochs_completed, + callbacks=callbacks, **kwargs ) return self diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index cc931ee..885e5e6 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -230,7 +230,6 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No logger.warning('Could not import tensorflow.keras.backend.clear_session(). Please install tensorflow.') raise - clear_session() x = sensor_data.sort_index() if normal_index is not None: y = normal_index.sort_index() From 070a1d2172d86a209d0cab20347d4a05f15f6185 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:44:02 +0200 Subject: [PATCH 06/14] Prevent the transform method mutating the original input data, when scale=True. --- energy_fault_detector/anomaly_scores/rmse_score.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/energy_fault_detector/anomaly_scores/rmse_score.py b/energy_fault_detector/anomaly_scores/rmse_score.py index 5bfc0a4..5c29a0d 100644 --- a/energy_fault_detector/anomaly_scores/rmse_score.py +++ b/energy_fault_detector/anomaly_scores/rmse_score.py @@ -70,16 +70,17 @@ def transform(self, x: DataType) -> pd.Series: check_is_fitted(self) + x_ = x if self.scale: # standardization of the reconstruction error in X if np.all(self.std_x_ > 0): - x = (x - self.mean_x_) / self.std_x_ + x_ = (x - self.mean_x_) / self.std_x_ else: - x = x - self.mean_x_ - # replace possible inf values with 0 - x[np.isinf(x)] = 0 + x_ = x - self.mean_x_ + # replace possible inf values with 0 + x_[np.isinf(x_)] = 0 - scores = np.sqrt(np.mean(x ** 2, axis=1)) + scores = np.sqrt(np.mean(x_ ** 2, axis=1)) if isinstance(x, (pd.DataFrame, pd.Series)): scores = pd.Series(scores, index=x.index) From e9c107ed8736e2daa37e961f40e012ce3d2ffffd Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Fri, 29 May 2026 16:30:55 +0200 Subject: [PATCH 07/14] =?UTF-8?q?Enable=20Python=203.12=20support=20and=20?= =?UTF-8?q?TensorFlow=202.16=E2=80=932.18=20compatibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/run-tests.yml | 11 +++-- .../bidirectional_lstm_seq2one_autoencoder.py | 15 +++--- .../autoencoders/cnn_seq2one_autoencoder.py | 17 +++---- .../autoencoders/cnn_seq_autoencoder.py | 4 +- .../autoencoders/conditional_autoencoder.py | 4 +- .../autoencoders/lstm_seq2one_autoencoder.py | 19 ++++--- .../autoencoders/lstm_seq2seq_autoencoder.py | 15 +++--- .../autoencoders/multilayer_autoencoder.py | 4 +- .../autoencoders/sequence_autoencoder.py | 2 +- energy_fault_detector/core/autoencoder.py | 12 +++-- energy_fault_detector/fault_detector.py | 4 +- .../root_cause_analysis/arcana.py | 2 +- .../_adaptive_threshold_model.py | 8 +-- .../example_models/example_autoencoder.py | 4 +- pyproject.toml | 8 +-- .../test_conditional_autoencoder.py | 1 - tests/root_cause_analysis/test_arcana.py | 6 +-- .../test_arcana_seq2seq.py | 4 +- tests/test_fault_detector.py | 7 +-- tests/utils/test_analysis.py | 49 ++++++++++--------- 20 files changed, 102 insertions(+), 94 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c0472c7..7acd6a4 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -7,6 +7,9 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout code @@ -14,10 +17,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | @@ -31,11 +34,11 @@ jobs: - name: Upload coverage report uses: actions/upload-artifact@v4 with: - name: coverage-report + name: coverage-report-${{ matrix.python-version }} path: coverage.xml - name: Upload JUnit report uses: actions/upload-artifact@v4 with: - name: junit-report + name: junit-report-${{ matrix.python-version }} path: test_report.xml diff --git a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py index adde6cd..8f31ac9 100644 --- a/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py @@ -2,9 +2,8 @@ from typing import List, Optional, Tuple -import tensorflow as tf -from tensorflow.keras import regularizers -from tensorflow.keras.layers import ( +from keras import regularizers +from keras.layers import ( Bidirectional, Concatenate, Dense, @@ -13,7 +12,7 @@ LSTM, RepeatVector, ) -from tensorflow.keras.models import Model as KerasModel +from keras.models import Model as KerasModel from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder @@ -153,13 +152,13 @@ def create_model( encoded = Dropout(rate=self.dropout_rate)(encoded) if conditional_input is not None: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=[main_input, conditional_input], outputs=encoded, name="encoder", ) else: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=main_input, outputs=encoded, name="encoder", @@ -182,13 +181,13 @@ def create_model( )(last_timestep) if conditional_input is not None: - self.model = tf.keras.Model( + self.model = KerasModel( inputs=[main_input, conditional_input], outputs=reconstruction, name="bidirectional_lstm_seq2one_autoencoder", ) else: - self.model = tf.keras.Model( + self.model = KerasModel( inputs=main_input, outputs=reconstruction, name="bidirectional_lstm_seq2one_autoencoder", diff --git a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py index 7f7b651..0cda7e3 100644 --- a/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py @@ -4,8 +4,7 @@ import logging import numpy as np -import tensorflow as tf -from tensorflow.keras.layers import ( +from keras.layers import ( Input, Conv1D, BatchNormalization, @@ -15,7 +14,7 @@ Flatten, MaxPooling1D, ) -from tensorflow.keras.models import Model as KerasModel +from keras.models import Model as KerasModel from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder @@ -161,13 +160,13 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op # Encoder model for latent representation if conditional_input is not None: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=[main_input, conditional_input], outputs=encoded, name="encoder", ) else: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=main_input, outputs=encoded, name="encoder", @@ -193,13 +192,13 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op # Stand-alone decoder model if conditional_input is not None: - self.decoder = tf.keras.Model( + self.decoder = KerasModel( inputs=[latent_input, cond_last_input], outputs=reconstruction, name="decoder", ) else: - self.decoder = tf.keras.Model( + self.decoder = KerasModel( inputs=latent_input, outputs=reconstruction, name="decoder", @@ -210,7 +209,7 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op encoded = self.encoder(inputs=[main_input, conditional_input]) cond_last = conditional_input[:, -1, :] decoded = self.decoder([encoded, cond_last]) - self.model = tf.keras.Model( + self.model = KerasModel( inputs=[main_input, conditional_input], outputs=decoded, name="cnn_seq2one_autoencoder", @@ -218,7 +217,7 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op else: encoded = self.encoder(main_input) decoded = self.decoder(encoded) - self.model = tf.keras.Model( + self.model = KerasModel( inputs=main_input, outputs=decoded, name="cnn_seq2one_autoencoder", diff --git a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py index 1681902..becfe85 100644 --- a/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py +++ b/energy_fault_detector/autoencoders/cnn_seq_autoencoder.py @@ -2,7 +2,7 @@ from typing import List, Optional, Tuple -from tensorflow.keras.layers import ( +from keras.layers import ( Input, Conv1D, Conv1DTranspose, @@ -10,7 +10,7 @@ Dropout, Concatenate ) -from tensorflow.keras.models import Model as KerasModel +from keras.models import Model as KerasModel from .seq2seq_autoencoder import Seq2SeqAutoencoder from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder diff --git a/energy_fault_detector/autoencoders/conditional_autoencoder.py b/energy_fault_detector/autoencoders/conditional_autoencoder.py index d661318..f923b50 100644 --- a/energy_fault_detector/autoencoders/conditional_autoencoder.py +++ b/energy_fault_detector/autoencoders/conditional_autoencoder.py @@ -2,8 +2,8 @@ from typing import List, Optional -from tensorflow.keras.models import Model as KerasModel -from tensorflow.keras.layers import Dense, PReLU, Input, Concatenate +from keras.models import Model as KerasModel +from keras.layers import Dense, PReLU, Input, Concatenate from energy_fault_detector.core.autoencoder import Autoencoder diff --git a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py index 9d01174..c771f4e 100644 --- a/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py +++ b/energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py @@ -2,16 +2,15 @@ from typing import List, Optional, Tuple -import tensorflow as tf -from tensorflow.keras import regularizers -from tensorflow.keras.layers import ( +from keras import regularizers +from keras.layers import ( Input, LSTM, Dropout, Dense, Concatenate, ) -from tensorflow.keras.models import Model as KerasModel +from keras.models import Model as KerasModel from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder @@ -144,13 +143,13 @@ def create_model( # Encoder model for latent representation if conditional_input is not None: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=[main_input, conditional_input], outputs=encoded, name="encoder", ) else: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=main_input, outputs=encoded, name="encoder", @@ -173,13 +172,13 @@ def create_model( # Stand-alone decoder model if conditional_input is not None: - self.decoder = tf.keras.Model( + self.decoder = KerasModel( inputs=[latent_input, cond_last_input], outputs=reconstruction, name="decoder", ) else: - self.decoder = tf.keras.Model( + self.decoder = KerasModel( inputs=latent_input, outputs=reconstruction, name="decoder", @@ -189,14 +188,14 @@ def create_model( enc = self.encoder(inputs=[main_input, conditional_input]) cond_last = conditional_input[:, -1, :] decoded = self.decoder([enc, cond_last]) - self.model = tf.keras.Model( + self.model = KerasModel( inputs=[main_input, conditional_input], outputs=decoded, ) else: enc = self.encoder(main_input) decoded = self.decoder(enc) - self.model = tf.keras.Model( + self.model = KerasModel( inputs=main_input, outputs=decoded, ) diff --git a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py index 467dc91..3d1fba6 100644 --- a/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py +++ b/energy_fault_detector/autoencoders/lstm_seq2seq_autoencoder.py @@ -2,9 +2,8 @@ from typing import List, Optional, Tuple -import tensorflow as tf -from tensorflow.keras import regularizers -from tensorflow.keras.layers import ( +from keras import regularizers +from keras.layers import ( Input, LSTM, Dropout, @@ -13,7 +12,7 @@ TimeDistributed, Concatenate, ) -from tensorflow.keras.models import Model as KerasModel +from keras.models import Model as KerasModel from .seq2seq_autoencoder import Seq2SeqAutoencoder from ..data_splitting.sequence_dataset import SequenceDatasetBuilder @@ -133,13 +132,13 @@ def create_model( # Encoder model (for latent representation) if conditional_input is not None: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=[main_input, conditional_input], outputs=encoded, name="encoder", ) else: - self.encoder = tf.keras.Model( + self.encoder = KerasModel( inputs=main_input, outputs=encoded, name="encoder", @@ -160,12 +159,12 @@ def create_model( )(decoder_output) if conditional_input is not None: - self.model = tf.keras.Model( + self.model = KerasModel( inputs=[main_input, conditional_input], outputs=reconstruction, ) else: - self.model = tf.keras.Model( + self.model = KerasModel( inputs=main_input, outputs=reconstruction, ) diff --git a/energy_fault_detector/autoencoders/multilayer_autoencoder.py b/energy_fault_detector/autoencoders/multilayer_autoencoder.py index fa71767..d5cb1b1 100644 --- a/energy_fault_detector/autoencoders/multilayer_autoencoder.py +++ b/energy_fault_detector/autoencoders/multilayer_autoencoder.py @@ -6,8 +6,8 @@ import pandas as pd # pylint: disable=E0401,E0611 -from tensorflow.keras.models import Model as KerasModel -from tensorflow.keras.layers import Dense, PReLU, Input +from keras.models import Model as KerasModel +from keras.layers import Dense, PReLU, Input from energy_fault_detector.core.autoencoder import Autoencoder diff --git a/energy_fault_detector/autoencoders/sequence_autoencoder.py b/energy_fault_detector/autoencoders/sequence_autoencoder.py index 9f7e34d..b3f8846 100644 --- a/energy_fault_detector/autoencoders/sequence_autoencoder.py +++ b/energy_fault_detector/autoencoders/sequence_autoencoder.py @@ -11,7 +11,7 @@ import numpy as np import pandas as pd import tensorflow as tf -from tensorflow.keras.callbacks import Callback +from keras.callbacks import Callback from energy_fault_detector.core.autoencoder import Autoencoder from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder diff --git a/energy_fault_detector/core/autoencoder.py b/energy_fault_detector/core/autoencoder.py index 049a2c9..7332132 100644 --- a/energy_fault_detector/core/autoencoder.py +++ b/energy_fault_detector/core/autoencoder.py @@ -12,10 +12,10 @@ tf.get_logger().setLevel("ERROR") # # pylint: disable=E0401,E0611,C0413 -from tensorflow.keras.models import load_model as load_keras_model, Model as KerasModel -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.optimizers.schedules import ExponentialDecay -from tensorflow.keras.callbacks import EarlyStopping, Callback +from keras.models import load_model as load_keras_model, Model as KerasModel +from keras.optimizers import Adam +from keras.optimizers.schedules import ExponentialDecay +from keras.callbacks import EarlyStopping, Callback from energy_fault_detector.core.save_load_mixin import SaveLoadMixin @@ -144,12 +144,16 @@ def summary(self, **kwargs) -> None: def __call__(self, x: Union[np.ndarray, tf.Tensor], conditions: Union[np.ndarray, tf.Tensor] = None) -> tf.Tensor: """Calls the model on new inputs.""" + + x = tf.convert_to_tensor(x, dtype=tf.float32) + if self.is_conditional: if conditions is None: raise ValueError( "To call an conditional autoencoder on new input, the conditions need to be provided" " as well: `Autoencoder(inputs, conditions)`." ) + conditions = tf.convert_to_tensor(conditions, dtype=tf.float32) return self.model([x, conditions]) return self.model(x) diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index 885e5e6..67d4944 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -126,7 +126,7 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo """ try: - from tensorflow.keras.backend import clear_session + from keras.backend import clear_session except ImportError: logger.warning('Could not import tensorflow.keras.backend.clear_session(). Please install tensorflow.') raise @@ -225,7 +225,7 @@ def tune(self, sensor_data: pd.DataFrame, normal_index: Optional[pd.Series] = No raise ValueError('No models loaded and no pretrained_model_path provided!') try: - from tensorflow.keras.backend import clear_session + from keras.backend import clear_session except ImportError: logger.warning('Could not import tensorflow.keras.backend.clear_session(). Please install tensorflow.') raise diff --git a/energy_fault_detector/root_cause_analysis/arcana.py b/energy_fault_detector/root_cause_analysis/arcana.py index a450dbf..11387af 100644 --- a/energy_fault_detector/root_cause_analysis/arcana.py +++ b/energy_fault_detector/root_cause_analysis/arcana.py @@ -10,7 +10,7 @@ tf.get_logger().setLevel('ERROR') -from tensorflow.keras.optimizers import Adam +from keras.optimizers import Adam from energy_fault_detector.core.autoencoder import Autoencoder from energy_fault_detector.autoencoders.sequence_autoencoder import SequenceAutoencoder diff --git a/energy_fault_detector/threshold_selectors/_adaptive_threshold_model.py b/energy_fault_detector/threshold_selectors/_adaptive_threshold_model.py index b92c9fe..dac2ab5 100644 --- a/energy_fault_detector/threshold_selectors/_adaptive_threshold_model.py +++ b/energy_fault_detector/threshold_selectors/_adaptive_threshold_model.py @@ -1,8 +1,8 @@ import numpy as np -from tensorflow.keras import Input, Model as KerasModel -from tensorflow.keras.callbacks import EarlyStopping -from tensorflow.keras.layers import Dense -from tensorflow.keras.optimizers import Adam +from keras import Input, Model as KerasModel +from keras.callbacks import EarlyStopping +from keras.layers import Dense +from keras.optimizers import Adam class RegressionNN: diff --git a/notebooks/example_models/example_autoencoder.py b/notebooks/example_models/example_autoencoder.py index 70faeb9..f9941de 100644 --- a/notebooks/example_models/example_autoencoder.py +++ b/notebooks/example_models/example_autoencoder.py @@ -1,7 +1,7 @@ from typing import List, Union, Tuple -from tensorflow.keras import Model as KerasModel -from tensorflow.keras import layers +from keras import Model as KerasModel +from keras import layers from energy_fault_detector.autoencoders import MultilayerAutoencoder diff --git a/pyproject.toml b/pyproject.toml index b9f90a5..0627365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,13 @@ authors = [ ] description = "Automated fault detection in renewable energy assets and power grids." readme = "README.md" -requires-python = ">=3.10,<3.12" +requires-python = ">=3.10,<3.13" classifiers = [ 'Development Status :: 4 - Beta', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', "Operating System :: OS Independent", "License :: OSI Approved :: MIT License", ] @@ -22,10 +23,11 @@ license = { file = "LICENSE" } dependencies = [ "numpy>=1.23.5,<2.0", "scipy>=1.11,<1.17", - "tensorflow==2.15.1", + "tensorflow>=2.16,<2.19; python_version>='3.12'", + "tensorflow>=2.15.1,<2.19; python_version<'3.12'", "pyyaml", "scikit-learn>=1.5.2,<=1.7.0", - "pandas>=2.0", + "pandas>=2.0,<3.0", "cerberus", "matplotlib>=3.9.0", "optuna>=4.0", diff --git a/tests/autoencoders/test_conditional_autoencoder.py b/tests/autoencoders/test_conditional_autoencoder.py index 4769d72..a7265a5 100644 --- a/tests/autoencoders/test_conditional_autoencoder.py +++ b/tests/autoencoders/test_conditional_autoencoder.py @@ -1,6 +1,5 @@ """Conditional AE tests""" -import os.path import shutil from typing import Dict from unittest import TestCase diff --git a/tests/root_cause_analysis/test_arcana.py b/tests/root_cause_analysis/test_arcana.py index 4210645..033362b 100644 --- a/tests/root_cause_analysis/test_arcana.py +++ b/tests/root_cause_analysis/test_arcana.py @@ -5,9 +5,9 @@ import numpy as np import pandas as pd from numpy.testing import assert_array_almost_equal -from tensorflow.keras.models import Model -from tensorflow.keras.layers import Dense -from tensorflow.keras.layers import Input +from keras.models import Model +from keras.layers import Dense +from keras.layers import Input import tensorflow as tf from energy_fault_detector.root_cause_analysis.arcana import Arcana diff --git a/tests/root_cause_analysis/test_arcana_seq2seq.py b/tests/root_cause_analysis/test_arcana_seq2seq.py index ba23f9a..36dd1a0 100644 --- a/tests/root_cause_analysis/test_arcana_seq2seq.py +++ b/tests/root_cause_analysis/test_arcana_seq2seq.py @@ -2,8 +2,8 @@ import numpy as np import pandas as pd -from tensorflow.keras import layers -from tensorflow.keras.models import Model as KerasModel +from keras import layers +from keras.models import Model as KerasModel from energy_fault_detector.root_cause_analysis.arcana import Arcana from energy_fault_detector.autoencoders.seq2seq_autoencoder import Seq2SeqAutoencoder diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index b17f99b..6f1f91c 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -215,12 +215,13 @@ def test_train(self): self.sensor_data] mock_autoencoder.get_reconstruction_error.side_effect = [self.recon_error, self.recon_error, self.recon_error] mock_score.transform.return_value = pd.Series([0.1, 0.2, 0.15]) + mock_score.reset_mock() _ = fault_detector.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) - mock_score.save.asset_not_called() + mock_score.save.assert_not_called() self.assertEqual(self.conf.write_config.call_count, 1) def test_tune(self): @@ -235,7 +236,7 @@ def test_tune(self): tune_results = fault_detector.tune(sensor_data=self.sensor_data, normal_index=self.normal_index, new_learning_rate=0.001, tune_epochs=1, tune_method='full', save_models=False) - mock_autoencoder.tune.called_once() + mock_autoencoder.tune.assert_called_once() mock_data_preprocessor.transform.side_effect = [self.sensor_data[self.normal_index], self.sensor_data] @@ -243,7 +244,7 @@ def test_tune(self): tune_results = fault_detector.tune(sensor_data=self.sensor_data, normal_index=self.normal_index, new_learning_rate=0.001, tune_epochs=1, tune_method='decoder', save_models=False) - mock_autoencoder.tune_decoder.called_once() + mock_autoencoder.tune_decoder.assert_called_once() @patch('energy_fault_detector.root_cause_analysis.arcana.Arcana.find_arcana_bias') def test_predict(self, mock_find_arcana_bias): diff --git a/tests/utils/test_analysis.py b/tests/utils/test_analysis.py index 80568f3..f8be152 100644 --- a/tests/utils/test_analysis.py +++ b/tests/utils/test_analysis.py @@ -3,7 +3,6 @@ import numpy as np import pandas as pd -import pytest from energy_fault_detector.utils import analysis from energy_fault_detector.utils.analysis import calculate_criticality, create_events @@ -49,11 +48,13 @@ def test_non_normal_periods_ignored(self): assert list(crit) == [1, 2, 2, 2, 3] -class TestCalculateCriticalityMultiIndex: +class TestCalculateCriticalityMultiIndex(unittest.TestCase): """MultiIndex (asset_id, timestamp) behavior.""" - @pytest.fixture - def multi_index_data(self): + def setUp(self) -> None: + self.multi_index_data = self.create_multi_index_data() + + def create_multi_index_data(self): """Two assets, each with 5 timestamps.""" times = pd.date_range("2024-01-01", periods=5, freq="10min") idx = pd.MultiIndex.from_product( @@ -61,9 +62,9 @@ def multi_index_data(self): ) return idx, times - def test_groups_are_independent(self, multi_index_data): + def test_groups_are_independent(self): """Criticality in asset_A does not leak into asset_B.""" - idx, times = multi_index_data + idx, times = self.multi_index_data # asset_A: all anomalies → criticality rises # asset_B: no anomalies → criticality stays 0 anomalies = pd.Series( @@ -74,9 +75,9 @@ def test_groups_are_independent(self, multi_index_data): assert list(crit.loc["asset_A"]) == [1, 2, 3, 4, 5] assert list(crit.loc["asset_B"]) == [0, 0, 0, 0, 0] - def test_multiindex_with_normal_idx(self, multi_index_data): + def test_multiindex_with_normal_idx(self): """Normal index is respected per group.""" - idx, times = multi_index_data + idx, times = self.multi_index_data anomalies = pd.Series([True] * 10, index=idx) # asset_A: normal everywhere, asset_B: not normal at indices 0,1 normal_idx = pd.Series( @@ -88,18 +89,18 @@ def test_multiindex_with_normal_idx(self, multi_index_data): # asset_B: 0, 0 (not normal), then +1, +1, +1 assert list(crit.loc["asset_B"]) == [0, 0, 1, 2, 3] - def test_multiindex_init_criticality_per_group(self, multi_index_data): + def test_multiindex_init_criticality_per_group(self): """init_criticality applies to each group independently.""" - idx, _ = multi_index_data + idx, _ = self.multi_index_data anomalies = pd.Series([False] * 10, index=idx) crit = calculate_criticality(anomalies, init_criticality=3) # Each group starts at 3 and decreases assert list(crit.loc["asset_A"]) == [2, 1, 0, 0, 0] assert list(crit.loc["asset_B"]) == [2, 1, 0, 0, 0] - def test_preserves_multiindex(self, multi_index_data): + def test_preserves_multiindex(self): """Output has the same MultiIndex as input.""" - idx, _ = multi_index_data + idx, _ = self.multi_index_data anomalies = pd.Series([True] * 10, index=idx) crit = calculate_criticality(anomalies) assert isinstance(crit.index, pd.MultiIndex) @@ -141,11 +142,13 @@ def test_multiple_events(self): assert len(events) == 2 -class TestCreateEventsMultiIndex: +class TestCreateEventsMultiIndex(unittest.TestCase): """MultiIndex (asset_id, timestamp) behavior.""" - @pytest.fixture - def multi_index_sensor_data(self): + def setUp(self): + self.multi_index_sensor_data = self.create_multi_index_sensor_data() + + def create_multi_index_sensor_data(self): times = pd.date_range("2024-01-01", periods=20, freq="10min") idx = pd.MultiIndex.from_product( [["asset_A", "asset_B"], times], names=["asset_id", "timestamp"] @@ -153,9 +156,9 @@ def multi_index_sensor_data(self): df = pd.DataFrame({"power": np.random.randn(40)}, index=idx) return df, idx, times - def test_events_per_group(self, multi_index_sensor_data): + def test_events_per_group(self): """Events are detected independently per asset.""" - df, idx, times = multi_index_sensor_data + df, idx, times = self.multi_index_sensor_data # asset_A: event from index 5-16 (12 timestamps) # asset_B: no event (only 3 consecutive True) bools_a = [False] * 5 + [True] * 12 + [False] * 3 @@ -168,9 +171,9 @@ def test_events_per_group(self, multi_index_sensor_data): assert meta.iloc[0]["group"] == "asset_A" assert len(events) == 1 - def test_events_in_multiple_groups(self, multi_index_sensor_data): + def test_events_in_multiple_groups(self): """Events detected in both assets.""" - df, idx, times = multi_index_sensor_data + df, idx, times = self.multi_index_sensor_data # Both assets have a 12-timestamp event bools_a = [False] * 5 + [True] * 12 + [False] * 3 bools_b = [False] * 2 + [True] * 12 + [False] * 6 @@ -182,17 +185,17 @@ def test_events_in_multiple_groups(self, multi_index_sensor_data): assert set(meta["group"]) == {"asset_A", "asset_B"} assert len(events) == 2 - def test_no_events_returns_empty(self, multi_index_sensor_data): + def test_no_events_returns_empty(self): """No events in any group.""" - df, idx, _ = multi_index_sensor_data + df, idx, _ = self.multi_index_sensor_data bools = pd.Series([False] * 40, index=idx) meta, events = create_events(df, bools, min_event_length=10) assert meta.empty assert len(events) == 0 - def test_meta_has_group_column(self, multi_index_sensor_data): + def test_meta_has_group_column(self): """MultiIndex results include a 'group' column.""" - df, idx, _ = multi_index_sensor_data + df, idx, _ = self.multi_index_sensor_data bools = pd.Series([True] * 20 + [False] * 20, index=idx) meta, events = create_events(df, bools, min_event_length=10) assert "group" in meta.columns From 159b2efb344bd2f995d6f6272e06c9e2bc415ff8 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Fri, 29 May 2026 16:48:02 +0200 Subject: [PATCH 08/14] Rename the submodule containing the quick_fault_detector function to pipeline.py to prevent shadowing the module in the root __init__.py file. Also updates the CLI tests to patch the objects directly. --- docs/quick_fault_detection.rst | 2 +- .../quick_fault_detection/__init__.py | 2 +- .../{quick_fault_detector.py => pipeline.py} | 0 .../test_quick_fault_detector.py | 32 ++++++++----------- 4 files changed, 16 insertions(+), 20 deletions(-) rename energy_fault_detector/quick_fault_detection/{quick_fault_detector.py => pipeline.py} (100%) diff --git a/docs/quick_fault_detection.rst b/docs/quick_fault_detection.rst index 46a7651..1b07b9c 100644 --- a/docs/quick_fault_detection.rst +++ b/docs/quick_fault_detection.rst @@ -123,7 +123,7 @@ The underlying helper functions are implemented in: - :mod:`energy_fault_detector.quick_fault_detection.data_loading` - :mod:`energy_fault_detector.quick_fault_detection.configuration` -- :mod:`energy_fault_detector.quick_fault_detection.quick_fault_detector` +- :mod:`energy_fault_detector.quick_fault_detection.pipeline` Output ------ diff --git a/energy_fault_detector/quick_fault_detection/__init__.py b/energy_fault_detector/quick_fault_detection/__init__.py index f6c551d..06f7dcc 100644 --- a/energy_fault_detector/quick_fault_detection/__init__.py +++ b/energy_fault_detector/quick_fault_detection/__init__.py @@ -1,2 +1,2 @@ -from .quick_fault_detector import quick_fault_detector +from .pipeline import quick_fault_detector diff --git a/energy_fault_detector/quick_fault_detection/quick_fault_detector.py b/energy_fault_detector/quick_fault_detection/pipeline.py similarity index 100% rename from energy_fault_detector/quick_fault_detection/quick_fault_detector.py rename to energy_fault_detector/quick_fault_detection/pipeline.py diff --git a/tests/quick_fault_detection/test_quick_fault_detector.py b/tests/quick_fault_detection/test_quick_fault_detector.py index 3081fc2..c547e24 100644 --- a/tests/quick_fault_detection/test_quick_fault_detector.py +++ b/tests/quick_fault_detection/test_quick_fault_detector.py @@ -4,15 +4,11 @@ import tempfile import unittest from unittest.mock import patch, MagicMock -from pathlib import Path import numpy as np import pandas as pd -from energy_fault_detector.quick_fault_detection.quick_fault_detector import ( - quick_fault_detector, - analyze_event, -) +import energy_fault_detector.quick_fault_detection.pipeline as qfd_module from energy_fault_detector.core.fault_detection_result import FaultDetectionResult @@ -48,8 +44,8 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.train_path, self.test_path = _make_csv_files(self.tmp_dir) - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.generate_output_plots") - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.FaultDetector") + @patch.object(qfd_module, "generate_output_plots") + @patch.object(qfd_module, "FaultDetector") def test_full_pipeline_no_anomalies(self, MockFaultDetector, mock_plots): """Pipeline runs end-to-end when no anomalies are detected.""" # Setup mock @@ -66,7 +62,7 @@ def test_full_pipeline_no_anomalies(self, MockFaultDetector, mock_plots): ) mock_model.predict.return_value = mock_result - result, events = quick_fault_detector( + result, events = qfd_module.quick_fault_detector( csv_data_path=self.train_path, csv_test_data_path=self.test_path, time_column_name="timestamp", @@ -80,8 +76,8 @@ def test_full_pipeline_no_anomalies(self, MockFaultDetector, mock_plots): self.assertEqual(len(events), 0) mock_plots.assert_called_once() - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.generate_output_plots") - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.FaultDetector") + @patch.object(qfd_module, "generate_output_plots") + @patch.object(qfd_module, "FaultDetector") def test_full_pipeline_with_anomalies(self, MockFaultDetector, mock_plots): """Pipeline runs ARCANA when anomalies are detected.""" n_rows = 200 @@ -112,7 +108,7 @@ def test_full_pipeline_with_anomalies(self, MockFaultDetector, mock_plots): [], # tracked_bias ) - result, events = quick_fault_detector( + result, events = qfd_module.quick_fault_detector( csv_data_path=self.train_path, csv_test_data_path=self.test_path, time_column_name="timestamp", @@ -124,8 +120,8 @@ def test_full_pipeline_with_anomalies(self, MockFaultDetector, mock_plots): self.assertGreater(len(events), 0) mock_model.run_root_cause_analysis.assert_called() - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.generate_output_plots") - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.FaultDetector") + @patch.object(qfd_module, "generate_output_plots") + @patch.object(qfd_module, "FaultDetector") def test_features_to_exclude_passed_to_config(self, MockFaultDetector, mock_plots): """Verify features_to_exclude reaches the config.""" n_rows = 200 @@ -139,7 +135,7 @@ def test_features_to_exclude_passed_to_config(self, MockFaultDetector, mock_plot ) mock_model.predict.return_value = mock_result - quick_fault_detector( + qfd_module.quick_fault_detector( csv_data_path=self.train_path, csv_test_data_path=self.test_path, time_column_name="timestamp", @@ -164,9 +160,9 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.train_path, self.test_path = _make_csv_files(self.tmp_dir) - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.generate_output_plots") - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.FaultDetector") - @patch("energy_fault_detector.quick_fault_detection.quick_fault_detector.load_train_test_data") + @patch.object(qfd_module, "generate_output_plots") + @patch.object(qfd_module, "FaultDetector") + @patch.object(qfd_module, "load_train_test_data") def test_cli_invocation(self, mock_load, MockFaultDetector, mock_plots): """CLI parses args and calls quick_fault_detector.""" from energy_fault_detector.main import main @@ -218,7 +214,7 @@ def test_returns_importances_and_losses(self): ) mock_detector.run_root_cause_analysis.return_value = (bias, pd.DataFrame(), []) - importances, losses = analyze_event(mock_detector, event_data, track_losses=False) + importances, losses = qfd_module.analyze_event(mock_detector, event_data, track_losses=False) self.assertEqual(len(importances), 3) # calculate_mean_arcana_importances normalizes: 0.5 / (0.1 + 0.5 + 0.0) From 536493c1c5e7a2c35c7b7fe9c9fc213a8a48676e Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:30:29 +0200 Subject: [PATCH 09/14] Update actions/checkout to latest documented version --- .github/workflows/sync-to-gitlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-to-gitlab.yml b/.github/workflows/sync-to-gitlab.yml index a6d278f..2d4b7c9 100644 --- a/.github/workflows/sync-to-gitlab.yml +++ b/.github/workflows/sync-to-gitlab.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest name: Git Repo Sync steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 with: fetch-depth: 0 - uses: wangchucheng/git-repo-sync@v0.1.0 From d56ba76929bcc582a65bd43a8d4c5e5c4f574e29 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:35:36 +0200 Subject: [PATCH 10/14] Remove outdated branch --- .github/workflows/sync-to-gitlab.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-to-gitlab.yml b/.github/workflows/sync-to-gitlab.yml index 2d4b7c9..9264dd6 100644 --- a/.github/workflows/sync-to-gitlab.yml +++ b/.github/workflows/sync-to-gitlab.yml @@ -5,7 +5,6 @@ on: branches: - main - develop - - time-series-handling jobs: sync: From 5c0f0e20421c26f1f5c1e06f64b16003744f4f82 Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:38:46 +0200 Subject: [PATCH 11/14] Remove outdated code, fix a typo and check class instance with the actual class instead of name. --- .../data_preprocessing/data_preprocessor.py | 2 +- energy_fault_detector/fault_detector.py | 7 ++++--- .../threshold_selectors/fbeta_threshold.py | 3 --- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/energy_fault_detector/data_preprocessing/data_preprocessor.py b/energy_fault_detector/data_preprocessing/data_preprocessor.py index 4c54ed2..2387ae7 100644 --- a/energy_fault_detector/data_preprocessing/data_preprocessor.py +++ b/energy_fault_detector/data_preprocessing/data_preprocessor.py @@ -362,7 +362,7 @@ def _order_steps_spec(self, steps_spec: List[Dict[str, Any]]) -> List[Dict[str, # Order the preprocessing steps ordered = [] - # can add NaN avalues or add new features that may be constant + # can add NaN values or add new features that may be constant ordered.extend(duplicates) ordered.extend(counter) # drop columns based on the values (NaNs, no variance) diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index 67d4944..4706877 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -13,7 +13,7 @@ from energy_fault_detector.data_preprocessing.data_preprocessor import DataPreprocessor from energy_fault_detector.data_preprocessing.data_clipper import DataClipper from energy_fault_detector.config import Config -from energy_fault_detector.threshold_selectors import FbetaSelector, FDRSelector +from energy_fault_detector.threshold_selectors import FbetaSelector, FDRSelector, AdaptiveThresholdSelector logger = logging.getLogger('energy_fault_detector') @@ -307,6 +307,7 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, if model_path is not None: self._load_from_path(model_path=model_path) + self.model_directory = model_path else: if self.data_preprocessor is None: raise ValueError('No models loaded and no model_path provided!') @@ -323,7 +324,7 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, x_predicted = self.autoencoder.predict(x_prepped, return_conditions=True) x_predicted = x_predicted[column_order] main_cols = [c for c in column_order if c not in self.autoencoder.conditional_features] - recon_error = self.autoencoder.get_reconstruction_error(x=x_prepped,reconstruction=x_predicted[main_cols]) + recon_error = self.autoencoder.get_reconstruction_error(x=x_prepped, reconstruction=x_predicted[main_cols]) else: x_predicted = self.autoencoder.predict(x_prepped) recon_error = self.autoencoder.get_reconstruction_error(x_prepped, reconstruction=x_predicted) @@ -434,7 +435,7 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit logger.info('Fit threshold.') # fit threshold - x_prepped and labels are filtered based on scores (all or validation data only) used - if self.threshold_selector.__class__.__name__ == 'AdaptiveThresholdSelector': + if isinstance(self.threshold_selector, AdaptiveThresholdSelector): self.threshold_selector.fit(scaled_ae_input=x_prepped_all.loc[scores.index], anomaly_score=scores, normal_index=y.loc[scores.index]) diff --git a/energy_fault_detector/threshold_selectors/fbeta_threshold.py b/energy_fault_detector/threshold_selectors/fbeta_threshold.py index d73c1b2..34bdc67 100644 --- a/energy_fault_detector/threshold_selectors/fbeta_threshold.py +++ b/energy_fault_detector/threshold_selectors/fbeta_threshold.py @@ -129,9 +129,6 @@ def mark_normal_outliers(self, anomaly_score: np.ndarray, normal_index: np.ndarr selection (np.array): Boolean array which is true for all samples that are either normal and below quantile or not normal. """ - if not hasattr(self, 'quantile'): - # backwards compatibility - self.quantile = 1. quantile = np.quantile(anomaly_score[normal_index], q=self.quantile) all_scores_compared = anomaly_score < quantile From c3ce60f12aab49311e5c80fc8f89ad93418ae5fb Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:16:04 +0200 Subject: [PATCH 12/14] - Keep the old behaviour of dropping conditional features the default, so we do not have any breaking changes. - Introduce a flag in the config `protect_conditional_features` to set to True if the user wants to protect these features. - Update the conditional feature list of the autoencoder if some of the conditions are missing. --- energy_fault_detector/config/config.py | 6 + energy_fault_detector/fault_detector.py | 149 +++++++++----- tests/config/test_config.py | 3 +- tests/test_fault_detector.py | 249 ++++++++++++++++++++++-- 4 files changed, 339 insertions(+), 68 deletions(-) diff --git a/energy_fault_detector/config/config.py b/energy_fault_detector/config/config.py index 9d82f1e..b1fa312 100644 --- a/energy_fault_detector/config/config.py +++ b/energy_fault_detector/config/config.py @@ -76,6 +76,7 @@ 'shuffle': {'type': 'boolean', 'required': False, 'dependencies': {'type': ['sklearn', 'train_test_split']}}, } }, + 'protect_conditional_features': {'type': 'boolean', 'required': False, 'default': False}, } ROOT_CAUSE_ANALYSIS_SCHEMA = { @@ -267,6 +268,11 @@ def dtype(self): """Data type, float32 by default.""" return self.config_dict.get('dtype', 'float32') + @property + def protect_conditional_features(self) -> bool: + """Whether to protect conditional features from being dropped by preprocessing.""" + return self.config_dict.get('train', {}).get('protect_conditional_features', False) + def _data_preprocessor_params_to_steps(params: Dict[str, Any]) -> List[Dict[str, Any]]: """Translate old data_preprocessor params into a 'steps' specification. diff --git a/energy_fault_detector/fault_detector.py b/energy_fault_detector/fault_detector.py index 4706877..8c4c21b 100644 --- a/energy_fault_detector/fault_detector.py +++ b/energy_fault_detector/fault_detector.py @@ -39,42 +39,28 @@ def __init__(self, config: Optional[Config] = None, model_directory: str | Path def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Series, fit_preprocessor: bool = True ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]: - """ Preprocesses the training data using the configured data_preprocessor - - Args: - sensor_data (pd.DataFrame): unprocessed training data - normal_index (pd.Series): unprocessed normal index - fit_preprocessor (bool, optional): if True the preprocessor is fitted. If False the preprocessor is not - fitted and the user has to provide a ready-to-use preprocessor by loading models before calling this - function. - - Returns: tuple of (pd.Dataframe, pd.Dataframe, pd.Series) - x_prepped (pd.DataFrame): preprocessed normal training data - x: ordered training data (unprocessed) # needed for _fit_threshold - y: ordered normal_index (unprocessed) # needed for _fit_threshold - - """ + """Preprocesses the training data using the configured data_preprocessor.""" x = sensor_data.sort_index() if normal_index is not None: y = normal_index.sort_index() else: - # assume only 'normal behaviour' in x y = pd.Series(np.full(len(x), True), index=x.index) if not x.loc[x.index.duplicated()].empty or not y.loc[y.index.duplicated()].empty: raise ValueError('There are duplicated indices in the input dataframe `sensor_data` and/or in the ' '`normal_index`, please check your input data.') - # Get conditional features early to protect them from transformations - protected_features = self.autoencoder.conditional_features if self.autoencoder.is_conditional else [] + # Determine which features to protect based on config flag + protect = self.config.protect_conditional_features if self.config else True + protected_features = ( + self.autoencoder.conditional_features or [] + ) if protect else [] if self.config.data_clipping: logger.debug('Clip data before scaling.') - # Don't clip conditional features clipper_params = self.config.data_clipping_params.copy() if protected_features: - # Add conditional features to exclusion list existing_exclusions = clipper_params.get('features_to_exclude', []) clipper_params['features_to_exclude'] = list(set(existing_exclusions + protected_features)) logger.debug(f'Excluding conditional features from clipping: {protected_features}') @@ -82,22 +68,18 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri data_clipper.fit(x=x) x = data_clipper.transform(x) - x_normal = x[y.values] # filter normal before data prep + x_normal = x[y.values] if fit_preprocessor: logger.info('Fit preprocessor pipeline.') - # Build fit params for pipeline steps that support protected_features fit_params = {} for step_name in self.data_preprocessor.named_steps.keys(): if 'column_selector' in step_name or 'low_unique_value_filter' in step_name: fit_params[f'{step_name}__protected_features'] = protected_features - # Add global validation that protected features are in the output if protected_features: fit_params['protected_features'] = protected_features self.data_preprocessor.fit(x_normal, **fit_params) x_prepped = self.data_preprocessor.transform(x_normal) - - # Use float32 by default for performance, unless specified otherwise in config x_prepped = x_prepped.astype(self.config.dtype) return x_prepped, x, y @@ -105,25 +87,7 @@ def preprocess_train_data(self, sensor_data: pd.DataFrame, normal_index: pd.Seri def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_models: bool = True, overwrite_models: bool = False, fit_autoencoder_only: bool = False, fit_preprocessor: bool = True, **kwargs) -> ModelMetadata: - """Fit models on the given sensor_data and save them locally and return the metadata. - - Args: - sensor_data (pd.DataFrame): DataFrame with the sensor data of one asset for a specific time window. - The timestamp should be the index and the sensor values as columns. - normal_index (Optional[pd.Series]): Series indicating normal behavior as boolean with the timestamp as - index. - Optional; if not provided, assumes all sensor_data represents normal behavior. - save_models (bool, optional): Whether to save models. Defaults to True. - overwrite_models (bool, optional): If True, existing model directories can be overwritten. Defaults to - False. - fit_autoencoder_only (bool, optional): If True, only fit the data preprocessor and autoencoder objects. - Defaults to False. - fit_preprocessor (bool, optional): If True, the preprocessor is fitted. Defaults to True. - - Returns: - ModelMetadata: metadata of the trained model: model_date, model_path, model reconstruction errors - of the training and validation data. - """ + """Fit models on the given sensor_data and save them locally and return the metadata.""" try: from keras.backend import clear_session @@ -141,9 +105,31 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo raise ValueError(f"`sensor_data` must be numeric. Non-numeric columns: {non_numeric}") clear_session() - model_path = None # default value (will be overwritten by self._save if the models are saved). - x_prepped, x, y = self.preprocess_train_data(sensor_data=sensor_data, normal_index=normal_index, - fit_preprocessor=fit_preprocessor) + + # --- Resolve conditional features against available data --- + self._resolve_conditional_features(sensor_data) + + model_path = None + x_prepped, x, y = self.preprocess_train_data( + sensor_data=sensor_data, normal_index=normal_index, fit_preprocessor=fit_preprocessor + ) + + # Post-preprocessing check: conditionals may have been dropped + if not self.config.protect_conditional_features and self.autoencoder.is_conditional: + # Check if conditionals survived preprocessing + surviving = [ + f for f in (self.autoencoder.conditional_features or []) + if f in x_prepped.columns + ] + dropped_by_pipeline = set(self.autoencoder.conditional_features or []) - set(surviving) + if dropped_by_pipeline: + logger.warning(f"Conditional features dropped by preprocessing pipeline: " + f"{sorted(dropped_by_pipeline)}. Remaining: {surviving or 'none'}") + if surviving: + self.autoencoder.conditional_features = surviving + else: + self._fallback_if_no_conditionals() + train_recon_error, val_recon_error = None, None x_train, x_val = self.train_val_split(x_prepped) logger.info('Train autoencoder.') @@ -157,7 +143,6 @@ def fit(self, sensor_data: pd.DataFrame, normal_index: pd.Series = None, save_mo if not fit_autoencoder_only: self._fit_threshold(x=x, y=y, x_val=x_val, fit_on_validation=self.config.fit_threshold_on_val) - # save the models if save_models: model_path, model_date = self.save(overwrite=overwrite_models) else: @@ -316,6 +301,15 @@ def predict(self, sensor_data: pd.DataFrame, model_path: Optional[str] = None, if not x.loc[x.index.duplicated()].empty: raise ValueError('There are duplicated indices in the input dataframe `sensor_data`.') + # Validate conditional features at predict time + if self.autoencoder.is_conditional: + required = self.autoencoder.conditional_features or [] + missing_at_predict = [f for f in required if f not in x.columns] + if missing_at_predict: + raise ValueError(f"Conditional features required by the trained model are missing from " + f"sensor_data: {sorted(missing_at_predict)}. The model was trained with " + f"these features and cannot predict without them.") + x_prepped = self.data_preprocessor.transform(x).sort_index() x_prepped = x_prepped.astype(self.config.dtype) column_order = x_prepped.columns @@ -441,3 +435,62 @@ def _fit_threshold(self, x: pd.DataFrame, y: pd.Series, x_val: pd.DataFrame, fit normal_index=y.loc[scores.index]) else: self.threshold_selector.fit(x=scores, y=y.loc[scores.index]) + + def _resolve_conditional_features(self, sensor_data: pd.DataFrame) -> List[str]: + """Resolve which conditional features are actually available in the data. + + If all conditional features are missing and the autoencoder is a ConditionalAE, + falls back to MultilayerAutoencoder. For sequence models, simply clears the + conditional_features list (they handle None gracefully). + + Args: + sensor_data: Input DataFrame to check column availability. + + Returns: + List of available conditional feature names (may be empty). + """ + configured = self.autoencoder.conditional_features or [] + if not configured: + return [] + + available = [f for f in configured if f in sensor_data.columns] + missing = set(configured) - set(available) + + if missing: + logger.warning(f"Conditional features not found in sensor_data and will be ignored: " + f"{sorted(missing)}. Using: {available or 'none'}") + + if not available: + self._fallback_if_no_conditionals() + else: + self.autoencoder.conditional_features = available + self.autoencoder.is_conditional = True + + return available + + def _fallback_if_no_conditionals(self) -> None: + """Handle the case where all conditional features are unavailable. + + For ConditionalAE: replaces with MultilayerAutoencoder (same architecture params). + For sequence models: just clears conditional_features (they work without). + """ + from energy_fault_detector.autoencoders.conditional_autoencoder import ConditionalAE + from energy_fault_detector.autoencoders.multilayer_autoencoder import MultilayerAutoencoder + from energy_fault_detector.autoencoders.sequence_autoencoder import SequenceAutoencoder + + if isinstance(self.autoencoder, SequenceAutoencoder): + logger.warning("All conditional features are unavailable. " + "Sequence model will proceed without conditional inputs.") + self.autoencoder.conditional_features = None + self.autoencoder.is_conditional = False + + elif isinstance(self.autoencoder, ConditionalAE): + logger.warning("All conditional features are unavailable. " + "Falling back from ConditionalAE to MultilayerAutoencoder.") + ae_params = dict(self.config['train']['autoencoder'].get('params', {})) + ae_params.pop('conditional_features', None) + self.autoencoder = MultilayerAutoencoder(**ae_params) + + else: + self.autoencoder.conditional_features = None + self.autoencoder.is_conditional = False diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 4009c78..23b1973 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -38,7 +38,8 @@ def test_init(self): 'params': {'target_false_discovery_rate': 0.8}, 'fit_on_val': False}, 'data_splitter': {'train_block_size': 7, 'val_block_size': 3, 'type': 'BlockDataSplitter'}, - 'data_clipping': {'lower_percentile': 0.01, 'upper_percentile': 0.99} + 'data_clipping': {'lower_percentile': 0.01, 'upper_percentile': 0.99}, + 'protect_conditional_features': False, }) self.assertDictEqual(conf.config_dict['root_cause_analysis'], {'alpha': 0.8, diff --git a/tests/test_fault_detector.py b/tests/test_fault_detector.py index 6f1f91c..60c1d32 100644 --- a/tests/test_fault_detector.py +++ b/tests/test_fault_detector.py @@ -489,19 +489,18 @@ def test_calls_predict_when_reconstruction_is_none(self): mock_predict.assert_called_once() -class TestFaultDetectorConditionalFeatureProtection(unittest.TestCase): - """Test that FaultDetector protects conditional features from being dropped during preprocessing.""" +class TestFaultDetectorConditionalFeatureResolution(unittest.TestCase): + """Test conditional feature resolution, fallback, and predict-time validation.""" def setUp(self) -> None: self.config_path = os.path.join(PROJECT_ROOT, 'tests/test_data/test_conditional_ae_config.yaml') self.conf = Config(self.config_path) self.test_dir = tempfile.mkdtemp() - # Create sensor data where conditional feature is constant (would normally be dropped) np.random.seed(42) length = 100 self.sensor_data = pd.DataFrame({ - 'feature_a': [180] * length, # Constant conditional feature + 'feature_a': np.random.random(size=length), 'feature_b': np.random.random(size=length), 'feature_c': np.random.random(size=length), 'feature_d': np.random.random(size=length), @@ -511,21 +510,233 @@ def setUp(self) -> None: def tearDown(self) -> None: shutil.rmtree(self.test_dir) - def test_conditional_features_protected_from_dropping(self): - """Test that conditional features specified in the autoencoder are protected during fit.""" - # The config specifies ConditionalAE with 'feature_a' as conditional feature - # It also has LowUniqueValueFilter which would normally drop constant features - fault_detector = FaultDetector(config=self.conf, model_directory=self.test_dir) + def test_resolve_partial_missing_conditionals(self): + """When some conditional features are missing, only available ones are kept.""" + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + # Config has conditional_features: ['feature_a', 'feature_b'] + self.assertEqual(fd.autoencoder.conditional_features, ['feature_a', 'feature_b']) - # Fit the model - this should NOT drop the conditional feature despite it being constant - fault_detector.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) + # Drop feature_b from sensor data + sensor_data_partial = self.sensor_data.drop(columns=['feature_b']) - # Check that the conditional feature is still present after preprocessing - feature_names = fault_detector.data_preprocessor.get_feature_names_out() - self.assertIn('feature_a', feature_names, - "Conditional feature 'feature_a' should be protected from dropping") + available = fd._resolve_conditional_features(sensor_data_partial) - # Verify we can predict with data containing the conditional feature - result = fault_detector.predict(sensor_data=self.sensor_data) - self.assertIsNotNone(result) - self.assertEqual(len(result.predicted_anomalies), len(self.sensor_data)) + self.assertEqual(available, ['feature_a']) + self.assertEqual(fd.autoencoder.conditional_features, ['feature_a']) + self.assertTrue(fd.autoencoder.is_conditional) + + def test_resolve_all_missing_conditionals_fallback_to_multilayer(self): + """When ALL conditional features are missing, ConditionalAE falls back to MultilayerAutoencoder.""" + from energy_fault_detector.autoencoders import MultilayerAutoencoder + + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + self.assertIsInstance(fd.autoencoder, ConditionalAE) + + # Drop both conditional features + sensor_data_no_cond = self.sensor_data.drop(columns=['feature_a', 'feature_b']) + + available = fd._resolve_conditional_features(sensor_data_no_cond) + + self.assertEqual(available, []) + self.assertIsInstance(fd.autoencoder, MultilayerAutoencoder) + self.assertFalse(fd.autoencoder.is_conditional) + self.assertIsNone(fd.autoencoder.conditional_features) + + def test_fallback_preserves_architecture_params(self): + """Fallback MultilayerAutoencoder should have the same architecture params from config.""" + from energy_fault_detector.autoencoders import MultilayerAutoencoder + + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + ae_params = self.conf['train']['autoencoder'].get('params', {}) + + sensor_data_no_cond = self.sensor_data.drop(columns=['feature_a', 'feature_b']) + fd._resolve_conditional_features(sensor_data_no_cond) + + self.assertIsInstance(fd.autoencoder, MultilayerAutoencoder) + self.assertEqual(fd.autoencoder.code_size, ae_params.get('code_size', 10)) + self.assertEqual(fd.autoencoder.layers, ae_params.get('layers', [200])) + + def test_resolve_no_conditionals_configured_is_noop(self): + """When no conditional features are configured, resolution is a no-op.""" + config = Config(os.path.join(PROJECT_ROOT, 'tests/test_data/test_config.yaml')) + fd = FaultDetector(config=config, model_directory=self.test_dir) + + self.assertFalse(fd.autoencoder.is_conditional) + + available = fd._resolve_conditional_features(self.sensor_data) + + self.assertEqual(available, []) + self.assertFalse(fd.autoencoder.is_conditional) + + def test_resolve_all_present_no_change(self): + """When all conditional features are present, no change occurs.""" + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + + available = fd._resolve_conditional_features(self.sensor_data) + + self.assertEqual(available, ['feature_a', 'feature_b']) + self.assertIsInstance(fd.autoencoder, ConditionalAE) + self.assertTrue(fd.autoencoder.is_conditional) + + def test_predict_raises_on_missing_trained_conditionals(self): + """Predict raises ValueError when trained conditional features are missing.""" + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + + # Train with all features present + fd.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) + + # Predict without feature_a + sensor_data_missing = self.sensor_data.drop(columns=['feature_a']) + + with self.assertRaises(ValueError) as ctx: + fd.predict(sensor_data=sensor_data_missing) + + self.assertIn('feature_a', str(ctx.exception)) + self.assertIn('missing', str(ctx.exception).lower()) + + def test_fit_with_all_conditionals_missing_trains_successfully(self): + """Fit with all conditionals missing falls back and trains successfully.""" + from energy_fault_detector.autoencoders import MultilayerAutoencoder + + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + sensor_data_no_cond = self.sensor_data.drop(columns=['feature_a', 'feature_b']) + + result = fd.fit(sensor_data=sensor_data_no_cond, normal_index=self.normal_index, save_models=False) + + self.assertIsInstance(fd.autoencoder, MultilayerAutoencoder) + self.assertIsNotNone(result.train_recon_error) + + def test_fit_with_partial_conditionals_trains_successfully(self): + """Fit with some conditionals missing still trains with remaining ones.""" + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + sensor_data_partial = self.sensor_data.drop(columns=['feature_b']) + + result = fd.fit(sensor_data=sensor_data_partial, normal_index=self.normal_index, save_models=False) + + self.assertIsInstance(fd.autoencoder, ConditionalAE) + self.assertEqual(fd.autoencoder.conditional_features, ['feature_a']) + self.assertIsNotNone(result.train_recon_error) + + +class TestFaultDetectorProtectConditionalFeaturesFalse(unittest.TestCase): + """Test behavior when protect_conditional_features is False.""" + + def setUp(self) -> None: + self.config_path = os.path.join(PROJECT_ROOT, 'tests/test_data/test_conditional_ae_config.yaml') + self.conf = Config(self.config_path) + # Set protect_conditional_features to False + self.conf.config_dict['train']['protect_conditional_features'] = False + self.test_dir = tempfile.mkdtemp() + + np.random.seed(42) + length = 100 + # feature_a is constant → will be dropped by LowUniqueValueFilter when unprotected + self.sensor_data = pd.DataFrame({ + 'feature_a': [1.0] * length, + 'feature_b': np.random.random(size=length), + 'feature_c': np.random.random(size=length), + 'feature_d': np.random.random(size=length), + }) + self.normal_index = pd.Series([True] * 80 + [False] * 20) + + def tearDown(self) -> None: + shutil.rmtree(self.test_dir) + + def test_conditional_features_can_be_dropped_when_unprotected(self): + """When protect=False, pipeline can drop constant conditional features.""" + from energy_fault_detector.autoencoders import MultilayerAutoencoder + + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + self.assertFalse(fd.config.protect_conditional_features) + + result = fd.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) + + # feature_a was constant, should have been dropped, triggering fallback + # Since feature_b survived, autoencoder should still be conditional with just feature_b + # OR if feature_a was the only one dropped, we still have feature_b + if fd.autoencoder.is_conditional: + self.assertNotIn('feature_a', fd.autoencoder.conditional_features) + self.assertIn('feature_b', fd.autoencoder.conditional_features) + else: + # Both were dropped → fallback + self.assertIsInstance(fd.autoencoder, MultilayerAutoencoder) + + def test_protect_true_keeps_constant_conditional(self): + """When protect=True (default), constant conditional features are kept.""" + self.conf.config_dict['train']['protect_conditional_features'] = True + + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + fd.fit(sensor_data=self.sensor_data, normal_index=self.normal_index, save_models=False) + + feature_names = fd.data_preprocessor.get_feature_names_out() + self.assertIn('feature_a', feature_names) + self.assertIsInstance(fd.autoencoder, ConditionalAE) + + +class TestFaultDetectorSequenceConditionalFallback(unittest.TestCase): + """Test conditional feature fallback for sequence models.""" + + def setUp(self) -> None: + self.config_path = os.path.join(PROJECT_ROOT, 'tests/test_data/test_config_ts_freq.yaml') + self.conf = Config(self.config_path) + self.test_dir = tempfile.mkdtemp() + + n = 200 + index = pd.date_range("2025-01-01", periods=n, freq="30s") + np.random.seed(42) + self.sensor_data = pd.DataFrame( + np.random.randn(n, 3), index=index, columns=["f1", "f2", "f3"] + ) + self.normal_index = pd.Series(True, index=index) + + def tearDown(self) -> None: + shutil.rmtree(self.test_dir) + + def test_sequence_model_clears_missing_conditionals(self): + """Sequence model with missing conditionals just clears them (no class swap).""" + # Manually set conditional features on the sequence model + fd = FaultDetector(config=self.conf, model_directory=self.test_dir) + fd.autoencoder.conditional_features = ['nonexistent_feature'] + fd.autoencoder.is_conditional = True + + available = fd._resolve_conditional_features(self.sensor_data) + + self.assertEqual(available, []) + self.assertIsInstance(fd.autoencoder, LSTMSeq2OneAutoencoder) + self.assertIsNone(fd.autoencoder.conditional_features) + self.assertFalse(fd.autoencoder.is_conditional) + + +class TestProtectConditionalFeaturesConfigProperty(unittest.TestCase): + """Test the config property for protect_conditional_features.""" + + def test_default_is_false(self): + """Default value should be False when not specified.""" + config = Config(os.path.join(PROJECT_ROOT, 'tests/test_data/test_config.yaml')) + self.assertFalse(config.protect_conditional_features) + + def test_explicit_false(self): + """Explicit False in config should be respected.""" + config = Config(config_dict={ + 'train': { + 'protect_conditional_features': False, + 'data_preprocessor': {'steps': []}, + 'autoencoder': {'name': 'default', 'params': {'epochs': 1}}, + 'anomaly_score': {'name': 'rmse'}, + 'threshold_selector': {'name': 'quantile', 'params': {'quantile': 0.95}}, + } + }) + self.assertFalse(config.protect_conditional_features) + + def test_explicit_true(self): + """Explicit True in config should be respected.""" + config = Config(config_dict={ + 'train': { + 'protect_conditional_features': True, + 'data_preprocessor': {'steps': []}, + 'autoencoder': {'name': 'default', 'params': {'epochs': 1}}, + 'anomaly_score': {'name': 'rmse'}, + 'threshold_selector': {'name': 'quantile', 'params': {'quantile': 0.95}}, + } + }) + self.assertTrue(config.protect_conditional_features) From e7847d8bee6f0e9146b6237e80033c0219cf84cc Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:19:14 +0200 Subject: [PATCH 13/14] Add badges --- .github/workflows/run-tests.yml | 2 +- README.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7acd6a4..9bb3f40 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,4 +1,4 @@ -name: Python Test Workflow +name: Tests on: pull_request: diff --git a/README.md b/README.md index fdce7fd..5e0020b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,15 @@ # Energy Fault Detector - Autoencoder-based Fault Detection for the Future Energy System +[![Python](https://img.shields.io/pypi/pyversions/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/) +[![PyPI version](https://img.shields.io/pypi/v/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Tests](https://github.com/AEFDI/EnergyFaultDetector/actions/workflows/run-tests.yml/badge.svg)](https://github.com/AEFDI/EnergyFaultDetector/actions/workflows/run-tests.yml) +[![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://aefdi.github.io/EnergyFaultDetector/) +[![Downloads](https://img.shields.io/pypi/dm/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/) + +Unsupervised fault detection for renewable energy assets using autoencoder-based anomaly detection. + **Energy Fault Detector** is an open-source Python package designed for the automated detection of anomalies in operational data from renewable energy systems as well as power grids. It uses autoencoder-based normal behaviour models to identify irregularities in operational data. In addition to the classic anomaly detection, the package From 002b5224277ce251cede3d19e365886881fd5ffc Mon Sep 17 00:00:00 2001 From: croelofs <25582572+roelofsc@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:30:08 +0200 Subject: [PATCH 14/14] Add missing DataClipper parameters to the Config Schema. --- energy_fault_detector/config/config.py | 2 ++ energy_fault_detector/data_preprocessing/data_clipper.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/energy_fault_detector/config/config.py b/energy_fault_detector/config/config.py index b1fa312..8fb87e6 100644 --- a/energy_fault_detector/config/config.py +++ b/energy_fault_detector/config/config.py @@ -62,6 +62,8 @@ 'schema': { 'lower_percentile': {'type': 'float', 'required': False}, 'upper_percentile': {'type': 'float', 'required': False}, + 'features_to_exclude': {'type': 'list', 'required': False, 'schema': {'type': 'string'}}, + 'features_to_clip': {'type': 'list', 'required': False, 'schema': {'type': 'string'}}, } }, 'data_splitter': { diff --git a/energy_fault_detector/data_preprocessing/data_clipper.py b/energy_fault_detector/data_preprocessing/data_clipper.py index a338c18..e735d2d 100644 --- a/energy_fault_detector/data_preprocessing/data_clipper.py +++ b/energy_fault_detector/data_preprocessing/data_clipper.py @@ -31,8 +31,8 @@ class DataClipper(DataTransformer): data_clipping: lower_percentile: 0.001 upper_percentile: 0.999 - features_to_exclude: - - do_not_clip_this_feature + features_to_exclude: # or use features_to_clip + - do_not_clip_this_feature """ def __init__(self, lower_percentile: float = 0.01, upper_percentile: float = 0.99,