Add Python 3.12 & TF 2.16–2.18 support and fix preprocessing/tuning bugs

.github/workflows/run-tests.yml

@@ -1,4 +1,4 @@
-name: Python Test Workflow
+name: Tests
 
 on:
   pull_request:
@@ -7,17 +7,20 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Set up Python
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
@@ -31,11 +34,11 @@ jobs:
       - name: Upload coverage report
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-report
+          name: coverage-report-${{ matrix.python-version }}
           path: coverage.xml
 
       - name: Upload JUnit report
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report
+          name: junit-report-${{ matrix.python-version }}
           path: test_report.xml

.github/workflows/sync-to-gitlab.yml

@@ -5,14 +5,13 @@ on:
     branches:
     - main
     - develop
-    - time-series-handling
 
 jobs:
   sync:
     runs-on: ubuntu-latest
     name: Git Repo Sync
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - uses: wangchucheng/git-repo-sync@v0.1.0

.gitignore

@@ -6,6 +6,10 @@ __pycache__/
 .ipynb_checkpoints/
 .vscode
 
+# env
+.venv/
+.venv*/
+
 # Coverage etc.
 .coverage
 htmlcov/
@@ -32,6 +36,12 @@ test_report.xml
 *.pkl
 *.attrs
 *.model
+*.encoder
+*.decoder
+*.keras
+notebooks/PreDist/models/
+notebooks/PreDist/predist_data/
+*.csv
 
 # logging
 *.log

README.md

@@ -8,6 +8,15 @@
 
 # Energy Fault Detector - Autoencoder-based Fault Detection for the Future Energy System
 
+[![Python](https://img.shields.io/pypi/pyversions/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/)
+[![PyPI version](https://img.shields.io/pypi/v/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://github.com/AEFDI/EnergyFaultDetector/actions/workflows/run-tests.yml/badge.svg)](https://github.com/AEFDI/EnergyFaultDetector/actions/workflows/run-tests.yml)
+[![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://aefdi.github.io/EnergyFaultDetector/)
+[![Downloads](https://img.shields.io/pypi/dm/energy-fault-detector)](https://pypi.org/project/energy-fault-detector/)
+
+Unsupervised fault detection for renewable energy assets using autoencoder-based anomaly detection.
+
 **Energy Fault Detector** is an open-source Python package designed for the automated detection of anomalies in
 operational data from renewable energy systems as well as power grids. It uses autoencoder-based normal behaviour
 models to identify irregularities in operational data. In addition to the classic anomaly detection, the package 

docs/quick_fault_detection.rst

@@ -123,7 +123,7 @@ The underlying helper functions are implemented in:
 
 - :mod:`energy_fault_detector.quick_fault_detection.data_loading`
 - :mod:`energy_fault_detector.quick_fault_detection.configuration`
-- :mod:`energy_fault_detector.quick_fault_detection.quick_fault_detector`
+- :mod:`energy_fault_detector.quick_fault_detection.pipeline`
 
 Output
 ------

energy_fault_detector/anomaly_scores/rmse_score.py

@@ -70,16 +70,17 @@ def transform(self, x: DataType) -> pd.Series:
 
         check_is_fitted(self)
 
+        x_ = x
         if self.scale:
             # standardization of the reconstruction error in X
             if np.all(self.std_x_ > 0):
-                x = (x - self.mean_x_) / self.std_x_
+                x_ = (x - self.mean_x_) / self.std_x_
             else:
-                x = x - self.mean_x_
-                # replace possible inf values with 0
-            x[np.isinf(x)] = 0
+                x_ = x - self.mean_x_
+            # replace possible inf values with 0
+            x_[np.isinf(x_)] = 0
 
-        scores = np.sqrt(np.mean(x ** 2, axis=1))
+        scores = np.sqrt(np.mean(x_ ** 2, axis=1))
         if isinstance(x, (pd.DataFrame, pd.Series)):
             scores = pd.Series(scores, index=x.index)
 

energy_fault_detector/autoencoders/bidirectional_lstm_seq2one_autoencoder.py

@@ -2,9 +2,8 @@
 
 from typing import List, Optional, Tuple
 
-import tensorflow as tf
-from tensorflow.keras import regularizers
-from tensorflow.keras.layers import (
+from keras import regularizers
+from keras.layers import (
     Bidirectional,
     Concatenate,
     Dense,
@@ -13,7 +12,7 @@
     LSTM,
     RepeatVector,
 )
-from tensorflow.keras.models import Model as KerasModel
+from keras.models import Model as KerasModel
 
 from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder
 from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder
@@ -153,13 +152,13 @@ def create_model(
         encoded = Dropout(rate=self.dropout_rate)(encoded)
 
         if conditional_input is not None:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=encoded,
                 name="encoder",
             )
         else:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=main_input,
                 outputs=encoded,
                 name="encoder",
@@ -182,13 +181,13 @@ def create_model(
         )(last_timestep)
 
         if conditional_input is not None:
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=reconstruction,
                 name="bidirectional_lstm_seq2one_autoencoder",
             )
         else:
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=main_input,
                 outputs=reconstruction,
                 name="bidirectional_lstm_seq2one_autoencoder",

energy_fault_detector/autoencoders/cnn_seq2one_autoencoder.py

@@ -4,8 +4,7 @@
 import logging
 
 import numpy as np
-import tensorflow as tf
-from tensorflow.keras.layers import (
+from keras.layers import (
     Input,
     Conv1D,
     BatchNormalization,
@@ -15,7 +14,7 @@
     Flatten,
     MaxPooling1D,
 )
-from tensorflow.keras.models import Model as KerasModel
+from keras.models import Model as KerasModel
 
 from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder
 from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder
@@ -161,13 +160,13 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op
 
         # Encoder model for latent representation
         if conditional_input is not None:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=encoded,
                 name="encoder",
             )
         else:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=main_input,
                 outputs=encoded,
                 name="encoder",
@@ -193,13 +192,13 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op
 
         # Stand-alone decoder model
         if conditional_input is not None:
-            self.decoder = tf.keras.Model(
+            self.decoder = KerasModel(
                 inputs=[latent_input, cond_last_input],
                 outputs=reconstruction,
                 name="decoder",
             )
         else:
-            self.decoder = tf.keras.Model(
+            self.decoder = KerasModel(
                 inputs=latent_input,
                 outputs=reconstruction,
                 name="decoder",
@@ -210,15 +209,15 @@ def create_model(self, input_dimension: Tuple[int, int], condition_dimension: Op
             encoded = self.encoder(inputs=[main_input, conditional_input])
             cond_last = conditional_input[:, -1, :]
             decoded = self.decoder([encoded, cond_last])
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=decoded,
                 name="cnn_seq2one_autoencoder",
             )
         else:
             encoded = self.encoder(main_input)
             decoded = self.decoder(encoded)
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=main_input,
                 outputs=decoded,
                 name="cnn_seq2one_autoencoder",

energy_fault_detector/autoencoders/cnn_seq_autoencoder.py

@@ -2,15 +2,15 @@
 
 from typing import List, Optional, Tuple
 
-from tensorflow.keras.layers import (
+from keras.layers import (
     Input,
     Conv1D,
     Conv1DTranspose,
     BatchNormalization,
     Dropout,
     Concatenate
 )
-from tensorflow.keras.models import Model as KerasModel
+from keras.models import Model as KerasModel
 
 from .seq2seq_autoencoder import Seq2SeqAutoencoder
 from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder

energy_fault_detector/autoencoders/conditional_autoencoder.py

@@ -2,8 +2,8 @@
 
 from typing import List, Optional
 
-from tensorflow.keras.models import Model as KerasModel
-from tensorflow.keras.layers import Dense, PReLU, Input, Concatenate
+from keras.models import Model as KerasModel
+from keras.layers import Dense, PReLU, Input, Concatenate
 
 from energy_fault_detector.core.autoencoder import Autoencoder
 

energy_fault_detector/autoencoders/lstm_seq2one_autoencoder.py

@@ -2,16 +2,15 @@
 
 from typing import List, Optional, Tuple
 
-import tensorflow as tf
-from tensorflow.keras import regularizers
-from tensorflow.keras.layers import (
+from keras import regularizers
+from keras.layers import (
     Input,
     LSTM,
     Dropout,
     Dense,
     Concatenate,
 )
-from tensorflow.keras.models import Model as KerasModel
+from keras.models import Model as KerasModel
 
 from energy_fault_detector.autoencoders.seq2one_autoencoder import Seq2OneAutoencoder
 from energy_fault_detector.data_splitting.sequence_dataset import SequenceDatasetBuilder
@@ -144,13 +143,13 @@ def create_model(
 
         # Encoder model for latent representation
         if conditional_input is not None:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=encoded,
                 name="encoder",
             )
         else:
-            self.encoder = tf.keras.Model(
+            self.encoder = KerasModel(
                 inputs=main_input,
                 outputs=encoded,
                 name="encoder",
@@ -173,13 +172,13 @@ def create_model(
 
         # Stand-alone decoder model
         if conditional_input is not None:
-            self.decoder = tf.keras.Model(
+            self.decoder = KerasModel(
                 inputs=[latent_input, cond_last_input],
                 outputs=reconstruction,
                 name="decoder",
             )
         else:
-            self.decoder = tf.keras.Model(
+            self.decoder = KerasModel(
                 inputs=latent_input,
                 outputs=reconstruction,
                 name="decoder",
@@ -189,14 +188,14 @@ def create_model(
             enc = self.encoder(inputs=[main_input, conditional_input])
             cond_last = conditional_input[:, -1, :]
             decoded = self.decoder([enc, cond_last])
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=[main_input, conditional_input],
                 outputs=decoded,
             )
         else:
             enc = self.encoder(main_input)
             decoded = self.decoder(enc)
-            self.model = tf.keras.Model(
+            self.model = KerasModel(
                 inputs=main_input,
                 outputs=decoded,
             )