diff --git a/.gitignore b/.gitignore index 1d40201..ef7fcd8 100644 --- a/.gitignore +++ b/.gitignore @@ -212,4 +212,7 @@ lips/tests/data/powergrid/l2rpn_idf_2023 logs.log lips_logs.log -trained_models/ \ No newline at end of file +trained_models/ +/draft.py +/score_scripts/NeurIPS_scoreV2_8/ +/score_scripts/PowerGrid_score/ diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini new file mode 100644 index 0000000..6c4fcfb --- /dev/null +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -0,0 +1,100 @@ +[DEFAULT] +ValueByColor = {"green": 2, "orange": 1, "red": 0} + +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.25, "Accuracy": 0.75} + +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}} +[AirfoilCompetition] +ValueByColor = {"green": 2, "orange": 1, "red": 0} + +Coefficients = {"ML": {"value": 0.4, "Accuracy": {"value": 0.75}, "SpeedUP": {"value": 0.25}}, + "OOD": {"value": 0.3, "Accuracy": {"value": 0.75}, "SpeedUP": {"value": 0.25}}, + "Physics": {"value": 0.3}} + +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}} + +[ML4PhysimCompetition] +ValueByColor = {"green": 2, "orange": 1, "red": 0} + +Coefficients = {"ID": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "OOD": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "SpeedUP": {"value": 0.4}} +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [32.79]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [50]}} \ No newline at end of file diff --git a/getting_started/PowerGridUsecase/05_Scoring.ipynb b/getting_started/PowerGridUsecase/05_Scoring.ipynb new file mode 100644 index 0000000..b1438b6 --- /dev/null +++ b/getting_started/PowerGridUsecase/05_Scoring.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5648efb1-f90f-4261-a1cc-dba902b1b173", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-10T09:47:19.828698Z", + "start_time": "2025-03-10T09:47:19.823375Z" + } + }, + "source": [ + "# Getting Started with the Scoring Feature" + ] + }, + { + "cell_type": "markdown", + "id": "4da8963b9847353", + "metadata": {}, + "source": [ + "This notebook demonstrates how to use the scoring feature from the LIPS package. We will cover the following topics:\n", + "\n", + "- **Defining Scoring Based on Configuration File**: We will show how to define scoring configurations and explain the structure of the configuration file.\n", + "- **Application to Competitions**: We will apply the scoring feature to two competitions: the airfoil competition and the powergrid competition." + ] + }, + { + "cell_type": "markdown", + "id": "35507c90-988e-4954-8351-9655d172503f", + "metadata": {}, + "source": [ + "## 1. Configuration file" + ] + }, + { + "cell_type": "markdown", + "id": "ee8cad8d-560d-4741-9274-038dbc2d86e4", + "metadata": {}, + "source": [ + "The core configuration for scoring is located in **configurations/powergrid/scoring/ScoreConfig.ini**. This file defines the scoring logic and allows for flexible customization of your score evaluation process. Three key sections are essential within this configuration: Thresholds, ValueByColor, and Coefficients." + ] + }, + { + "cell_type": "markdown", + "id": "4a9daea4-1b29-4d96-9bd8-7879cd561f54", + "metadata": {}, + "source": [ + "### 1. Thresholds: Defining Performance Boundaries\n", + "\n", + "The `Thresholds` section specifies the performance benchmarks for individual metrics. It includes both the threshold values and the comparison type, indicating whether to minimize, maximize, or evaluate a ratio.\n", + "\n", + "* **Comparison Types:**\n", + " * `minimize`: Lower values are considered better.\n", + " * `maximize`: Higher values are considered better.\n", + " * `ratio`: used to compare two values, if the ratio is under or above the thresholds.\n", + "* **Example:**\n", + " ```ini\n", + " \"a_or\": {\"comparison_type\": \"minimize\", \"thresholds\": [0.02, 0.05]}\n", + " ```\n", + " * **Explanation:** This entry defines thresholds for the metric \"a_or\". If the \"a_or\" value is less than 0.02, it falls within the best (green) range. If it's between 0.02 and 0.05, it falls within the middle (orange) range. If it's greater than 0.05, it falls within the worst (red) range. Because the comparison type is minimize, the lower the a_or value is the better.\n", + "\n", + "### 2. ValueByColor: Assigning Scores to Performance Ranges\n", + "\n", + "The `ValueByColor` section maps performance ranges (represented by colors) to numerical scores. This allows you to quantify the qualitative assessment of your metrics.\n", + "\n", + "* **Example:**\n", + " ```ini\n", + " {\"green\": 2, \"orange\": 1, \"red\": 0}\n", + " ```\n", + " * **Explanation:** In this example, a metric falling within the \"green\" range receives a score of 2, \"orange\" receives 1, and \"red\" receives 0.\n", + "\n", + "### 3. Coefficients: Weighting Metric Contributions\n", + "\n", + "The `Coefficients` section defines the relative importance of different metrics and sub-metrics in the overall score. This allows you to prioritize certain aspects of your model's performance.\n", + "\n", + "* **Hierarchical Structure:** Coefficients can be organized hierarchically, allowing you to assign weights to both top-level metrics (e.g., \"ML\", \"OOD\", \"Physics\") and their sub-metrics (e.g., \"Accuracy\", \"SpeedUP\").\n", + "* **Example:**\n", + " ```ini\n", + " {\"ML\": {\"value\": 0.4, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}},\n", + " \"OOD\": {\"value\": 0.3, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}},\n", + " \"Physics\": {\"value\": 0.3}}\n", + " ```\n", + " * **Explanation:**\n", + " * The \"ML\" metric contributes 40% (0.4) to the overall score.\n", + " * Within \"ML\", \"Accuracy\" contributes 75% (0.75) and \"SpeedUP\" contributes 25% (0.25) to the \"ML\" sub-score.\n", + " * The \"OOD\" metric contributes 30% to the overall score, and also uses the same accuracy and speedup sub metric weights.\n", + " * The \"Physics\" metric contributes 30% to the overall score, and has no submetrics.\n", + " * This structure allows for fine-grained control over the scoring process, ensuring that the final score reflects the relative importance of different aspects of your model's performance.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1d9eda4-5953-4fda-b3fd-a8c73bab433d", + "metadata": {}, + "source": [ + "## 2. Metric format" + ] + }, + { + "cell_type": "markdown", + "id": "94f32854-1470-4274-b5b0-dbfdb3715cef", + "metadata": {}, + "source": [ + " The scoring feature expects metrics data in a nested dictionary format. The structure should represent a tree-like hierarchy, where each node\n", + " contains either sub-metrics or leaf metrics. Leaf metrics are the actual values that will be colorized and scored.\n", + "\n", + " Here's an example of the expected format:\n", + "\n", + " ```json\n", + " {\n", + " \"ML\": {\n", + " \"metric1\": 0.85,\n", + " \"metric2\": 0.20\n", + " },\n", + " \"OOD\": {\n", + " \"metric3\": 0.92,\n", + " \"metric4\": 0.15\n", + " },\n", + " \"Physics\": {\n", + " \"metric5\": 0.78,\n", + " \"metric6\": 0.30\n", + " }\n", + " }\n", + " ```\n", + "\n", + " In this example:\n", + " - The top-level keys (`ML`, `OOD`, `Physics`) represent different categories or components.\n", + " - Each category contains several metrics (e.g., `metric1`, `metric2`).\n", + " - The values associated with each metric are numerical values.\n", + "\n", + " The metrics can also be structured in deeper hierarchies:\n", + "\n", + " ```json\n", + " {\n", + " \"Category1\": {\n", + " \"SubCategory1\": {\n", + " \"metric1\": 0.75,\n", + " \"metric2\": 0.25\n", + " },\n", + " \"SubCategory2\": {\n", + " \"metric3\": 0.60,\n", + " \"metric4\": 0.40\n", + " }\n", + " },\n", + " \"Category2\": {\n", + " \"metric5\": 0.90,\n", + " \"metric6\": 0.10\n", + " }\n", + " }\n", + " ```\n", + "\n", + " It's important to maintain a consistent branching structure throughout the metrics data. This means that if one sub-category contains further nested\n", + " categories, all other sub-categories at the same level should also have the same structure.\n", + "\n", + " The keys of the metrics should match the keys defined in the configuration file. For example, if the configuration file defines thresholds for `metric1`, the metrics data must also contain `metric1`." + ] + }, + { + "cell_type": "markdown", + "id": "9fd4161c-5ed3-48af-b2ee-9e1fe7e7725a", + "metadata": {}, + "source": [ + "## End-to-End Scoring Example\n", + "This section demonstrates an end-to-end example of how to use the scoring feature. We will load a configuration file, read a metrics file, colorize the metrics, calculate sub-scores, and then calculate the global score." + ] + }, + { + "cell_type": "markdown", + "id": "233f97d0-f68c-4fea-88c1-e67b0583c2a3", + "metadata": {}, + "source": [ + "### 1. Define Example Configuration File" + ] + }, + { + "cell_type": "markdown", + "id": "985c202e-dc1d-4307-8a81-1df47cee3d57", + "metadata": {}, + "source": [ + "### 1. Define Example Configuration File\n", + "\n", + "First, let's define an example configuration file (`config.ini`).\n", + "This file contains the thresholds, value_by_color mappings, and coefficients needed for scoring." + ] + }, + { + "cell_type": "markdown", + "id": "783fb53d-6f20-49cc-89fc-0fc028ae6145", + "metadata": {}, + "source": [ + "```ini\n", + "[thresholds]\n", + "a_or = {\"comparison_type\": \"minimize\", \"thresholds\": [0.02, 0.05]}\n", + "spearman_correlation_drag = {\"comparison_type\": \"maximize\", \"thresholds\": [0.7, 0.9]}\n", + "inference_time = {\"comparison_type\": \"minimize\", \"thresholds\": [500, 700]}\n", + "[valuebycolor]\n", + "green = 2\n", + "orange = 1\n", + "red = 0\n", + "[coefficients]\n", + "ML = {\"value\": 0.4, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}}\n", + "OOD = {\"value\": 0.3, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}}\n", + "Physics = {\"value\": 0.3}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5dc2a889-8f22-4a4c-838c-9cf56c3fde2e", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#%% [markdown]\n", + "# ### 2. Define Example Metrics File\n", + "#\n", + "# Next, let's define an example metrics file (`metrics.json`).\n", + "# This file contains the metric values that we want to score.\n", + "#\n", + "# ```json\n", + "# {\n", + "# \"ML\": {\n", + "# \"a_or\": 0.01,\n", + "# \"spearman_correlation_drag\": 0.95\n", + "# },\n", + "# \"OOD\": {\n", + "# \"inference_time\": 400\n", + "# },\n", + "# \"Physics\": {\n", + "# \"a_or\": 0.06,\n", + "# \"spearman_correlation_drag\": 0.75\n", + "# },\n", + "# \"Speed\": {\n", + "# \"inference_time\": 600\n", + "# }\n", + "# }\n", + "# ```\n", + "\n", + "#%%\n", + "import json\n", + "from scoring import Scoring\n", + "from utils import read_json\n", + "\n", + "# Initialize the Scoring class with the path to the configuration file\n", + "scoring = Scoring(config_path=\"config.ini\")\n", + "\n", + "# Load metrics data from the JSON file\n", + "metrics_path = \"metrics.json\"\n", + "metrics_data = read_json(json_path=metrics_path)\n", + "\n", + "#%% [markdown]\n", + "# ### 3. Colorize Metrics\n", + "#\n", + "# Now, let's colorize the metrics using the `colorize_metrics` function.\n", + "# This will convert the numerical metric values into color strings based on the\n", + "# thresholds defined in the configuration file.\n", + "\n", + "#%%\n", + "# Colorize the metrics data\n", + "colorized_metrics = scoring.colorize_metrics(metrics_data)\n", + "print(\"Colorized Metrics:\", json.dumps(colorized_metrics, indent=4))\n", + "\n", + "#%% [markdown]\n", + "# ### 4. Calculate Sub-Scores\n", + "#\n", + "# Next, let's calculate the sub-scores using the `calculate_sub_scores` function.\n", + "# This will calculate the score for each sub-tree in the metrics data.\n", + "\n", + "#%%\n", + "# Calculate sub-scores\n", + "sub_scores = scoring.calculate_sub_scores(colorized_metrics)\n", + "print(\"Sub-Scores:\", json.dumps(sub_scores, indent=4))\n", + "\n", + "#%% [markdown]\n", + "# ### 5. Calculate Global Score\n", + "#\n", + "# Finally, let's calculate the global score using the `calculate_global_score` function.\n", + "# This will calculate the overall score for the entire metrics tree.\n", + "\n", + "#%%\n", + "# Calculate global score\n", + "global_score = scoring.calculate_global_score(sub_scores)\n", + "print(\"Global Score:\", global_score)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py new file mode 100644 index 0000000..199bfad --- /dev/null +++ b/lips/scoring/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + +from __future__ import absolute_import + +from lips.scoring.scoring import Scoring +from lips.scoring.airfoil_powergrid_scoring import AirfoilPowerGridScoring +from lips.scoring.ml4physim_powergrid_socring import ML4PhysimPowerGridScoring + + +__all__ = [ + "Scoring", "AirfoilPowerGridScoring", "ML4PhysimPowerGridScoring" +] \ No newline at end of file diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py new file mode 100644 index 0000000..f119f98 --- /dev/null +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -0,0 +1,187 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + +import math +from typing import Union, Dict, List + +from .scoring import Scoring +from .utils import get_nested_value, filter_metrics, read_json +from ..config import ConfigManager + + +class AirfoilPowerGridScoring(Scoring): + """ + Calculates the score for the AirFoil Power Grid competition: https://www.codabench.org/competitions/3282/ + """ + + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the AirfoilPowerGridScoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ + super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) + + def _reconstruct_ml_metrics(self, raw_metrics: Dict, ml_key_path: List[str]) -> Dict: + """ + Construct ML metrics by retrieving and filtering data from the raw-JSON metrics. + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_key_path: List of keys representing the path to the ML metrics + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered ML metrics. + + Raises: + ValueError: If the specified path is invalid or ML metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + all_ml_metrics = get_nested_value(raw_metrics, ml_key_path) + if all_ml_metrics is None: + raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") + + if not isinstance(all_ml_metrics, dict): + raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") + + ml_metrics = {"ML": filter_metrics(all_ml_metrics, self.thresholds.keys())} + + pressure_surfacic_value_path = ml_key_path[:-1] + ["MSE_normalized_surfacic", "pressure"] + ml_metrics["ML"]["pressure_surfacic"] = get_nested_value(raw_metrics, pressure_surfacic_value_path) + + return ml_metrics + + def _reconstruct_physic_metrics(self, raw_metrics: Dict, physic_key_path: List[str]) -> Dict: + """ + Construct Physic metrics by retrieving and filtering data from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + physic_key_path: List of keys representing the path to the physics metrics + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered physics metrics. + + Raises: + ValueError: If the specified path is invalid or physics metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + all_physic_metrics = get_nested_value(raw_metrics, physic_key_path) + if all_physic_metrics is None: + raise ValueError(f"Invalid path {physic_key_path}. Could not retrieve Physic metrics.") + + if not isinstance(all_physic_metrics, dict): + raise TypeError(f"Expected a dictionary at {physic_key_path}, but got {type(all_physic_metrics).__name__}.") + + physic_metrics = {"Physics": filter_metrics(all_physic_metrics, self.thresholds.keys())} + return physic_metrics + + def _reconstruct_ood_metrics(self, raw_metrics: Dict, ml_ood_key_path: List[str], + physic_ood_key_path: List[str]) -> Dict: + """ + Construct OOD metrics by retrieving and Combining ML and Physic OOD-metrics from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_ood_key_path: Path to the ML OOD metrics. + physic_ood_key_path: Path to the Physics OOD metrics. + + Returns: + Dictionary containing the combined OOD metrics. + """ + ml_ood_metrics = self._reconstruct_ml_metrics(raw_metrics, ml_ood_key_path)["ML"] + physic_ood_metrics = self._reconstruct_physic_metrics(raw_metrics, physic_ood_key_path)["Physics"] + + return {"OOD": {**ml_ood_metrics, **physic_ood_metrics}} + + def compute_speed_score(self, time_inference: float) -> float: + """ + Computes the speed score based on: + + Score_Speed = min( (log10(SpeedUp) / log10(SpeedUpMax)), 1) + + Where : SpeedUp = time_ClassicalSolver / time_Inference + + Args: + time_inference: Inference time in seconds. + + Returns: + The speed score (between 0 and 1). + """ + + speed_up = self._calculate_speed_up(time_inference) + max_speed_ratio_allowed = self.thresholds["max_speed_ratio_allowed"]["thresholds"][0] + res = min((math.log10(speed_up) / math.log10(max_speed_ratio_allowed)), 1) + return max(res, 0) + + def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + """ + Computes the competition score based on the provided metrics in metrics_dict or metrics_path + + Args: + metrics_dict: Dictionary containing the raw metrics data. + metrics_path: Path to the JSON file containing the raw metrics data. + + Returns: + Dictionary containing the score colors, the score values, and the global score. + Raises: + ValueError: If both metrics_dict and metrics_path are None. + """ + + if metrics_dict is not None: + metrics = metrics_dict.copy() + elif metrics_path != "": + metrics = read_json(json_path=metrics_path, json_object=metrics_dict) + else: + raise ValueError("metrics_path and metrics_dict cant' both be None") + + time_inference = metrics["test_mean_simulation_time"] + + ml_metrics = self._reconstruct_ml_metrics(metrics, + ml_key_path=["fc_metrics_test", "test", "ML", "MSE_normalized"]) + physic_metrics = self._reconstruct_physic_metrics(metrics, + physic_key_path=["fc_metrics_test", "test", "Physics"]) + ood_metrics = self._reconstruct_ood_metrics(metrics, ml_ood_key_path=['fc_metrics_test_ood', 'test_ood', 'ML', + 'MSE_normalized'], + physic_ood_key_path=['fc_metrics_test_ood', 'test_ood', 'Physics']) + + metrics = {**ml_metrics, **physic_metrics, **ood_metrics} + + sub_scores_color = self.colorize_metrics(metrics) + sub_scores_values = self.calculate_sub_scores(sub_scores_color) + + speed_score = self.compute_speed_score(time_inference) + sub_scores_values["ML"] = {"Accuracy": sub_scores_values["ML"], "SpeedUP": speed_score} + sub_scores_values["OOD"] = {"Accuracy": sub_scores_values["OOD"], "SpeedUP": speed_score} + + global_score = self.calculate_global_score(sub_scores_values) + + return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} + + def _calculate_speed_up(self, time_inference: float) -> float: + """ + Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + Args: + time_inference: The inference time in seconds. + + Returns: + The calculated speedup factor. + """ + + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] + return time_classical_solver / time_inference \ No newline at end of file diff --git a/lips/scoring/ml4physim_powergrid_socring.py b/lips/scoring/ml4physim_powergrid_socring.py new file mode 100644 index 0000000..b7cda3b --- /dev/null +++ b/lips/scoring/ml4physim_powergrid_socring.py @@ -0,0 +1,200 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + +import math +from typing import Union, Dict, List + +from .scoring import Scoring +from .utils import get_nested_value, read_json +from ..config import ConfigManager + + +class ML4PhysimPowerGridScoring(Scoring): + """ + Calculates the score for the ML4Physim Power Grid competition: https://www.codabench.org/competitions/2378/ + """ + + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the ML4PhysimPowerGridScoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ + super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) + + def _reconstruct_ml_metrics(self, raw_metrics: Dict, ml_section_path: List[str]) -> Dict: + """ + Construct ML metrics by retrieving data from the raw-JSON metrics. + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_section_path: List of keys representing the path to the ML section + within the raw_metrics dictionary. + + Returns: + Dictionary containing the desired ML metrics. + + Raises: + ValueError: If the specified path is invalid or ML metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + ml_section = get_nested_value(raw_metrics, ml_section_path) + + if ml_section is None: + raise ValueError(f"Invalid path {ml_section_path}. Could not retrieve ML metrics.") + + if not isinstance(ml_section, dict): + raise TypeError(f"Expected a dictionary at {ml_section_path}, but got {type(ml_section).__name__}.") + + ml_metrics = {} + + ml_metrics["a_or"] = ml_section["MAPE_90_avg"]["a_or"] + ml_metrics["a_ex"] = ml_section["MAPE_90_avg"]["a_ex"] + ml_metrics["p_or"] = ml_section["MAPE_10_avg"]["p_or"] + ml_metrics["p_ex"] = ml_section["MAPE_10_avg"]["p_ex"] + ml_metrics["v_or"] = ml_section["MAE_avg"]["v_or"] + ml_metrics["v_ex"] = ml_section["MAE_avg"]["v_ex"] + + return {"ML": ml_metrics} + + def _reconstruct_physic_metrics(self, raw_metrics: Dict, physic_section_path: List[str]) -> Dict: + """ + Construct Physic metrics by retrieving and filtering data from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + physic_section_path: List of keys representing the path to the physics section + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered physics metrics. + + Raises: + ValueError: If the specified path is invalid or physics metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + physic_section = get_nested_value(raw_metrics, physic_section_path) + + if physic_section is None: + raise ValueError(f"Invalid path {physic_section_path}. Could not retrieve Physic metrics.") + + if not isinstance(physic_section, dict): + raise TypeError(f"Expected a dictionary at {physic_section_path}, but got {type(physic_section).__name__}.") + + physic_metrics = {} + + physic_metrics["CURRENT_POS"] = physic_section["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100. + physic_metrics["VOLTAGE_POS"] = physic_section["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100. + physic_metrics["LOSS_POS"] = physic_section["LOSS_POS"]["violation_proportion"] * 100. + physic_metrics["DISC_LINES"] = physic_section["DISC_LINES"]["violation_proportion"] * 100. + physic_metrics["CHECK_LOSS"] = physic_section["CHECK_LOSS"]["violation_percentage"] + physic_metrics["CHECK_GC"] = physic_section["CHECK_GC"]["violation_percentage"] + physic_metrics["CHECK_LC"] = physic_section["CHECK_LC"]["violation_percentage"] + physic_metrics["CHECK_JOULE_LAW"] = physic_section["CHECK_JOULE_LAW"]["violation_proportion"] * 100. + + return {"Physics": physic_metrics} + + def _reconstruct_ood_metrics(self, raw_metrics: Dict, ml_ood_section_path: List[str], + physic_ood_section_path: List[str]) -> Dict: + """ + Construct OOD metrics by retrieving and Combining ML and Physic OOD-metrics from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_ood_section_path: Path to the ML OOD section. + physic_ood_section_path: Path to the Physics OOD section. + + Returns: + Dictionary containing the combined OOD metrics. + """ + ml_ood_metrics = self._reconstruct_ml_metrics(raw_metrics, ml_ood_section_path) + physic_ood_metrics = self._reconstruct_physic_metrics(raw_metrics, physic_ood_section_path) + + return {"OOD": {**ml_ood_metrics, **physic_ood_metrics}} + + def compute_speed_score(self, time_inference: float) -> float: + """ + Computes the speed score based on: + + Score_Speed = min( weibull(SpeedUp), 1) + + Where : SpeedUp = time_ClassicalSolver / time_Inference + + Args: + time_inference: Inference time in seconds. + + Returns: + The speed score (between 0 and 1). + """ + speed_up = self._calculate_speed_up(time_inference) + res = min(self._weibull(5, 1.7, speed_up), 1) + return max(res, 0) + + def _weibull(self, c, b, x): + a = c * ((-math.log(0.9)) ** (-1 / b)) + return 1. - math.exp(-(x / a) ** b) + + def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + """ + Computes the competition score based on the provided metrics in metrics_dict or metrics_path + + Args: + metrics_dict: Dictionary containing the raw metrics data. + metrics_path: Path to the JSON file containing the raw metrics data. + + Returns: + Dictionary containing the score colors, the score values, and the global score. + Raises: + ValueError: If both metrics_dict and metrics_path are None. + """ + + if metrics_dict is not None: + metrics = metrics_dict.copy() + elif metrics_path != "": + metrics = read_json(json_path=metrics_path, json_object=metrics_dict) + else: + raise ValueError("metrics_path and metrics_dict cant' both be None") + + time_inference = metrics["test"]["ML"]["TIME_INF"] + + ml_metrics = self._reconstruct_ml_metrics(metrics, ["test", "ML"]) + physic_metrics = self._reconstruct_physic_metrics(metrics, ["test", "Physics"]) + ood_metrics = self._reconstruct_ood_metrics(metrics, ["test_ood_topo", "ML"], ["test_ood_topo", "Physics"]) + + metrics = {"ID": {**ml_metrics, **physic_metrics}, **ood_metrics} + sub_scores_color = self.colorize_metrics(metrics) + + sub_scores_values = self.calculate_sub_scores(sub_scores_color) + + speed_score = self.compute_speed_score(time_inference) + sub_scores_values["SpeedUP"] = speed_score + + global_score = self.calculate_global_score(sub_scores_values) + + return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} + + def _calculate_speed_up(self, time_inference: float) -> float: + """ + Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + Args: + time_inference: The inference time in seconds. + + Returns: + The calculated speedup factor. + """ + + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] + return time_classical_solver / time_inference \ No newline at end of file diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py new file mode 100644 index 0000000..a919213 --- /dev/null +++ b/lips/scoring/scoring.py @@ -0,0 +1,188 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + + +import bisect +from abc import ABC +from typing import Union, List, Dict, Any + +from ..config import ConfigManager +from ..logger import CustomLogger + +# Constants +VALID_COMPARISONS = {"minimize", "maximize"} + + +class Scoring(ABC): + """ + Base class for calculating scores based on metrics and thresholds. + """ + + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the Scoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ + self.config = config if config else ConfigManager(section_name=config_section, path=config_path) + self.logger = CustomLogger(__class__.__name__, log_path).logger + + self.thresholds = self.config.get_option("thresholds") + self.value_by_color = self.config.get_option("valuebycolor") + self.coefficients = self.config.get_option("coefficients") + self._validate_configuration() + + def colorize_metrics(self, metrics: Dict[str, Any]) -> Dict[str, Any]: + """ + Recursively colorizes metric values based on thresholds. + + Args: + metrics: A dictionary of metric names and their corresponding values (can be nested). + + Returns: + A dictionary with the same structure as `metrics` but with colorized values. + """ + colorized_metrics = {} + for key, value in metrics.items(): + if isinstance(value, dict): + colorized_metrics[key] = self.colorize_metrics(value) + else: + colorized_metrics[key] = self._colorize_metric_value(key, value) + return colorized_metrics + + def _colorize_metric_value(self, metric_name: str, metric_value: float) -> str: + """ + Assigns a color to a single metric value based on its threshold. + + Args: + metric_name: The name of the metric. + metric_value: The value of the metric. + + Returns: + The color corresponding to the metric value. + + Raises: + ValueError: If the comparison type is invalid. + """ + threshold_data = self.thresholds[metric_name] + comparison_type = threshold_data["comparison_type"] + thresholds = threshold_data["thresholds"] + + if comparison_type not in VALID_COMPARISONS: + raise ValueError(f"Invalid comparison type: {comparison_type}. Must be 'minimize' or 'maximize'.") + + index = bisect.bisect_left(thresholds, metric_value) + colors = list(self.value_by_color.keys()) + return colors[index] if comparison_type == "minimize" else colors[-(index + 1)] + + def _validate_configuration(self) -> None: + """ + Validates the thresholds and value_by_color configurations. + + Raises: + ValueError: If the configuration is invalid. + """ + if not self.thresholds: + raise ValueError("Thresholds configuration is missing.") + if not self.value_by_color: + raise ValueError("Value by color configuration is missing.") + + expected_threshold_count = len(self.value_by_color) - 1 + for metric_name, threshold_data in self.thresholds.items(): + if not isinstance(threshold_data, + dict) or "thresholds" not in threshold_data or "comparison_type" not in threshold_data: + raise ValueError( + f"Invalid thresholds data for metric '{metric_name}'. Must be a dict with 'thresholds' and 'comparison_type' keys.") + if (threshold_data["comparison_type"] in VALID_COMPARISONS) and ( + len(threshold_data["thresholds"]) != expected_threshold_count): + raise ValueError( + f"Metric '{metric_name}': Thresholds count must be {expected_threshold_count} (length of ValueByColor - 1).") + + def _calculate_leaf_score(self, colors: List[str]) -> float: + """ + Calculates the score for a leaf node (set of colorized metrics). + + Args: + colors: A list of color strings representing the colorized metrics. + + Returns: + The calculated score for the leaf node. + """ + return sum(self.value_by_color[color] for color in colors) / (len(colors) * max(self.value_by_color.values())) + + def calculate_sub_scores(self, node: Dict[str, Any]) -> Union[float, Dict[str, Any]]: + """ + Calculates sub-scores recursively for a node in the metrics tree. + + Args: + node: A node in the metrics tree (can be a leaf or a sub-tree). + + Returns: + The sub-score for the node (float for leaf, dict for sub-tree). + + Raises: + ValueError: If the input JSON is not a dictionary or if a parent node is inconsistently branched. + """ + if not isinstance(node, dict): + raise ValueError("Input must be a dictionary.") + + if all(isinstance(value, str) for value in node.values()): # Leaf node + return self._calculate_leaf_score(list(node.values())) + elif any(isinstance(value, str) for value in node.values()): # Inconsistent branching + raise ValueError("Parent node is not uniformly branched (mix of leaf and sub-tree children).") + else: # Sub-tree node + return {key: self.calculate_sub_scores(value) for key, value in node.items()} + + def calculate_global_score(self, tree: Union[float, Dict[str, Any]], key_path: List[str] = None) -> float: + """ + Calculates the global score for the entire metrics tree. + + Args: + tree: a pre-calculated sub-score tree + key_path: the path to the current node in the tree + + Returns: + The global score. + + """ + + if isinstance(tree, (int, float)): # Base case: already a sub-score + return tree + + key_path = key_path or [] + global_score = 0 + for key, subtree in tree.items(): + new_path = key_path + [key] + + weight = self._get_coefficient(new_path) or 1 # Default weight is 1 + global_score += weight * self.calculate_global_score(subtree, new_path) + + return global_score + + def _get_coefficient(self, key_path: List[str]) -> Union[float, None]: + """ + Retrieves the coefficient value from a nested dictionary based on a given path. + Args: + key_path: A list of keys representing the path to the desired coefficient. + + Returns: + The coefficient value if found, otherwise None. + """ + current = self.coefficients + for key in key_path: + if isinstance(current, dict) and key in current: + current = current[key] + else: + self.logger.warning(f"Coefficient not found for path: {' -> '.join(key_path)}. Using default value 1.") + return None + return current.get("value") diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py new file mode 100644 index 0000000..915bad6 --- /dev/null +++ b/lips/scoring/utils.py @@ -0,0 +1,83 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + +import json +import logging +from typing import Union, Dict, Any, List + + +def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None) -> Any: + """Reads a JSON file from the specified path or a JSON object. + + Args: + json_path: Path to the JSON file. If empty, `json_object` is used. + json_object: A JSON object (as a dict or a JSON string). + Used if `json_path` is empty. + + Returns: + Parsed JSON data as a Python object. + + Raises: + ValueError: If both `json_path` and `json_object` are empty or + if `json_object` is not a valid type. + FileNotFoundError: If the file specified by `json_path` + does not exist. + json.JSONDecodeError: If the JSON data is invalid. + """ + if json_path: + try: + with open(json_path, "r") as file: + return json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"JSON file not found at path: {json_path}") from e + except json.JSONDecodeError as e: + raise json.JSONDecodeError(f"Invalid JSON format in file: {json_path}", e.doc, e.pos) from e + elif json_object: + if isinstance(json_object, str): + try: + return json.loads(json_object) + except json.JSONDecodeError as e: + raise json.JSONDecodeError("Invalid JSON string provided.", e.doc, e.pos) from e + elif isinstance(json_object, dict): + return json_object + else: + raise ValueError("`json_object` must be a valid JSON string or dictionary.") + else: + raise ValueError("Both `json_path` and `json_object` are empty. Provide at least one.") + + +def get_nested_value(data: Dict, keys: List[str]) -> Any: + """Retrieves a nested value from a dictionary using a list of keys. + + Args: + data: The dictionary to retrieve the value from. + keys: A list of keys representing the path to the nested value. + + Returns: + The nested value if found, otherwise None. + """ + for key in keys: + if isinstance(data, dict) and key in data: + data = data[key] + else: + logging.warning(f"Path '{keys}' not found in data. Returning None.") + return None + return data + + +def filter_metrics(data: Dict, metrics: List[str]) -> Dict: + """Filters a dictionary to include only specified keys. + + Args: + data: The dictionary to filter. + metrics: A list of keys to keep in the filtered dictionary. + + Returns: + A new dictionary containing only the specified keys and their values. + """ + return {key: value for key, value in data.items() if key in metrics} \ No newline at end of file diff --git a/lips/tests/scoring/test_airfoil_powergrid_scoring.py b/lips/tests/scoring/test_airfoil_powergrid_scoring.py new file mode 100644 index 0000000..c9baf74 --- /dev/null +++ b/lips/tests/scoring/test_airfoil_powergrid_scoring.py @@ -0,0 +1,108 @@ +import json +import unittest +from unittest.mock import MagicMock, patch, mock_open + +from lips.scoring import AirfoilPowerGridScoring + + +class TestAirfoilPowerGridScoring(unittest.TestCase): + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "pressure": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + "coefficients": {"ML": {"value": 0.3, "Accuracy": {"value": 0.75}, "Speed": {"value": 0.25}}, + "OOD": {"value": 0.3, "Accuracy": {"value": 0.75}, "Speed": {"value": 0.25}}, + "Physics": {"value": 0.3}}}[key] + self.scoring = AirfoilPowerGridScoring(config=self.mock_config) + + def test__reconstruct_ml_metrics(self): + raw_metrics = {"fc_metrics_test": {"test": { + "ML": {"MSE_normalized": {"a_or": 0.4, "spearman_correlation_drag": 0.6}, + "MSE_normalized_surfacic": {"pressure": 0.2}}}}} + ml_key_path = ["fc_metrics_test", "test", "ML", "MSE_normalized"] + ml_metrics = self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + self.assertEqual(ml_metrics["ML"]["a_or"], 0.4) + self.assertEqual(ml_metrics["ML"]["pressure_surfacic"], 0.2) + + def test__reconstruct_ml_metrics_invalid_path(self): + raw_metrics = {} + ml_key_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + + def test__reconstruct_ml_metrics_invalid_type(self): + raw_metrics = {"fc_metrics_test": {"test": {"ML": {"MSE_normalized": "not a dict"}}}} + ml_key_path = ["fc_metrics_test", "test", "ML", "MSE_normalized"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + + def test__reconstruct_physic_metrics(self): + raw_metrics = {"fc_metrics_test": { + "test": {"Physics": {"spearman_correlation_drag": 0.7, "spearman_correlation_lift": 0.8}}}} + physic_key_path = ["fc_metrics_test", "test", "Physics"] + physic_metrics = self.scoring._reconstruct_physic_metrics(raw_metrics, physic_key_path) + self.assertEqual(physic_metrics["Physics"]["spearman_correlation_drag"], 0.7) + + def test__reconstruct_ood_metrics(self): + raw_metrics = {"fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}}, "Physics": {"spearman_correlation_drag": 0.9}}}, + "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"some_metric": 0.4}}, "Physics": {"some_metric": 0.7}}}} + ml_ood_key_path = ["fc_metrics_test_ood", "test_ood", "ML", "MSE_normalized"] + physic_ood_key_path = ["fc_metrics_test_ood", "test_ood", "Physics"] + ood_metrics = self.scoring._reconstruct_ood_metrics(raw_metrics, ml_ood_key_path, physic_ood_key_path) + self.assertEqual(ood_metrics["OOD"]["spearman_correlation_drag"], 0.9) + + def test_compute_speed_score(self): + time_inference = 5.0 + speed_score = self.scoring.compute_speed_score(time_inference) + self.assertAlmostEqual(speed_score, 0.61928031) + + def test_compute_speed_score_max_speed(self): + time_inference = 1600 + speed_score = self.scoring.compute_speed_score(time_inference) + self.assertAlmostEqual(speed_score, 0) + + @patch("builtins.open", new_callable=mock_open) + def test_compute_scores_from_path(self, mock_file): + mock_json_data = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"a_or": 0.4}, "MSE_normalized_surfacic": {"pressure": 0.008}}, + "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, + "Physics": {"spearman_correlation_lift": 0.9}}}} + mock_file.return_value.read.return_value = json.dumps(mock_json_data) + + scores = self.scoring.compute_scores(metrics_path="dummy.json") + self.assertAlmostEqual(scores["Global Score"], 0.484068188) + + def test_compute_scores_from_dict(self): + metrics_dict = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"a_or": 0.4}, "MSE_normalized_surfacic": {"pressure": 0.008}}, + "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, + "Physics": {"spearman_correlation_lift": 0.9}}}} + scores = self.scoring.compute_scores(metrics_dict=metrics_dict) + self.assertAlmostEqual(scores["Global Score"], 0.484068188) + + def test_compute_scores_invalid_input(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + def test_compute_scores_missing_time(self): + metrics_dict = {"fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"some_metric": 0.4}}, "Physics": {"some_metric": 0.7}}}, + "fc_metrics_test_ood": {"test_ood": {"ML": {"MSE_normalized": {"some_metric": 0.3}}, + "Physics": {"some_metric": 0.9}}}} # Dummy metrics + with self.assertRaises(KeyError): + self.scoring.compute_scores(metrics_dict=metrics_dict) + + +if __name__ == '__main__': + unittest.main() diff --git a/lips/tests/scoring/test_ml4physim_powergrid_socring.py b/lips/tests/scoring/test_ml4physim_powergrid_socring.py new file mode 100644 index 0000000..b94e9cc --- /dev/null +++ b/lips/tests/scoring/test_ml4physim_powergrid_socring.py @@ -0,0 +1,162 @@ +import json +import unittest +from unittest.mock import patch, MagicMock, mock_open + +from lips.scoring import ML4PhysimPowerGridScoring + + +class TestML4PhysimPowerGridScoring(unittest.TestCase): + + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.03, 0.06]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.04, 0.07]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.05, 0.08]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.06, 0.09]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.07, 0.1]}, + + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [0.1, 0.2]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [0.2, 0.3]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [0.3, 0.4]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [0.4, 0.5]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [0.5, 0.6]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.6, 0.7]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.7, 0.8]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [0.8, 0.9]}, + + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}}, + + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + "coefficients": {"ID": {"value": 0.3, "ML": {"value": 0.4}, "Physics": {"value": 0.6}}, + "OOD": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "SpeedUP": {"value": 0.4}}}[key] + + self.scoring = ML4PhysimPowerGridScoring(config=self.mock_config) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_valid_path(self, mock_get_nested_value): + raw_metrics = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.1, "a_ex": 0.2}, "MAPE_10_avg": {"p_or": 0.3, "p_ex": 0.4}, + "MAE_avg": {"v_or": 0.5, "v_ex": 0.6}}}} + mock_get_nested_value.return_value = raw_metrics["test"]["ML"] + ml_section_path = ["test", "ML"] + expected_result = {"ML": {"a_or": 0.1, "a_ex": 0.2, "p_or": 0.3, "p_ex": 0.4, "v_or": 0.5, "v_ex": 0.6}} + result = self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + self.assertEqual(result, expected_result) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_invalid_path(self, mock_get_nested_value): + raw_metrics = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.1, "a_ex": 0.2}, "MAPE_10_avg": {"p_or": 0.3, "p_ex": 0.4}, + "MAE_avg": {"v_or": 0.5, "v_ex": 0.6}}}} + mock_get_nested_value.return_value = None + ml_section_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_invalid_type(self, mock_get_nested_value): + raw_metrics = {"test": {"ML": "invalid_type"}} + mock_get_nested_value.return_value = raw_metrics["test"]["ML"] + ml_section_path = ["test", "ML"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_valid_path(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.1}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.2}}, + "LOSS_POS": {"violation_proportion": 0.3}, + "DISC_LINES": {"violation_proportion": 0.4}, + "CHECK_LOSS": {"violation_percentage": 0.5}, + "CHECK_GC": {"violation_percentage": 0.6}, + "CHECK_LC": {"violation_percentage": 0.7}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.8}}}} + mock_get_nested_value.return_value = raw_metrics["test"]["Physics"] + physic_section_path = ["test", "Physics"] + expected_result = {"Physics": {"CURRENT_POS": 10.0, "VOLTAGE_POS": 20.0, "LOSS_POS": 30.0, "DISC_LINES": 40.0, + "CHECK_LOSS": 0.5, "CHECK_GC": 0.6, "CHECK_LC": 0.7, "CHECK_JOULE_LAW": 80.0}} + result = self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + self.assertEqual(result, expected_result) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_invalid_path(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.5}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.2}}, + "LOSS_POS": {"violation_proportion": 0.3}, + "DISC_LINES": {"violation_proportion": 0.4}, + "CHECK_LOSS": {"violation_percentage": 0.5}, + "CHECK_GC": {"violation_percentage": 0.6}, + "CHECK_LC": {"violation_percentage": 0.7}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.8}}}} + mock_get_nested_value.return_value = None + physic_section_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_invalid_type(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": "invalid_type"}} + mock_get_nested_value.return_value = raw_metrics["test"]["Physics"] + physic_section_path = ["test", "Physics"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + + def test_calculate_speed_up(self): + time_inference = 0.5 + expected_speed_up = 3000.0 + result = self.scoring._calculate_speed_up(time_inference) + self.assertEqual(result, expected_speed_up) + + def test_compute_speed_score(self): + time_inference = 0.5 + expected_speed_score = 1.0 + result = self.scoring.compute_speed_score(time_inference) + self.assertEqual(result, expected_speed_score) + + def test_weibull(self): + c = 5 + b = 1.7 + x = 3000.0 + expected_weibull_value = 1.0 + result = self.scoring._weibull(c, b, x) + self.assertAlmostEqual(result, expected_weibull_value, places=5) + + @patch("builtins.open", new_callable=mock_open) + def test_compute_scores(self, mock_file): + mock_json_data = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.15, "a_ex": 0.25}, "MAPE_10_avg": {"p_or": 0.35, "p_ex": 0.45}, + "MAE_avg": {"v_or": 0.55, "v_ex": 0.65}, "TIME_INF": 12.0}, + "Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.15}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.25}}, + "LOSS_POS": {"violation_proportion": 0.35}, "DISC_LINES": {"violation_proportion": 0.45}, + "CHECK_LOSS": {"violation_percentage": 0.55}, "CHECK_GC": {"violation_percentage": 0.65}, + "CHECK_LC": {"violation_percentage": 0.75}, "CHECK_JOULE_LAW": {"violation_proportion": 0.85}}}, + "test_ood_topo": { + "ML": {"MAPE_90_avg": {"a_or": 0.12, "a_ex": 0.22}, "MAPE_10_avg": {"p_or": 0.32, "p_ex": 0.42}, + "MAE_avg": {"v_or": 0.52, "v_ex": 0.62}}, + "Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.12}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.22}}, + "LOSS_POS": {"violation_proportion": 0.32}, "DISC_LINES": {"violation_proportion": 0.42}, + "CHECK_LOSS": {"violation_percentage": 0.52}, "CHECK_GC": {"violation_percentage": 0.62}, + "CHECK_LC": {"violation_percentage": 0.72}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.82}}}} + + mock_file.return_value.read.return_value = json.dumps(mock_json_data) + + scores = self.scoring.compute_scores(metrics_path="dummy.json") + self.assertAlmostEqual(scores["Global Score"], 0.452874999) + + def test_compute_scores_invalid_input(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + def test_compute_scores_no_metrics(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + +if __name__ == '__main__': + unittest.main() diff --git a/lips/tests/scoring/test_scoring.py b/lips/tests/scoring/test_scoring.py new file mode 100644 index 0000000..16a1fd8 --- /dev/null +++ b/lips/tests/scoring/test_scoring.py @@ -0,0 +1,161 @@ +import unittest +from unittest.mock import MagicMock + +from lips.scoring import Scoring + + +class TestScoring(unittest.TestCase): + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + "coefficients": {"ML": {"value": 0.3}, "OOD": {"value": 0.3}, "Physics": {"value": 0.3}, + "Speed": {"value": 0.1}}}[key] + self.scoring = Scoring(config=self.mock_config) + + def test_colorize_metrics(self): + metrics = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, + "OOD": {"a_or": 0.03, "inference_time": 400}, + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.8}, "Speed": {"inference_time": 600}} + expected_colorized_metrics = {"ML": {"a_or": "green", "spearman_correlation_drag": "green"}, + "OOD": {"a_or": "orange", "inference_time": "green"}, + "Physics": {"a_or": "red", "spearman_correlation_drag": "red"}, + "Speed": {"inference_time": "orange"}} + colorized_metrics = self.scoring.colorize_metrics(metrics) + self.assertEqual(colorized_metrics, expected_colorized_metrics) + + def test__colorize_metric_value(self): + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.01), "green") + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.04), "orange") + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.06), "red") + + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.92), "green") + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.85), "orange") + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.7), "red") + + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 450), "green") + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 750), "orange") + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 1100), "red") + + def test__colorize_metric_value_invalid_comparison_type(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "invalid", "thresholds": [0.02, 0.05]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[ + key] # empty coefficients to avoid other errors. + self.scoring = Scoring(config=self.mock_config) # recreate scoring object with invalid config. + with self.assertRaises(ValueError) as context: + self.scoring._colorize_metric_value("a_or", 0.01) + self.assertIn("Invalid comparison type", str(context.exception)) + + def test__validate_configuration_missing_thresholds(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": None, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn("Thresholds configuration is missing", str(context.exception)) + + def test__validate_configuration_missing_valuebycolor(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}}, "valuebycolor": None, + "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn("Value by color configuration is missing", str(context.exception)) + + def test__validate_configuration_invalid_threshold_data(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"thresholds": [0.02, 0.05]}}, # Missing comparison_type + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn( + "Invalid thresholds data for metric 'a_or'. Must be a dict with 'thresholds' and 'comparison_type' keys.", + str(context.exception)) + + def test__validate_configuration_invalid_threshold_count(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05, 0.1]}}, + # Too many thresholds + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn("Metric 'a_or': Thresholds count must be 2 (length of ValueByColor - 1).", str(context.exception)) + + def test__calculate_leaf_score(self): + colors = ["green", "orange", "red"] + expected_score = (2 + 1 + 0) / (3 * 2) # (sum of values) / (number of colors * max value) + self.assertEqual(self.scoring._calculate_leaf_score(colors), expected_score) + + def test_calculate_sub_scores_leaf(self): + node = {"a_or": "green", "spearman_correlation_drag": "orange"} + expected_score = self.scoring._calculate_leaf_score(list(node.values())) + self.assertEqual(self.scoring.calculate_sub_scores(node), expected_score) + + def test_calculate_sub_scores_subtree(self): + node = {"ML": {"a_or": "green", "spearman_correlation_drag": "orange"}, "OOD": {"inference_time": "red"}} + expected_scores = {"ML": self.scoring._calculate_leaf_score(list(node["ML"].values())), + "OOD": self.scoring._calculate_leaf_score(list(node["OOD"].values()))} + self.assertEqual(self.scoring.calculate_sub_scores(node), expected_scores) + + def test_calculate_sub_scores_invalid_input(self): + with self.assertRaises(ValueError) as context: + self.scoring.calculate_sub_scores("not a dict") + self.assertIn("Input must be a dictionary.", str(context.exception)) + + def test_calculate_sub_scores_inconsistent_branching(self): + node = {"a": "green", "b": {"c": "red"}} + with self.assertRaises(ValueError) as context: + self.scoring.calculate_sub_scores(node) + self.assertIn("Parent node is not uniformly branched", str(context.exception)) + + def test_calculate_global_score_subtree(self): + tree = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, "OOD": {"inference_time": 400}, + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.75}, "Speed": {"inference_time": 600}, } + tree_score = {"ML": 1, "OOD": 1, "Physics": 0, "Speed": 0.5} + + # Calculate expected score manually based on coefficients and leaf scores + ml_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["ML"]).values())) + ood_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["OOD"]).values())) + physics_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["Physics"]).values())) + speed_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["Speed"]).values())) + + expected_global_score = (0.3 * ml_score + 0.3 * ood_score + 0.3 * physics_score + 0.1 * speed_score) + + global_score = self.scoring.calculate_global_score(tree_score) # Operate on the original tree + + self.assertAlmostEqual(global_score, expected_global_score, places=6) # Use assertAlmostEqual + + def test_calculate_global_score_subtree_with_missing_coefficient(self): + tree = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, # Original float values + "OOD": {"inference_time": 400}, # Original float values + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.75}, # Original float values + "Speed": {"inference_time": 600}, # Original float values + "NewComponent": {"a_or": 0.03} # Original float values + } + tree_score = {"ML": 1, "OOD": 1, "Physics": 0, "Speed": 0.5, "NewComponent": 0.5} + + ml_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["ML"]).values())) + ood_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["OOD"]).values())) + physics_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["Physics"]).values())) + speed_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["Speed"]).values())) + new_component_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["NewComponent"]).values())) + + expected_global_score = ( + 0.3 * ml_score + 0.3 * ood_score + 0.3 * physics_score + 0.1 * speed_score + new_component_score + # Default weight of 1 + ) + + global_score = self.scoring.calculate_global_score(tree_score) # Operate on the original tree + self.assertAlmostEqual(global_score, expected_global_score, places=6) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup.py b/setup.py index f2a92d1..aaa7ea0 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: MPL-2.0 # This file is part of leap_net, leap_net a keras implementation of the LEAP Net model. + import os import setuptools from setuptools import setup