From 9870a3ad8a0d2b30f9f1438a3b94918e732e6ef2 Mon Sep 17 00:00:00 2001 From: seif ATTOUI Date: Thu, 28 Nov 2024 16:48:29 +0000 Subject: [PATCH 01/18] add scoring feature --- .../powergrid/scoring/ScoreConfig.ini | 27 ++++++++++++ lips/scoring/__init__.py | 9 ++++ lips/scoring/powergrid_scoring.py | 34 +++++++++++++++ lips/scoring/scoring.py | 34 +++++++++++++++ lips/scoring/utils.py | 42 +++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 configurations/powergrid/scoring/ScoreConfig.ini create mode 100644 lips/scoring/__init__.py create mode 100644 lips/scoring/powergrid_scoring.py create mode 100644 lips/scoring/scoring.py create mode 100644 lips/scoring/utils.py diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini new file mode 100644 index 0000000..30072d5 --- /dev/null +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -0,0 +1,27 @@ +[DEFAULT] +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.1} + +ML = {"test": 0.3, "test_ood": 0.6} +OOD = {"test": 0.3, "test_ood": 0.6} +Physics = {"test": 0.3, "test_ood": 0.6} +SpeedUP = {"test": 0.3, "test_ood": 0.6} + +ValueByColor = {"g": 2, "o": 1, "r": 0} +MaxSpeedRatioAllowed = 10000 +ReferenceMeanSimulationTime = 1500 + +Thresholds = {"a_or":(0.02,0.05,"min"), + "a_ex":(0.02,0.05,"min"), + "p_or":(0.02,0.05,"min"), + "p_ex":(0.02,0.05,"min"), + "v_or":(0.2,0.5,"min"), + "v_ex":(0.2,0.5,"min"), + "CURRENT_POS":(1., 5.,"min"), + "VOLTAGE_POS":(1.,5.,"min"), + "LOSS_POS":(1.,5.,"min"), + "DISC_LINES":(1.,5.,"min"), + "CHECK_LOSS":(1.,5.,"min"), + "CHECK_GC":(0.05,0.10,"min"), + "CHECK_LC":(0.05,0.10,"min"), + "CHECK_JOULE_LAW":(1.,5.,"min") + } \ No newline at end of file diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py new file mode 100644 index 0000000..6bb86c9 --- /dev/null +++ b/lips/scoring/__init__.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from lips.scoring.scoring import Scoring +from lips.scoring.powergrid_scoring import PowerGridScoring + + +__all__ = [ + "Scoring", "PowerGridScoring" +] \ No newline at end of file diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py new file mode 100644 index 0000000..a7c2b97 --- /dev/null +++ b/lips/scoring/powergrid_scoring.py @@ -0,0 +1,34 @@ +from abc import ABC +from typing import Union, Dict + +from lips.config import ConfigManager +from lips.logger import CustomLogger +from lips.scoring import Scoring +from lips.scoring.utils import read_json + + +class PowerGridScoring(Scoring, ABC): + + def __init__(self, + config: Union[ConfigManager, None] = None, + config_path: Union[str, None] = None, + scenario: Union[str, None] = None, + log_path: Union[str, None] = None + ): + super().__init__(config=config, + config_path=config_path, + config_section=scenario, + log_path=log_path + ) + self.logger = CustomLogger(__class__.__name__, self.log_path).logger + + self.thresholds = self.config.get_option("thresholds") + self.coefficients = self.config.get_option("coefficients") + self.value_by_color = self.config.get_option("valuebycolor") + + def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = None): + ##return read_json(json_path=metrics_path, json_object=metrics_dict) + pass + + def _sub_soring(self): + pass diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py new file mode 100644 index 0000000..150150c --- /dev/null +++ b/lips/scoring/scoring.py @@ -0,0 +1,34 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking +from abc import ABC, abstractmethod +from typing import Union + +from ..config import ConfigManager +from ..logger import CustomLogger + + +class Scoring(ABC): + + def __init__(self, + config: Union[ConfigManager, None] = None, + config_path: Union[str, None] = None, + config_section: Union[str, None] = None, + log_path: Union[str, None] = None + ): + if config is None: + self.config = ConfigManager(section_name=config_section, path=config_path) + else: + self.config = config + + # logger + self.log_path = log_path + self.logger = CustomLogger(__class__.__name__, self.log_path).logger + + @abstractmethod + def scoring(self): + pass diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py new file mode 100644 index 0000000..ff927f5 --- /dev/null +++ b/lips/scoring/utils.py @@ -0,0 +1,42 @@ +import json +from typing import Union, Dict + + +def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): + """ + Reads a JSON file from the specified path or a JSON object if the path is empty. + + Args: + json_path (str): Path to the JSON file. If empty, the json_object is used. + json_object (Union[Dict, str, None]): A JSON object (as a dict or a JSON string). Used if json_path is empty. + + Returns: + Any: Parsed JSON data as a Python object (dict, list, etc.). + + Raises: + ValueError: If both json_path and json_object are empty. + FileNotFoundError: If the json_path does not exist. + json.JSONDecodeError: If the JSON data is invalid. + """ + if json_path: + # Read from JSON file + try: + with open(json_path, "r") as file: + return json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"JSON file not found at path: {json_path}") from e + except json.JSONDecodeError as e: + raise json.JSONDecodeError(f"Invalid JSON format in file: {json_path}", e.doc, e.pos) + elif json_object: + # Read from provided JSON object + if isinstance(json_object, str): + try: + return json.loads(json_object) # Parse JSON string + except json.JSONDecodeError as e: + raise json.JSONDecodeError("Invalid JSON string provided.", e.doc, e.pos) + elif isinstance(json_object, dict): + return json_object # Already a parsed dictionary + else: + raise ValueError("json_metrics must be a valid JSON string or dictionary.") + else: + raise ValueError("Both json_path and json_object are empty. Provide at least one.") From 3f7bc9aae3c30c843a521a9277458280af133acc Mon Sep 17 00:00:00 2001 From: seif ATTOUI Date: Fri, 29 Nov 2024 16:03:11 +0000 Subject: [PATCH 02/18] integrate color scoring feature --- .../powergrid/scoring/ScoreConfig.ini | 17 +++-- lips/scoring/powergrid_scoring.py | 64 +++++++++++++++++-- 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index 30072d5..e49ea0f 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,12 +1,8 @@ [DEFAULT] Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.1} -ML = {"test": 0.3, "test_ood": 0.6} -OOD = {"test": 0.3, "test_ood": 0.6} -Physics = {"test": 0.3, "test_ood": 0.6} -SpeedUP = {"test": 0.3, "test_ood": 0.6} - ValueByColor = {"g": 2, "o": 1, "r": 0} + MaxSpeedRatioAllowed = 10000 ReferenceMeanSimulationTime = 1500 @@ -23,5 +19,14 @@ Thresholds = {"a_or":(0.02,0.05,"min"), "CHECK_LOSS":(1.,5.,"min"), "CHECK_GC":(0.05,0.10,"min"), "CHECK_LC":(0.05,0.10,"min"), - "CHECK_JOULE_LAW":(1.,5.,"min") + "CHECK_JOULE_LAW":(1.,5.,"min"), + "x-velocity":(0.01,0.02,"min"), + "y-velocity":(0.01,0.02,"min"), + "pressure":(0.002,0.01,"min"), + "pressure_surfacic":(0.008,0.02,"min"), + "turbulent_viscosity":(0.05,0.1,"min"), + "mean_relative_drag":(0.4,5.0,"min"), + "mean_relative_lift":(0.1,0.3,"min"), + "spearman_correlation_drag":(0.8,0.9,"max"), + "spearman_correlation_lift":(0.96,0.99,"max") } \ No newline at end of file diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index a7c2b97..6e9be73 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,3 +1,4 @@ +import math from abc import ABC from typing import Union, Dict @@ -27,8 +28,63 @@ def __init__(self, self.value_by_color = self.config.get_option("valuebycolor") def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = None): - ##return read_json(json_path=metrics_path, json_object=metrics_dict) - pass + return read_json(json_path=metrics_path, json_object=metrics_dict) - def _sub_soring(self): - pass + @staticmethod + def calculate_score_color(metrics, thresholds): + tree = {} + for key, value in metrics.items(): + if isinstance(value, dict): + tree[key] = PowerGridScoring.calculate_score_color(value, thresholds) + else: + discrete_metric = PowerGridScoring._discretize_metric(metric_name=key, metric_value=value, + thresholds=thresholds) + tree[key] = discrete_metric + return tree + + @staticmethod + def _discretize_metric(metric_name, metric_value, thresholds): + """ + Discretize a metric value into a qualitative evaluation (g, o, r). + + :param metric_name: Name of the metric to evaluate + :param metric_value: The value of the metric to be evaluated + :param thresholds: Dictionary with thresholds for each metric. Format: + { + "metric_name": (threshold_min, threshold_max, eval_type) + } + eval_type can be "min" or "max". + :return: Evaluation string ("g", "o", or "r") + :raises ValueError: If the metric_name is not in thresholds or eval_type is invalid + """ + # Ensure the metric name exists in thresholds + if metric_name not in thresholds: + available_metrics = ", ".join(thresholds.keys()) + raise ValueError( + f"Metric '{metric_name}' not found in thresholds. Available metrics in thresholds: {available_metrics}") + + # Extract thresholds and evaluation type + threshold_min, threshold_max, eval_type = thresholds[metric_name] + + # Validation for eval_type + if eval_type not in {"min", "max"}: + raise ValueError(f"Invalid eval_type '{eval_type}' for metric '{metric_name}'. Must be 'min' or 'max'.") + + # Determine evaluation based on thresholds and eval_type + if eval_type == "min": + # "min" means smaller values are better + evaluation = "g" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "r" + else: + # "max" means larger values are better + evaluation = "r" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "g" + + return evaluation + + @staticmethod + def _calculate_speed_score(time_inference, time_classical_solver, max_speed_ratio_allowed): + speed_up = PowerGridScoring._calculate_speed_up(time_classical_solver, time_inference) + return max(min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1), 0) + + @staticmethod + def _calculate_speed_up(time_classical_solver, time_inference): + return time_classical_solver / time_inference From 3df05a40afcdc41f023625fbf7f31c9029d64450 Mon Sep 17 00:00:00 2001 From: seif ATTOUI Date: Fri, 29 Nov 2024 16:56:41 +0000 Subject: [PATCH 03/18] calculate sub scoring --- lips/scoring/powergrid_scoring.py | 32 ++++++++++++++++++++++++++----- lips/scoring/utils.py | 25 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 6e9be73..434d9d7 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,11 +1,10 @@ -import math from abc import ABC -from typing import Union, Dict +from typing import Union, Dict, List from lips.config import ConfigManager from lips.logger import CustomLogger from lips.scoring import Scoring -from lips.scoring.utils import read_json +from lips.scoring import utils class PowerGridScoring(Scoring, ABC): @@ -28,7 +27,25 @@ def __init__(self, self.value_by_color = self.config.get_option("valuebycolor") def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = None): - return read_json(json_path=metrics_path, json_object=metrics_dict) + + if metrics_dict is not None: + metrics = metrics_dict + elif metrics_path != "": + metrics = utils.read_json(json_path=metrics_path, json_object=metrics_dict) + else: + raise ValueError("metrics_path and metrics_dict cant' both be None") + + + score_color = PowerGridScoring.calculate_score_color(metrics, self.thresholds) + + score = {} + + for key in self.coefficients: + if key in score_color: + flat_dict = utils.flatten_dict(score_color[key]) + score[key] = self.calculate_sub_score(flat_dict.values()) + return score + @staticmethod def calculate_score_color(metrics, thresholds): @@ -42,6 +59,10 @@ def calculate_score_color(metrics, thresholds): tree[key] = discrete_metric return tree + def calculate_sub_score(self, colors: List[str]): + s = sum([self.value_by_color[color] for color in colors]) + return s / (len(colors) * max(self.value_by_color.values())) + @staticmethod def _discretize_metric(metric_name, metric_value, thresholds): """ @@ -83,7 +104,8 @@ def _discretize_metric(metric_name, metric_value, thresholds): @staticmethod def _calculate_speed_score(time_inference, time_classical_solver, max_speed_ratio_allowed): speed_up = PowerGridScoring._calculate_speed_up(time_classical_solver, time_inference) - return max(min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1), 0) + res = utils.weibull(5, 1.7, speed_up) + return max(min(res, 1), 0) @staticmethod def _calculate_speed_up(time_classical_solver, time_inference): diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index ff927f5..665c258 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -1,4 +1,5 @@ import json +import math from typing import Union, Dict @@ -40,3 +41,27 @@ def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): raise ValueError("json_metrics must be a valid JSON string or dictionary.") else: raise ValueError("Both json_path and json_object are empty. Provide at least one.") + + +def flatten_dict(input_dict, parent_key=""): + """ + Flatten a nested dictionary structure. + + :param input_dict: Dictionary to flatten + :param parent_key: Key to prepend (used for recursion) + :return: Flattened dictionary + """ + flattened = {} + for key, value in input_dict.items(): + if isinstance(value, dict): + # Recursively flatten if the value is a dictionary + flattened.update(flatten_dict(value, parent_key)) + else: + # Add to flattened dictionary + flattened[key] = value + return flattened + + +def weibull(c, b, x): + a = c * ((-math.log(0.9)) ** (-1 / b)) + return 1. - math.exp(-(x / a) ** b) From cae4f3ea7182a1c3d8871690248ce17df1ea6451 Mon Sep 17 00:00:00 2001 From: seif ATTOUI Date: Fri, 29 Nov 2024 16:57:20 +0000 Subject: [PATCH 04/18] calculate sub scoring --- lips/scoring/powergrid_scoring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 434d9d7..3421f5c 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -44,6 +44,8 @@ def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = if key in score_color: flat_dict = utils.flatten_dict(score_color[key]) score[key] = self.calculate_sub_score(flat_dict.values()) + + ##TODO add speedup score return score From b44c1348df24f577a2b8fda0cf5505907cc60d2c Mon Sep 17 00:00:00 2001 From: seif ATTOUI Date: Wed, 18 Dec 2024 16:18:03 +0000 Subject: [PATCH 05/18] updates --- .../powergrid/scoring/ScoreConfig.ini | 8 +- lips/scoring/powergrid_scoring.py | 113 +++++++++++++++--- lips/scoring/utils.py | 27 +++++ 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index e49ea0f..569105b 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,10 +1,12 @@ [DEFAULT] -Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.1} +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} ValueByColor = {"g": 2, "o": 1, "r": 0} -MaxSpeedRatioAllowed = 10000 -ReferenceMeanSimulationTime = 1500 +SpeedConfig = {"speed_score_formula" :"PowerGrid Competition", + "max_speed_ratio_allowed" : 10000, + "reference_mean_simulation_time": 1500 + } Thresholds = {"a_or":(0.02,0.05,"min"), "a_ex":(0.02,0.05,"min"), diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 3421f5c..a3aaf6d 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,3 +1,4 @@ +import math from abc import ABC from typing import Union, Dict, List @@ -5,9 +6,10 @@ from lips.logger import CustomLogger from lips.scoring import Scoring from lips.scoring import utils +from lips.scoring.utils import get_nested_value, filter_metrics -class PowerGridScoring(Scoring, ABC): +class PowerGridScoring(Scoring): def __init__(self, config: Union[ConfigManager, None] = None, @@ -26,42 +28,50 @@ def __init__(self, self.coefficients = self.config.get_option("coefficients") self.value_by_color = self.config.get_option("valuebycolor") + self.speed_config = self.config.get_option("speedconfig") + def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = None): if metrics_dict is not None: - metrics = metrics_dict + metrics = metrics_dict.copy() elif metrics_path != "": metrics = utils.read_json(json_path=metrics_path, json_object=metrics_dict) else: raise ValueError("metrics_path and metrics_dict cant' both be None") + # calculate speed score + time_inference = metrics.pop("Speed")["inference_time"] + speed_score = self._calculate_speed_score(time_inference) + #score discretize + score_color = PowerGridScoring._calculate_score_color(metrics, self.thresholds) - score_color = PowerGridScoring.calculate_score_color(metrics, self.thresholds) - - score = {} + score_values = dict() for key in self.coefficients: if key in score_color: flat_dict = utils.flatten_dict(score_color[key]) - score[key] = self.calculate_sub_score(flat_dict.values()) + score_values[key] = self._calculate_sub_score(flat_dict.values()) + score_values["Speed"] = speed_score - ##TODO add speedup score - return score + # calculate global score value + global_score = self._calculate_global_score(score_values) + score_values["Global Score"] = global_score + return {"Score Colors": score_color, "Score Values": score_values} @staticmethod - def calculate_score_color(metrics, thresholds): + def _calculate_score_color(metrics, thresholds): tree = {} for key, value in metrics.items(): if isinstance(value, dict): - tree[key] = PowerGridScoring.calculate_score_color(value, thresholds) + tree[key] = PowerGridScoring._calculate_score_color(value, thresholds) else: discrete_metric = PowerGridScoring._discretize_metric(metric_name=key, metric_value=value, thresholds=thresholds) tree[key] = discrete_metric return tree - def calculate_sub_score(self, colors: List[str]): + def _calculate_sub_score(self, colors: List[str]): s = sum([self.value_by_color[color] for color in colors]) return s / (len(colors) * max(self.value_by_color.values())) @@ -102,13 +112,86 @@ def _discretize_metric(metric_name, metric_value, thresholds): evaluation = "r" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "g" return evaluation + # separate competions from powergrid_scoring + def _calculate_speed_score(self, time_inference): - @staticmethod - def _calculate_speed_score(time_inference, time_classical_solver, max_speed_ratio_allowed): + time_classical_solver = self.speed_config["reference_mean_simulation_time"] speed_up = PowerGridScoring._calculate_speed_up(time_classical_solver, time_inference) - res = utils.weibull(5, 1.7, speed_up) - return max(min(res, 1), 0) + + if self.speed_config["speed_score_formula"] == "PowerGrid Competition": + return PowerGridScoring._calculate_speed_score_powergrid_competition_formula(speed_up) + elif self.speed_config["speed_score_formula"] == "AirFoil Competition": + + max_speed_ratio_allowed = self.speed_config["max_speed_ratio_allowed"] + + return PowerGridScoring._calculate_speed_score_airfoil_competition_formula(speed_up, + max_speed_ratio_allowed) + else: + raise ValueError(f'{self.speed_config["speed_score_formula"]} formula not found, please implement it first') @staticmethod def _calculate_speed_up(time_classical_solver, time_inference): return time_classical_solver / time_inference + + @staticmethod + def _calculate_speed_score_airfoil_competition_formula(speed_up, max_speed_ratio_allowed): + res = min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1) + return max(res, 0) + + @staticmethod + def _calculate_speed_score_powergrid_competition_formula(speed_up): + res = utils.weibull(5, 1.7, speed_up) + return max(min(res, 1), 0) + + def _calculate_global_score(self, sub_scores): + global_score = 0 + for coef in self.coefficients.keys(): + global_score += self.coefficients[coef] * sub_scores[coef] + return global_score + + @staticmethod + def reconstruct_ml_metrics(input_json, ml_key_path, used_metric_list): + """ + Construct ML metrics by retrieving and filtering data from the given JSON. + + Parameters: + - input_json (dict): The input JSON containing the ML metrics. + - ml_key_path (list): Path to the ML section in the JSON as a list of keys. + - used_metric_list (list): List of metrics to include in the output. + + Returns: + - dict: Filtered ML metrics containing only the specified metrics. + """ + all_ml_metrics = get_nested_value(input_json, ml_key_path) + if all_ml_metrics is None: + raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") + + if not isinstance(all_ml_metrics, dict): + raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") + + return {"ML": filter_metrics(all_ml_metrics, used_metric_list)} + + @staticmethod + def reconstruct_speed_metric(input_json, speed_key_path): + """ + Construct a dictionary containing the speed metric. + + Parameters: + - input_json (dict): The input JSON containing the speed metric. + - speed_key_path (list): Path to the inference time in the JSON as a list of keys. + + Returns: + - dict: A dictionary with the speed metric in the format {"Speed": {"inference_time": value}}. + + Raises: + - ValueError: If the specified path does not exist or the value is None. + """ + inference_time = get_nested_value(input_json, speed_key_path) + + if inference_time is None: + raise ValueError(f"Invalid path {speed_key_path}. Could not retrieve inference time.") + + if not isinstance(inference_time, (int, float)): + raise TypeError(f"Inference time must be a numeric value, but got {type(inference_time).__name__}.") + + return {"Speed": {"inference_time": inference_time}} diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index 665c258..a28abe9 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -65,3 +65,30 @@ def flatten_dict(input_dict, parent_key=""): def weibull(c, b, x): a = c * ((-math.log(0.9)) ** (-1 / b)) return 1. - math.exp(-(x / a) ** b) + +def merge_dicts(dict_list): + """ + Merges a list of dictionaries into a single dictionary. + + Parameters: + - dict_list (list): A list of dictionaries to merge. + + Returns: + - dict: A single dictionary containing all key-value pairs. + """ + merged_dict = {} + for d in dict_list: + merged_dict.update(d) # Update merged_dict with each dictionary in the list + return merged_dict + +def get_nested_value(data, keys): + """Retrieve a nested value from a dictionary using a list of keys.""" + for key in keys: + if key not in data: + return None + data = data[key] + return data + +def filter_metrics(data, metric_list): + """Filter the data dictionary to include only the specified metrics.""" + return {key: value for key, value in data.items() if key in metric_list} \ No newline at end of file From e457f0e02bbfd4a3d874e072190202a01a6e54f1 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Thu, 19 Dec 2024 12:17:11 +0100 Subject: [PATCH 06/18] updates --- lips/scoring/powergrid_scoring.py | 46 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index a3aaf6d..b4bc1e8 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,5 +1,4 @@ import math -from abc import ABC from typing import Union, Dict, List from lips.config import ConfigManager @@ -11,17 +10,9 @@ class PowerGridScoring(Scoring): - def __init__(self, - config: Union[ConfigManager, None] = None, - config_path: Union[str, None] = None, - scenario: Union[str, None] = None, - log_path: Union[str, None] = None - ): - super().__init__(config=config, - config_path=config_path, - config_section=scenario, - log_path=log_path - ) + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + scenario: Union[str, None] = None, log_path: Union[str, None] = None): + super().__init__(config=config, config_path=config_path, config_section=scenario, log_path=log_path) self.logger = CustomLogger(__class__.__name__, self.log_path).logger self.thresholds = self.config.get_option("thresholds") @@ -42,7 +33,7 @@ def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = # calculate speed score time_inference = metrics.pop("Speed")["inference_time"] speed_score = self._calculate_speed_score(time_inference) - #score discretize + # score discretize score_color = PowerGridScoring._calculate_score_color(metrics, self.thresholds) score_values = dict() @@ -112,6 +103,7 @@ def _discretize_metric(metric_name, metric_value, thresholds): evaluation = "r" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "g" return evaluation + # separate competions from powergrid_scoring def _calculate_speed_score(self, time_inference): @@ -195,3 +187,31 @@ def reconstruct_speed_metric(input_json, speed_key_path): raise TypeError(f"Inference time must be a numeric value, but got {type(inference_time).__name__}.") return {"Speed": {"inference_time": inference_time}} + + @staticmethod + def reconstruct_physic_metrics(input_json, physic_key_path, competition_name, used_metric_list=None): + + all_physic_metrics = get_nested_value(input_json, physic_key_path) + if all_physic_metrics is None: + raise ValueError(f"Invalid path {physic_key_path}. Could not retrieve physic metrics.") + + if not isinstance(all_physic_metrics, dict): + raise TypeError(f"Expected a dictionary at {physic_key_path}, but got {type(all_physic_metrics).__name__}.") + + if competition_name == "AirFoil Competition": + physic_metrics = filter_metrics(all_physic_metrics, used_metric_list) + + elif competition_name == "PowerGrid Competition": + physic_metrics = {"CURRENT_POS": all_physic_metrics["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100., + "VOLTAGE_POS": all_physic_metrics["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100., + "LOSS_POS": all_physic_metrics["LOSS_POS"]["violation_proportion"] * 100., + "DISC_LINES": all_physic_metrics["DISC_LINES"]["violation_proportion"] * 100., + "CHECK_LOSS": all_physic_metrics["CHECK_LOSS"]["violation_percentage"], + "CHECK_GC": all_physic_metrics["CHECK_GC"]["violation_percentage"], + "CHECK_LC": all_physic_metrics["CHECK_LC"]["violation_percentage"], + "CHECK_JOULE_LAW": all_physic_metrics["CHECK_JOULE_LAW"]["violation_proportion"] * 100.} + + else: + raise ValueError(f'{competition_name} not in options') + + return {"Physics": physic_metrics} From d398d8614251695c05e764a3bce68643c35ba216 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Mon, 10 Feb 2025 16:53:39 +0100 Subject: [PATCH 07/18] commit all --- draft_notebook.ipynb | 365 ++++++++++++++++++ draft_script.py | 158 ++++++++ score_scripts/NeurIPS_scoreV2_8/metadata | 2 + .../NeurIPS_scoreV2_8/res/json_metrics.json | 72 ++++ .../res/json_metrics_new.json | 29 ++ score_scripts/NeurIPS_scoreV2_8/score.py | 241 ++++++++++++ score_scripts/NeurIPS_scoreV2_8/scores.html | 1 + score_scripts/NeurIPS_scoreV2_8/scores.txt | 4 + score_scripts/PowerGrid_score/metadata | 2 + .../PowerGrid_score/res/json_metrics.json | 197 ++++++++++ .../PowerGrid_score/res/json_metrics_new.json | 39 ++ score_scripts/PowerGrid_score/score.py | 88 +++++ score_scripts/PowerGrid_score/scores.html | 0 score_scripts/PowerGrid_score/scores.txt | 0 .../PowerGrid_score/utils/__init__.py | 0 .../PowerGrid_score/utils/compute_score.py | 162 ++++++++ 16 files changed, 1360 insertions(+) create mode 100644 draft_notebook.ipynb create mode 100644 draft_script.py create mode 100644 score_scripts/NeurIPS_scoreV2_8/metadata create mode 100644 score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json create mode 100644 score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json create mode 100644 score_scripts/NeurIPS_scoreV2_8/score.py create mode 100644 score_scripts/NeurIPS_scoreV2_8/scores.html create mode 100644 score_scripts/NeurIPS_scoreV2_8/scores.txt create mode 100644 score_scripts/PowerGrid_score/metadata create mode 100644 score_scripts/PowerGrid_score/res/json_metrics.json create mode 100644 score_scripts/PowerGrid_score/res/json_metrics_new.json create mode 100644 score_scripts/PowerGrid_score/score.py create mode 100644 score_scripts/PowerGrid_score/scores.html create mode 100644 score_scripts/PowerGrid_score/scores.txt create mode 100644 score_scripts/PowerGrid_score/utils/__init__.py create mode 100644 score_scripts/PowerGrid_score/utils/compute_score.py diff --git a/draft_notebook.ipynb b/draft_notebook.ipynb new file mode 100644 index 0000000..7822b26 --- /dev/null +++ b/draft_notebook.ipynb @@ -0,0 +1,365 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "dbff8eac-c04b-4fe2-8462-ed3be79c8f7f", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:22:59.172122Z", + "start_time": "2024-12-18T17:22:59.153473Z" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "initial_id", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:00.903459Z", + "start_time": "2024-12-18T17:23:00.884193Z" + } + }, + "outputs": [], + "source": [ + "from lips.scoring import PowerGridScoring" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "542ed9d6-5949-409a-bcb5-380abc2a441c", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:02.170728Z", + "start_time": "2024-12-18T17:23:02.155588Z" + } + }, + "outputs": [], + "source": [ + "scoring = PowerGridScoring(config_path=\"configurations/powergrid/scoring/ScoreConfig.ini\", scenario=\"DEFAULT\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4c318392-fc30-4e95-b9b2-7d8fad33ee06", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:03.691169Z", + "start_time": "2024-12-18T17:23:03.666826Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Score Colors': {'ML': {'x-velocity': 'g',\n", + " 'y-velocity': 'g',\n", + " 'pressure': 'o',\n", + " 'turbulent_viscosity': 'g'},\n", + " 'Physics': {'spearman_correlation_drag': 'o',\n", + " 'spearman_correlation_lift': 'g',\n", + " 'mean_relative_drag': 'g',\n", + " 'mean_relative_lift': 'g'},\n", + " 'OOD': {'ML': {'x-velocity': 'g',\n", + " 'y-velocity': 'g',\n", + " 'pressure': 'r',\n", + " 'turbulent_viscosity': 'g'},\n", + " 'Physics': {'spearman_correlation_drag': 'r',\n", + " 'spearman_correlation_lift': 'g',\n", + " 'mean_relative_drag': 'g',\n", + " 'mean_relative_lift': 'g'}}},\n", + " 'Score Values': {'ML': 0.875,\n", + " 'OOD': 0.75,\n", + " 'Physics': 0.875,\n", + " 'Speed': 0.024655622815809464,\n", + " 'Global Score': 0.7524655622815809}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoring.scoring(metrics_path=\"./score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "44738300-76ab-4834-b8d9-bf96b6b04f8f", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:05.955460Z", + "start_time": "2024-12-18T17:23:05.932678Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'test': {'ML': {'MSE_avg': {'v_or': 0.00010645988368196413,\n", + " 'v_ex': 0.00011296240700175986,\n", + " 'p_or': 0.18411502242088318,\n", + " 'p_ex': 0.1777658313512802,\n", + " 'a_or': 16.51224136352539,\n", + " 'a_ex': 16.86044692993164},\n", + " 'MAE_avg': {'v_or': 0.001553428708575666,\n", + " 'v_ex': 0.001500570448115468,\n", + " 'p_or': 0.14518952369689941,\n", + " 'p_ex': 0.14409466087818146,\n", + " 'a_or': 1.4204329252243042,\n", + " 'a_ex': 1.4652109146118164},\n", + " 'MAPE_avg': {'v_or': 9.848606168816332e-06,\n", + " 'v_ex': 1.0132289389730431e-05,\n", + " 'p_or': 63017717760.0,\n", + " 'p_ex': 22879956992.0,\n", + " 'a_or': 121335185408.0,\n", + " 'a_ex': 52108500992.0},\n", + " 'MAPE_90_avg': {'v_or': 5.658699643585033e-06,\n", + " 'v_ex': 5.919416245742559e-06,\n", + " 'p_or': 0.0033098454767013486,\n", + " 'p_ex': 0.0032964480543750145,\n", + " 'a_or': 0.007386949784782199,\n", + " 'a_ex': 0.007292245940119987},\n", + " 'MAPE_10_avg': {'v_or': 7.93336312753172e-06,\n", + " 'v_ex': 8.195988824322852e-06,\n", + " 'p_or': 0.006397687786608211,\n", + " 'p_ex': 0.006388048696331843,\n", + " 'a_or': 0.021526892545594315,\n", + " 'a_ex': 0.022162465019418712},\n", + " 'TIME_INF': 4.163906573085114},\n", + " 'Physics': {'CURRENT_POS': {'a_or': {'Error': 10040.3349609375,\n", + " 'Violation_proportion': 0.00044102150537634406},\n", + " 'a_ex': {'Error': 11018.400390625,\n", + " 'Violation_proportion': 0.0005930645161290322}},\n", + " 'VOLTAGE_POS': {'v_or': {'Violation_proportion': 0.0},\n", + " 'v_ex': {'Violation_proportion': 0.0}},\n", + " 'LOSS_POS': {'loss_criterion': 209.59898,\n", + " 'violation_proportion': 0.008733548387096774},\n", + " 'DISC_LINES': {'p_or': 0.0,\n", + " 'p_ex': 0.0,\n", + " 'a_or': 0.0,\n", + " 'a_ex': 0.0,\n", + " 'v_or': 0.0,\n", + " 'v_ex': 0.0,\n", + " 'violation_proportion': 0.0},\n", + " 'CHECK_LOSS': {'violation_percentage': 1.539},\n", + " 'CHECK_GC': {'violation_percentage': 0.0,\n", + " 'mae': 9.128214e-05,\n", + " 'wmape': 1.2492486e-06},\n", + " 'CHECK_LC': {'violation_percentage': 77.35536440677966,\n", + " 'mae': 0.10304196395080921,\n", + " 'mape': 0.002859223697836113},\n", + " 'CHECK_JOULE_LAW': {'violation_proportion': 0.0,\n", + " 'mae': 3.054519562577347e-07,\n", + " 'wmape': 2.268769940016548e-06}},\n", + " 'IndRed': {'TIME_INF': 3.9108043271116912}},\n", + " 'test_ood_topo': {'ML': {'MSE_avg': {'v_or': 0.0008741252822801471,\n", + " 'v_ex': 0.0009632128058001399,\n", + " 'p_or': 0.47769251465797424,\n", + " 'p_ex': 0.4475671350955963,\n", + " 'a_or': 40.1962890625,\n", + " 'a_ex': 41.00724411010742},\n", + " 'MAE_avg': {'v_or': 0.0023060280364006758,\n", + " 'v_ex': 0.0023030198644846678,\n", + " 'p_or': 0.18595275282859802,\n", + " 'p_ex': 0.1843600869178772,\n", + " 'a_or': 1.7880399227142334,\n", + " 'a_ex': 1.8418012857437134},\n", + " 'MAPE_avg': {'v_or': 1.5199337212834507e-05,\n", + " 'v_ex': 1.5929399523884058e-05,\n", + " 'p_or': 189589094400.0,\n", + " 'p_ex': 56932474880.0,\n", + " 'a_or': 386140471296.0,\n", + " 'a_ex': 157286449152.0},\n", + " 'MAPE_90_avg': {'v_or': 7.414690380649798e-06,\n", + " 'v_ex': 8.032887808804589e-06,\n", + " 'p_or': 0.004977200835027814,\n", + " 'p_ex': 0.004950829903700901,\n", + " 'a_or': 0.010261113576488012,\n", + " 'a_ex': 0.01012922324923192},\n", + " 'MAPE_10_avg': {'v_or': 1.0476555723965095e-05,\n", + " 'v_ex': 1.1106274117571183e-05,\n", + " 'p_or': 0.008155790825441843,\n", + " 'p_ex': 0.008147190351295156,\n", + " 'a_or': 0.027774648502433164,\n", + " 'a_ex': 0.028373457148515593}},\n", + " 'Physics': {'CURRENT_POS': {'a_or': {'Error': 21769.7578125,\n", + " 'Violation_proportion': 0.0004038709677419355},\n", + " 'a_ex': {'Error': 23943.244140625,\n", + " 'Violation_proportion': 0.0005124731182795699}},\n", + " 'VOLTAGE_POS': {'v_or': {'Violation_proportion': 0.0},\n", + " 'v_ex': {'Violation_proportion': 0.0}},\n", + " 'LOSS_POS': {'loss_criterion': 1076.4073,\n", + " 'violation_proportion': 0.00829997311827957},\n", + " 'DISC_LINES': {'p_or': 0.0,\n", + " 'p_ex': 0.0,\n", + " 'a_or': 0.0,\n", + " 'a_ex': 0.0,\n", + " 'v_or': 0.0,\n", + " 'v_ex': 0.0,\n", + " 'violation_proportion': 0.0},\n", + " 'CHECK_LOSS': {'violation_percentage': 3.2485},\n", + " 'CHECK_GC': {'violation_percentage': 0.0,\n", + " 'mae': 9.088051e-05,\n", + " 'wmape': 1.2194811e-06},\n", + " 'CHECK_LC': {'violation_percentage': 83.45940254237289,\n", + " 'mae': 0.14096058704044898,\n", + " 'mape': 0.0039999913278262085},\n", + " 'CHECK_JOULE_LAW': {'violation_proportion': 0.0,\n", + " 'mae': 2.9970232693607007e-07,\n", + " 'wmape': 2.1807441470482367e-06}},\n", + " 'IndRed': {}}}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "\n", + "with open(\"./score_scripts/PowerGrid_score/res/json_metrics.json\", \"r\") as f:\n", + " raw_metrics = json.load(f)\n", + "raw_metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3e05c2fd12981d8d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:16.572933Z", + "start_time": "2024-12-18T17:23:16.554868Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Speed': {'inference_time': 4.163906573085114}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoring.reconstruct_speed_metric(raw_metrics, [\"test\", \"ML\", \"TIME_INF\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b40bcc1a2a0a9ad9", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:23:18.497174Z", + "start_time": "2024-12-18T17:23:18.481366Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'ML': {'v_or': 0.00010645988368196413,\n", + " 'v_ex': 0.00011296240700175986,\n", + " 'p_or': 0.18411502242088318,\n", + " 'p_ex': 0.1777658313512802,\n", + " 'a_or': 16.51224136352539,\n", + " 'a_ex': 16.86044692993164}}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoring.reconstruct_ml_metrics(raw_metrics, [\"test\", \"ML\", \"MSE_avg\"], scoring.thresholds.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "19c352a2e2f0f9cb", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-18T17:43:48.497718Z", + "start_time": "2024-12-18T17:43:48.478015Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Physics': {'CURRENT_POS': 0.04410215053763441,\n", + " 'VOLTAGE_POS': 0.0,\n", + " 'LOSS_POS': 0.8733548387096773,\n", + " 'DISC_LINES': 0.0,\n", + " 'CHECK_LOSS': 1.539,\n", + " 'CHECK_GC': 0.0,\n", + " 'CHECK_LC': 77.35536440677966,\n", + " 'CHECK_JOULE_LAW': 0.0}}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scoring.reconstruct_physic_metrics(raw_metrics, [\"test\", \"Physics\"], competition_name=\"PowerGrid Competition\",used_metric_list=scoring.thresholds.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e1066c2-198c-4a6a-8e89-bfc28ac42dcb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dabb70f-110b-4151-aea4-62521e48e04d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/draft_script.py b/draft_script.py new file mode 100644 index 0000000..1d7a78c --- /dev/null +++ b/draft_script.py @@ -0,0 +1,158 @@ +from typing import List, Dict + +from lips.scoring import PowerGridScoring +from lips.scoring import utils + + + +metrics = utils.read_json(json_path="/mnt/seif/HSA/LIPS/score_scripts/PowerGrid_score/res/json_metrics.json") + +# thresholds={"a_or":(0.02,0.05,"min"), +# "a_ex":(0.02,0.05,"min"), +# "p_or":(0.02,0.05,"min"), +# "p_ex":(0.02,0.05,"min"), +# "v_or":(0.2,0.5,"min"), +# "v_ex":(0.2,0.5,"min"), +# "CURRENT_POS":(1., 5.,"min"), +# "VOLTAGE_POS":(1.,5.,"min"), +# "LOSS_POS":(1.,5.,"min"), +# "DISC_LINES":(1.,5.,"min"), +# "CHECK_LOSS":(1.,5.,"min"), +# "CHECK_GC":(0.05,0.10,"min"), +# "CHECK_LC":(0.05,0.10,"min"), +# "CHECK_JOULE_LAW":(1.,5.,"min") +# }.keys() + + +thresholds={"x-velocity":(0.01,0.02,"min"), + "y-velocity":(0.01,0.02,"min"), + "pressure":(0.002,0.01,"min"), + "pressure_surfacic":(0.008,0.02,"min"), + "turbulent_viscosity":(0.05,0.1,"min"), + "mean_relative_drag":(0.4,5.0,"min"), + "mean_relative_lift":(0.1,0.3,"min"), + "spearman_correlation_drag":(0.8,0.9,"max"), + "spearman_correlation_lift":(0.96,0.99,"max") + }.keys() +import json + +def transform_json_generic( + input_json, + used_metric_list, + ml_metric, + in_distrubution_key_path, + ood_key_path, + inference_time +): + """ + Transforms a JSON structure into a desired format based on selected metrics and paths. + + Parameters: + - input_json (dict): The original JSON data. + - used_metric_list (list): List of metrics to include in the output (e.g., ["a_or", "a_ex"]). + - ml_metric (str): Metric key to use in the ML section. + - in_distribution_key_path (list): Path to the in-distribution (test) section. + - ood_key_path (list): Path to the OOD test section. + - inference_time (float): The inference time value. + + Returns: + - dict: Transformed JSON in the desired format. + """ + def get_nested_value(data, keys): + """Retrieve a nested value from a dictionary using a list of keys.""" + for key in keys: + if key not in data: + return None + data = data[key] + return data + + def filter_metrics(data, metric_list): + """Filter the data dictionary to include only the specified metrics.""" + return {key: value for key, value in data.items() if key in metric_list} + + def transform_section(section, metric_list, ml_metric): + """Transform a section (e.g., in-distribution or OOD) based on the desired metrics.""" + transformed = {} + for subsection, data in section.items(): + if subsection == "ML": + # Process ML section with the chosen ml_metric + transformed["ML"] = filter_metrics(data.get(ml_metric, {}), metric_list) + elif subsection == "Physics": + # Process Physics section with the chosen metrics + transformed["Physics"] = filter_metrics(data, metric_list) + else: + # Process other subsections if needed + transformed[subsection] = filter_metrics(data, metric_list) + return transformed + + # Extract in-distribution section + in_distribution_section = get_nested_value(input_json, in_distrubution_key_path) + if not in_distribution_section: + raise ValueError(f"In-distribution section not found at path: {in_distrubution_key_path}") + + in_distribution_transformed = transform_section(in_distribution_section, used_metric_list, ml_metric) + + # Extract OOD section + ood_section = get_nested_value(input_json, ood_key_path) + if not ood_section: + raise ValueError(f"OOD section not found at path: {ood_key_path}") + + ood_transformed = transform_section(ood_section, used_metric_list, ml_metric) + + # Combine into the desired format + transformed_json = { + **in_distribution_transformed, + "Speed": {"inference_time": inference_time}, + "OOD": ood_transformed, + } + + return transformed_json + +# Example usage: + +# used_metric_list = thresholds +# ml_metric = "MSE_avg" +# in_distrubution_key_path = ["test"] +# ood_key_path = ["test_ood_topo"] +# inference_time = 4.163906573085114 +# +# transformed_json = transform_json_generic( +# input_json=metrics, +# used_metric_list=used_metric_list, +# ml_metric=ml_metric, +# in_distrubution_key_path=in_distrubution_key_path, +# ood_key_path=ood_key_path, +# inference_time=inference_time +# ) + +used_metric_list = thresholds + +from lips.scoring.utils import filter_metrics, get_nested_value + + +def construct_ml_metrics(input_json, ml_key_path, used_metric_list): + """ + Construct ML metrics by retrieving and filtering data from the given JSON. + + Parameters: + - input_json (dict): The input JSON containing the ML metrics. + - ml_key_path (list): Path to the ML section in the JSON as a list of keys. + - used_metric_list (list): List of metrics to include in the output. + + Returns: + - dict: Filtered ML metrics containing only the specified metrics. + """ + all_ml_metrics = get_nested_value(input_json, ml_key_path) + if all_ml_metrics is None: + raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") + + if not isinstance(all_ml_metrics, dict): + raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") + + return {"ML" : filter_metrics(all_ml_metrics, used_metric_list)} + + +def construct_speed_metric(input_json, speed_key_path): + return {"Speed": {"inference_time": get_nested_value(input_json, speed_key_path)}} + +print(construct_speed_metric(metrics, ['test', 'ML', 'TIME_INF'])) \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/metadata b/score_scripts/NeurIPS_scoreV2_8/metadata new file mode 100644 index 0000000..04b3b36 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/metadata @@ -0,0 +1,2 @@ +command: python $program/score.py $input $output +description: Compute scores for the competition diff --git a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json new file mode 100644 index 0000000..33619e5 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json @@ -0,0 +1,72 @@ +{ + "total_time": 9447.697538614273, + "training_time": 6113.95232629776, + "test_evaluation_time": 699.8024816513062, + "test_mean_simulation_time": 3.499012408256531, + "test_ood_evaluation_time": 1759.7924394607544, + "test_ood_mean_simulation_time": 3.5479686279450693, + "fc_metrics_test": { + "test": { + "ML": { + "MSE_normalized": { + "x-velocity": 0.00262499232487287, + "y-velocity": 0.0012471767764183842, + "pressure": 0.0031353064368674637, + "turbulent_viscosity": 0.02861798351376769 + }, + "MSE_normalized_surfacic": { + "pressure": 0.008781626853551213 + }, + "MAPE_normalized": { + "x-velocity": 0.23535119740284544, + "y-velocity": 0.17397767012401485, + "pressure": 0.165990814219197, + "turbulent_viscosity": 0.43160989540100914 + }, + "MAPE_normalized_surfacic": { + "pressure": 0.24832149427530473 + } + }, + "Physics": { + "spearman_correlation_drag": 0.8187369684242107, + "spearman_correlation_lift": 0.9983799594989875, + "mean_relative_drag": 0.23041081909925168, + "std_relative_drag": 0.2891833638823285, + "mean_relative_lift": 0.0611844697009299, + "std_relative_lift": 0.15944521815240825 + } + } + }, + "fc_metrics_test_ood": { + "test_ood": { + "ML": { + "MSE_normalized": { + "x-velocity": 0.009187407726621506, + "y-velocity": 0.0027280858877766494, + "pressure": 0.016331825251314956, + "turbulent_viscosity": 0.049333853470072006 + }, + "MSE_normalized_surfacic": { + "pressure": 0.06845324941442453 + }, + "MAPE_normalized": { + "x-velocity": 0.4080580649129622, + "y-velocity": 0.210093600068407, + "pressure": 0.25951081418011784, + "turbulent_viscosity": 0.46848256950820355 + }, + "MAPE_normalized_surfacic": { + "pressure": 0.32298146380673 + } + }, + "Physics": { + "spearman_correlation_drag": 0.7894712360182601, + "spearman_correlation_lift": 0.9971664788339026, + "mean_relative_drag": 0.3940285291162256, + "std_relative_drag": 0.46237913032263045, + "mean_relative_lift": 0.08437221869348716, + "std_relative_lift": 0.2164953143653425 + } + } + } +} \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json new file mode 100644 index 0000000..cdd7bc0 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json @@ -0,0 +1,29 @@ +{ + "ML": { + "x-velocity": 0.00262499232487287, + "y-velocity": 0.0012471767764183842, + "pressure": 0.0031353064368674637, + "turbulent_viscosity": 0.02861798351376769 + }, + "Physics": { + "spearman_correlation_drag": 0.8187369684242107, + "spearman_correlation_lift": 0.9983799594989875, + "mean_relative_drag": 0.23041081909925168, + "mean_relative_lift": 0.0611844697009299 + }, + "Speed" : {"inference_time": 699.8024816513062}, + "OOD": { + "ML": { + "x-velocity": 0.009187407726621506, + "y-velocity": 0.0027280858877766494, + "pressure": 0.016331825251314956, + "turbulent_viscosity": 0.049333853470072006 + }, + "Physics": { + "spearman_correlation_drag": 0.7894712360182601, + "spearman_correlation_lift": 0.9971664788339026, + "mean_relative_drag": 0.3940285291162256, + "mean_relative_lift": 0.08437221869348716 + } + } +} diff --git a/score_scripts/NeurIPS_scoreV2_8/score.py b/score_scripts/NeurIPS_scoreV2_8/score.py new file mode 100644 index 0000000..3784999 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/score.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# Scoring program for the AirfRANSModel challenge. +# Use directly the outputfile from the LIPS evaluation program. +# Some libraries and options +import os +from sys import argv +import sys +import json +import math + + +# Default I/O directories: +root_dir = "/app/" +default_input_dir = root_dir + "scoring_input" +default_output_dir = root_dir + "scoring_output" +default_input_dir = "./" +default_output_dir="./" + +# Constant used for a missing score +missing_score = -0.999999 + +def _HERE(*args): + h = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(h, *args) + +class ModelApiError(Exception): + """Model api error""" + + def __init__(self, msg=""): + self.msg = msg + print(msg) + + +def import_metrics(input_dir): + ## import parameters.json as a dictionary + path_submission_parameters = os.path.join(input_dir, 'res', 'json_metrics.json') + if not os.path.exists(path_submission_parameters): + raise ModelApiError("Missing json_metrics.json file") + exit_program() + with open(path_submission_parameters) as json_file: + metrics = json.load(json_file) + return metrics + +def exit_program(): + print("Error exiting") + sys.exit(0) + +def SpeedMetric(speedUp,speedMax): + return max(min(math.log10(speedUp)/math.log10(speedMax),1),0) + +# =============================== MAIN ======================================== + +if __name__ == "__main__": + + #### INPUT/OUTPUT: Get input and output directory names + if len(argv) == 1: # Use the default input and output directories if no arguments are provided + input_dir = default_input_dir + output_dir = default_output_dir + else: + input_dir = argv[1] + output_dir = argv[2] + # Create the output directory, if it does not already exist and open output files + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + score_file = open(os.path.join(output_dir, 'scores.txt'), 'w') + html_file = open(os.path.join(output_dir, 'scores.html'), 'w') + + # Init HTML + html_file.write("Scoring program output") + + ## thresholds + # First competition thresholds + # thresholds={"x-velocity":(0.1,0.2,"min"), + # "y-velocity":(0.1,0.2,"min"), + # "pressure":(0.02,0.1,"min"), + # "pressure_surfacic":(0.08,0.2,"min"), + # "turbulent_viscosity":(0.5,1.0,"min"), + # "mean_relative_drag":(1.0,10.0,"min"), + # "mean_relative_lift":(0.2,0.5,"min"), + # "spearman_correlation_drag":(0.5,0.8,"max"), + # "spearman_correlation_lift":(0.94,0.98,"max") + # } + + # NeurIPS competition thresholds + thresholds={"x-velocity":(0.01,0.02,"min"), + "y-velocity":(0.01,0.02,"min"), + "pressure":(0.002,0.01,"min"), + "pressure_surfacic":(0.008,0.02,"min"), + "turbulent_viscosity":(0.05,0.1,"min"), + "mean_relative_drag":(0.4,5.0,"min"), + "mean_relative_lift":(0.1,0.3,"min"), + "spearman_correlation_drag":(0.8,0.9,"max"), + "spearman_correlation_lift":(0.96,0.99,"max") + } + + + # scoring configuration + configuration={ + "coefficients":{"ML":0.4,"OOD":0.3,"Physics":0.3}, + "ratioRelevance":{"Speed-up":0.25,"Accuracy":0.75}, + "valueByColor":{"g":2,"o":1,"r":0}, + "maxSpeedRatioAllowed":10000, + "reference_mean_simulation_time":1500 + } + + coefficients=configuration["coefficients"] + ratioRelevance=configuration["ratioRelevance"] + valueByColor=configuration["valueByColor"] + maxSpeedRatioAllowed=configuration["maxSpeedRatioAllowed"] + + #Import metrics + metrics = import_metrics(input_dir) + fc_metrics_test = metrics["fc_metrics_test"] + fc_metrics_test_ood = metrics["fc_metrics_test_ood"] + test_mean_simulation_time = metrics["test_mean_simulation_time"] + test_ood_mean_simulation_time = metrics["test_ood_mean_simulation_time"] + + + ## Extract metrics of interest + ML_metrics = "ML" + ml_metrics = fc_metrics_test["test"][ML_metrics]["MSE_normalized"] + ml_metrics["pressure_surfacic"] = fc_metrics_test["test"][ML_metrics]["MSE_normalized_surfacic"]["pressure"] + + physic_compliances = "Physics" + phy_variables_to_keep = ["mean_relative_drag","mean_relative_lift","spearman_correlation_drag","spearman_correlation_lift"] + phy_metrics = {phy_variable:fc_metrics_test["test"][physic_compliances][phy_variable] for phy_variable in phy_variables_to_keep} + + ml_ood_metrics = fc_metrics_test_ood["test_ood"][ML_metrics]["MSE_normalized"] + ml_ood_metrics["pressure_surfacic"] = fc_metrics_test_ood["test_ood"][ML_metrics]["MSE_normalized_surfacic"]["pressure"] + phy_ood_metrics = {phy_variable:fc_metrics_test_ood["test_ood"][physic_compliances][phy_variable] for phy_variable in phy_variables_to_keep} + ood_metrics = {**ml_ood_metrics,**phy_ood_metrics} + + allmetrics={ + ML_metrics:ml_metrics, + physic_compliances:phy_metrics, + "OOD":ood_metrics + } + + print(allmetrics) + + ## Compute speed-up : + + + reference_mean_simulation_time=configuration["reference_mean_simulation_time"] + speedUp={ + ML_metrics:reference_mean_simulation_time/test_mean_simulation_time, + "OOD":reference_mean_simulation_time/test_mean_simulation_time + } + + accuracyResults=dict() + + + for subcategoryName, subcategoryVal in allmetrics.items(): + accuracyResults[subcategoryName]=[] + html_file.write("

" + subcategoryName + "

") + for variableName, variableError in subcategoryVal.items(): + thresholdMin,thresholdMax,evalType = thresholds[variableName] + if evalType=="min": + if variableError" + subcategoryName + " - " + variableName + " - " + ": "+accuracyEval+"

") + html_file.write("
") + + + + + + ## ML subscore + + + mlSubscore=0 + + #Compute accuracy + accuracyMaxPoints=ratioRelevance["Accuracy"] + accuracyResult=sum([valueByColor[color] for color in accuracyResults["ML"]]) + accuracyResult=accuracyResult*accuracyMaxPoints/(len(accuracyResults["ML"])*max(valueByColor.values())) + mlSubscore+=accuracyResult + + #Compute speed-up + speedUpMaxPoints=ratioRelevance["Speed-up"] + speedUpResult=SpeedMetric(speedUp=speedUp["ML"],speedMax=maxSpeedRatioAllowed) + speedUpResult=speedUpResult*speedUpMaxPoints + mlSubscore+=speedUpResult + + ## compute Physics subscore + # Compute accuracy + accuracyResult=sum([valueByColor[color] for color in accuracyResults["Physics"]]) + accuracyResult=accuracyResult/(len(accuracyResults["Physics"])*max(valueByColor.values())) + physicsSubscore=accuracyResult + + ## Compute OOD subscore + + oodSubscore=0 + + #Compute accuracy + accuracyMaxPoints=ratioRelevance["Accuracy"] + accuracyResult=sum([valueByColor[color] for color in accuracyResults["OOD"]]) + accuracyResult=accuracyResult*accuracyMaxPoints/(len(accuracyResults["OOD"])*max(valueByColor.values())) + oodSubscore+=accuracyResult + + #Compute speed-up + speedUpMaxPoints=ratioRelevance["Speed-up"] + speedUpResult=SpeedMetric(speedUp=speedUp["OOD"],speedMax=maxSpeedRatioAllowed) + speedUpResult=speedUpResult*speedUpMaxPoints + oodSubscore+=speedUpResult + + ## Compute global score + globalScore=100*(coefficients["ML"]*mlSubscore+coefficients["Physics"]*physicsSubscore+coefficients["OOD"]*oodSubscore) + + print(globalScore) + + # Write scores in the output file + # exemple write score + score_file.write("global_warmup" + ": %0.12f\n" % globalScore) + score_file.write("ML_warmup" + ": %0.12f\n" % mlSubscore) + score_file.write("Physics_warmup" + ": %0.12f\n" % physicsSubscore) + score_file.write("OOD_warmup" + ": %0.12f\n" % oodSubscore) + + html_file.write("
") + html_file.write("

Global score: %0.12f

" % globalScore) + html_file.write("

ML score: %0.12f

" % mlSubscore) + html_file.write("

Physics score: %0.12f

" % physicsSubscore) + html_file.write("

OOD score: %0.12f

" % oodSubscore) + html_file.write("") + + html_file.close() + score_file.close() \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/scores.html b/score_scripts/NeurIPS_scoreV2_8/scores.html new file mode 100644 index 0000000..1b38d27 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/scores.html @@ -0,0 +1 @@ +Scoring program output

ML

ML - x-velocity - : g

ML - y-velocity - : g

ML - pressure - : o

ML - turbulent_viscosity - : g

ML - pressure_surfacic - : o


Physics

Physics - mean_relative_drag - : g

Physics - mean_relative_lift - : g

Physics - spearman_correlation_drag - : o

Physics - spearman_correlation_lift - : g


OOD

OOD - x-velocity - : g

OOD - y-velocity - : g

OOD - pressure - : r

OOD - turbulent_viscosity - : g

OOD - pressure_surfacic - : r

OOD - mean_relative_drag - : g

OOD - mean_relative_lift - : g

OOD - spearman_correlation_drag - : r

OOD - spearman_correlation_lift - : g



Global score: 76.765637772046

ML score: 0.764509111029

Physics score: 0.875000000000

OOD score: 0.664509111029

\ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/scores.txt b/score_scripts/NeurIPS_scoreV2_8/scores.txt new file mode 100644 index 0000000..81e03a4 --- /dev/null +++ b/score_scripts/NeurIPS_scoreV2_8/scores.txt @@ -0,0 +1,4 @@ +global_warmup: 76.765637772046 +ML_warmup: 0.764509111029 +Physics_warmup: 0.875000000000 +OOD_warmup: 0.664509111029 diff --git a/score_scripts/PowerGrid_score/metadata b/score_scripts/PowerGrid_score/metadata new file mode 100644 index 0000000..04b3b36 --- /dev/null +++ b/score_scripts/PowerGrid_score/metadata @@ -0,0 +1,2 @@ +command: python $program/score.py $input $output +description: Compute scores for the competition diff --git a/score_scripts/PowerGrid_score/res/json_metrics.json b/score_scripts/PowerGrid_score/res/json_metrics.json new file mode 100644 index 0000000..b3bb13f --- /dev/null +++ b/score_scripts/PowerGrid_score/res/json_metrics.json @@ -0,0 +1,197 @@ +{ + "test": { + "ML": { + "MSE_avg": { + "v_or": 0.00010645988368196413, + "v_ex": 0.00011296240700175986, + "p_or": 0.18411502242088318, + "p_ex": 0.1777658313512802, + "a_or": 16.51224136352539, + "a_ex": 16.86044692993164 + }, + "MAE_avg": { + "v_or": 0.001553428708575666, + "v_ex": 0.001500570448115468, + "p_or": 0.14518952369689941, + "p_ex": 0.14409466087818146, + "a_or": 1.4204329252243042, + "a_ex": 1.4652109146118164 + }, + "MAPE_avg": { + "v_or": 9.848606168816332e-06, + "v_ex": 1.0132289389730431e-05, + "p_or": 63017717760.0, + "p_ex": 22879956992.0, + "a_or": 121335185408.0, + "a_ex": 52108500992.0 + }, + "MAPE_90_avg": { + "v_or": 5.658699643585033e-06, + "v_ex": 5.919416245742559e-06, + "p_or": 0.0033098454767013486, + "p_ex": 0.0032964480543750145, + "a_or": 0.007386949784782199, + "a_ex": 0.007292245940119987 + }, + "MAPE_10_avg": { + "v_or": 7.93336312753172e-06, + "v_ex": 8.195988824322852e-06, + "p_or": 0.006397687786608211, + "p_ex": 0.006388048696331843, + "a_or": 0.021526892545594315, + "a_ex": 0.022162465019418712 + }, + "TIME_INF": 4.163906573085114 + }, + "Physics": { + "CURRENT_POS": { + "a_or": { + "Error": 10040.3349609375, + "Violation_proportion": 0.00044102150537634406 + }, + "a_ex": { + "Error": 11018.400390625, + "Violation_proportion": 0.0005930645161290322 + } + }, + "VOLTAGE_POS": { + "v_or": { + "Violation_proportion": 0.0 + }, + "v_ex": { + "Violation_proportion": 0.0 + } + }, + "LOSS_POS": { + "loss_criterion": 209.59898, + "violation_proportion": 0.008733548387096774 + }, + "DISC_LINES": { + "p_or": 0.0, + "p_ex": 0.0, + "a_or": 0.0, + "a_ex": 0.0, + "v_or": 0.0, + "v_ex": 0.0, + "violation_proportion": 0.0 + }, + "CHECK_LOSS": { + "violation_percentage": 1.539 + }, + "CHECK_GC": { + "violation_percentage": 0.0, + "mae": 9.128214e-05, + "wmape": 1.2492486e-06 + }, + "CHECK_LC": { + "violation_percentage": 77.35536440677966, + "mae": 0.10304196395080921, + "mape": 0.002859223697836113 + }, + "CHECK_JOULE_LAW": { + "violation_proportion": 0.0, + "mae": 3.054519562577347e-07, + "wmape": 2.268769940016548e-06 + } + }, + "IndRed": { + "TIME_INF": 3.9108043271116912 + } + }, + "test_ood_topo": { + "ML": { + "MSE_avg": { + "v_or": 0.0008741252822801471, + "v_ex": 0.0009632128058001399, + "p_or": 0.47769251465797424, + "p_ex": 0.4475671350955963, + "a_or": 40.1962890625, + "a_ex": 41.00724411010742 + }, + "MAE_avg": { + "v_or": 0.0023060280364006758, + "v_ex": 0.0023030198644846678, + "p_or": 0.18595275282859802, + "p_ex": 0.1843600869178772, + "a_or": 1.7880399227142334, + "a_ex": 1.8418012857437134 + }, + "MAPE_avg": { + "v_or": 1.5199337212834507e-05, + "v_ex": 1.5929399523884058e-05, + "p_or": 189589094400.0, + "p_ex": 56932474880.0, + "a_or": 386140471296.0, + "a_ex": 157286449152.0 + }, + "MAPE_90_avg": { + "v_or": 7.414690380649798e-06, + "v_ex": 8.032887808804589e-06, + "p_or": 0.004977200835027814, + "p_ex": 0.004950829903700901, + "a_or": 0.010261113576488012, + "a_ex": 0.01012922324923192 + }, + "MAPE_10_avg": { + "v_or": 1.0476555723965095e-05, + "v_ex": 1.1106274117571183e-05, + "p_or": 0.008155790825441843, + "p_ex": 0.008147190351295156, + "a_or": 0.027774648502433164, + "a_ex": 0.028373457148515593 + } + }, + "Physics": { + "CURRENT_POS": { + "a_or": { + "Error": 21769.7578125, + "Violation_proportion": 0.0004038709677419355 + }, + "a_ex": { + "Error": 23943.244140625, + "Violation_proportion": 0.0005124731182795699 + } + }, + "VOLTAGE_POS": { + "v_or": { + "Violation_proportion": 0.0 + }, + "v_ex": { + "Violation_proportion": 0.0 + } + }, + "LOSS_POS": { + "loss_criterion": 1076.4073, + "violation_proportion": 0.00829997311827957 + }, + "DISC_LINES": { + "p_or": 0.0, + "p_ex": 0.0, + "a_or": 0.0, + "a_ex": 0.0, + "v_or": 0.0, + "v_ex": 0.0, + "violation_proportion": 0.0 + }, + "CHECK_LOSS": { + "violation_percentage": 3.2485 + }, + "CHECK_GC": { + "violation_percentage": 0.0, + "mae": 9.088051e-05, + "wmape": 1.2194811e-06 + }, + "CHECK_LC": { + "violation_percentage": 83.45940254237289, + "mae": 0.14096058704044898, + "mape": 0.0039999913278262085 + }, + "CHECK_JOULE_LAW": { + "violation_proportion": 0.0, + "mae": 2.9970232693607007e-07, + "wmape": 2.1807441470482367e-06 + } + }, + "IndRed": {} + } +} \ No newline at end of file diff --git a/score_scripts/PowerGrid_score/res/json_metrics_new.json b/score_scripts/PowerGrid_score/res/json_metrics_new.json new file mode 100644 index 0000000..8a4b677 --- /dev/null +++ b/score_scripts/PowerGrid_score/res/json_metrics_new.json @@ -0,0 +1,39 @@ +{ + "ML": { + "a_ex": 0.007292245940119987, + "a_or": 0.007386949784782199, + "p_ex": 0.006388048696331843, + "p_or": 0.006397687786608211, + "v_ex": 0.001500570448115468, + "v_or": 0.001553428708575666}, + "Physics": { + "CHECK_GC": 0.0, + "CHECK_JOULE_LAW": 0.0, + "CHECK_LC": 77.35536440677966, + "CHECK_LOSS": 1.539, + "CURRENT_POS": 0.04410215053763441, + "DISC_LINES": 0.0, + "LOSS_POS": 0.8733548387096773, + "VOLTAGE_POS": 0.0}, + "Speed" : {"inference_time": 699.8024816513062}, + "OOD": { + "ML": { + "a_ex": 0.01012922324923192, + "a_or": 0.010261113576488012, + "p_ex": 0.008147190351295156, + "p_or": 0.008155790825441843, + "v_ex": 0.0023030198644846678, + "v_or": 0.0023060280364006758}, + "Physics": { + "CHECK_GC": 0.0, + "CHECK_JOULE_LAW": 0.0, + "CHECK_LC": 83.45940254237289, + "CHECK_LOSS": 3.2485, + "CURRENT_POS": 0.04038709677419355, + "DISC_LINES": 0.0, + "LOSS_POS": 0.8299973118279571, + "VOLTAGE_POS": 0.0} + } +} + + diff --git a/score_scripts/PowerGrid_score/score.py b/score_scripts/PowerGrid_score/score.py new file mode 100644 index 0000000..e230b1f --- /dev/null +++ b/score_scripts/PowerGrid_score/score.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# Use directly the outputfile from the LIPS evaluation program. +# Some libraries and options +import os +from sys import argv +import sys +import json +import math + +import utils.compute_score as cs + +# Default I/O directories: +root_dir = "/app/" +default_input_dir = root_dir + "scoring_input" +default_output_dir = root_dir + "scoring_output" +default_input_dir = "./" +default_output_dir="./" + + + +def _HERE(*args): + h = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(h, *args) + +class ModelApiError(Exception): + """Model api error""" + + def __init__(self, msg=""): + self.msg = msg + print(msg) + + +def import_metrics(input_dir): + ## import parameters.json as a dictionary + path_submission_parameters = os.path.join(input_dir, 'res', 'json_metrics.json') + # path_submission_parameters = os.path.join(input_dir, 'json_metrics.json') + print(path_submission_parameters) + if not os.path.exists(path_submission_parameters): + raise ModelApiError("Missing json_metrics.json file") + exit_program() + with open(path_submission_parameters) as json_file: + metrics = json.load(json_file) + return metrics + + + +# =============================== MAIN ======================================== + +if __name__ == "__main__": + + #### INPUT/OUTPUT: Get input and output directory names + if len(argv) == 1: # Use the default input and output directories if no arguments are provided + input_dir = default_input_dir + output_dir = default_output_dir + else: + input_dir = argv[1] + output_dir = argv[2] + # Create the output directory, if it does not already exist and open output files + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + + + score_file = open(os.path.join(output_dir, 'scores.txt'), 'w') + html_file = open(os.path.join(output_dir, 'scores.html'), 'w') + print("Scoring file : ", os.path.join(output_dir, 'scores.txt')) + metrics = import_metrics(input_dir) + metrics["solver_time"] = 32.79 + + globalScore, test_ml_subscore, test_physics_subscore, test_ood_ml_subscore, test_ood_physics_subscore, speedup_score = cs.compute_global_score(metrics) + + print("scoring done ", globalScore, test_ml_subscore, test_physics_subscore, test_ood_ml_subscore, test_ood_physics_subscore, speedup_score) + + score_file.write("global" + ": %0.12f\n" % globalScore) + score_file.write("ML_test" + ": %0.12f\n" % test_ml_subscore) + score_file.write("Physics_test" + ": %0.12f\n" % test_physics_subscore) + score_file.write("ML_ood" + ": %0.12f\n" % test_ood_ml_subscore) + score_file.write("Physics_ood" + ": %0.12f\n" % test_ood_physics_subscore) + score_file.write("speedup" + ": %0.12f\n" % speedup_score) + + html_file.write("Scoring program output") + html_file.write("

Global Score :

") + html_file.write("

" + ": %0.12f\n" % globalScore) + html_file.write("

") + + html_file.close() + score_file.close() \ No newline at end of file diff --git a/score_scripts/PowerGrid_score/scores.html b/score_scripts/PowerGrid_score/scores.html new file mode 100644 index 0000000..e69de29 diff --git a/score_scripts/PowerGrid_score/scores.txt b/score_scripts/PowerGrid_score/scores.txt new file mode 100644 index 0000000..e69de29 diff --git a/score_scripts/PowerGrid_score/utils/__init__.py b/score_scripts/PowerGrid_score/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/score_scripts/PowerGrid_score/utils/compute_score.py b/score_scripts/PowerGrid_score/utils/compute_score.py new file mode 100644 index 0000000..cea5ff6 --- /dev/null +++ b/score_scripts/PowerGrid_score/utils/compute_score.py @@ -0,0 +1,162 @@ +import math +from lips.metrics.power_grid.compute_solver_time import compute_solver_time +from lips.metrics.power_grid.compute_solver_time_grid2op import compute_solver_time_grid2op + +thresholds={"a_or":(0.02,0.05,"min"), + "a_ex":(0.02,0.05,"min"), + "p_or":(0.02,0.05,"min"), + "p_ex":(0.02,0.05,"min"), + "v_or":(0.2,0.5,"min"), + "v_ex":(0.2,0.5,"min"), + "CURRENT_POS":(1., 5.,"min"), + "VOLTAGE_POS":(1.,5.,"min"), + "LOSS_POS":(1.,5.,"min"), + "DISC_LINES":(1.,5.,"min"), + "CHECK_LOSS":(1.,5.,"min"), + "CHECK_GC":(0.05,0.10,"min"), + "CHECK_LC":(0.05,0.10,"min"), + "CHECK_JOULE_LAW":(1.,5.,"min") + } + +configuration={ + "coefficients":{"test":0.3, "test_ood":0.3, "speed_up":0.4}, + "test_ratio":{"ml": 0.66, "physics":0.34}, + "test_ood_ratio":{"ml": 0.66, "physics":0.34}, + "value_by_color":{"g":2,"o":1,"r":0}, + "max_speed_ratio_allowed":50 +} + +def evaluate_model(benchmark, model): + metrics = benchmark.evaluate_simulator(augmented_simulator=model, + eval_batch_size=128, + dataset="all", + shuffle=False, + save_path=None, + save_predictions=False + ) + return metrics + +def compute_speed_up_metrics( metrics): + speed_up = metrics["solver_time"] / metrics["test"]["ML"]["TIME_INF"] + return speed_up + +def compute_speed_up(config, metrics): + # solver_time = compute_solver_time(nb_samples=int(1e5), config=config) + solver_time = compute_solver_time_grid2op(config_path=config.path_config, benchmark_name=config.section_name, nb_samples=int(1e5)) + speed_up = solver_time / metrics["test"]["ML"]["TIME_INF"] + return speed_up + +def reconstruct_metric_dict(metrics, dataset: str="test"): + rec_metrics = dict() + rec_metrics["ML"] = dict() + rec_metrics["Physics"] = dict() + + rec_metrics["ML"]["a_or"] = metrics[dataset]["ML"]["MAPE_90_avg"]["a_or"] + rec_metrics["ML"]["a_ex"] = metrics[dataset]["ML"]["MAPE_90_avg"]["a_ex"] + rec_metrics["ML"]["p_or"] = metrics[dataset]["ML"]["MAPE_10_avg"]["p_or"] + rec_metrics["ML"]["p_ex"] = metrics[dataset]["ML"]["MAPE_10_avg"]["p_ex"] + rec_metrics["ML"]["v_or"] = metrics[dataset]["ML"]["MAE_avg"]["v_or"] + rec_metrics["ML"]["v_ex"] = metrics[dataset]["ML"]["MAE_avg"]["v_ex"] + + rec_metrics["Physics"]["CURRENT_POS"] = metrics[dataset]["Physics"]["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100. + rec_metrics["Physics"]["VOLTAGE_POS"] = metrics[dataset]["Physics"]["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100. + rec_metrics["Physics"]["LOSS_POS"] = metrics[dataset]["Physics"]["LOSS_POS"]["violation_proportion"] * 100. + rec_metrics["Physics"]["DISC_LINES"] = metrics[dataset]["Physics"]["DISC_LINES"]["violation_proportion"] * 100. + rec_metrics["Physics"]["CHECK_LOSS"] = metrics[dataset]["Physics"]["CHECK_LOSS"]["violation_percentage"] + rec_metrics["Physics"]["CHECK_GC"] = metrics[dataset]["Physics"]["CHECK_GC"]["violation_percentage"] + rec_metrics["Physics"]["CHECK_LC"] = metrics[dataset]["Physics"]["CHECK_LC"]["violation_percentage"] + rec_metrics["Physics"]["CHECK_JOULE_LAW"] = metrics[dataset]["Physics"]["CHECK_JOULE_LAW"]["violation_proportion"] * 100. + + return rec_metrics + +def discretize_results(metrics): + results=dict() + for subcategoryName, subcategoryVal in metrics.items(): + results[subcategoryName]=[] + for variableName, variableError in subcategoryVal.items(): + thresholdMin,thresholdMax,evalType=thresholds[variableName] + if evalType=="min": + if variableError Date: Wed, 19 Feb 2025 16:24:38 +0100 Subject: [PATCH 08/18] make the scoring feature more generic --- .../powergrid/scoring/ScoreConfig.ini | 30 ++++- lips/scoring/powergrid_scoring.py | 2 + lips/scoring/scoring.py | 117 ++++++++++++++---- lips/tests/scoring/test_scoring.py | 51 ++++++++ 4 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 lips/tests/scoring/test_scoring.py diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index 569105b..d86a305 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,8 +1,34 @@ [DEFAULT] -Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} +ValueByColor = {"green": 2, "orange": 1, "red": 0} -ValueByColor = {"g": 2, "o": 1, "r": 0} +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}} +[USE CASE 1] +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} SpeedConfig = {"speed_score_formula" :"PowerGrid Competition", "max_speed_ratio_allowed" : 10000, "reference_mean_simulation_time": 1500 diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index b4bc1e8..843cfca 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -57,6 +57,7 @@ def _calculate_score_color(metrics, thresholds): if isinstance(value, dict): tree[key] = PowerGridScoring._calculate_score_color(value, thresholds) else: + discrete_metric = PowerGridScoring._discretize_metric(metric_name=key, metric_value=value, thresholds=thresholds) tree[key] = discrete_metric @@ -215,3 +216,4 @@ def reconstruct_physic_metrics(input_json, physic_key_path, competition_name, us raise ValueError(f'{competition_name} not in options') return {"Physics": physic_metrics} + diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py index 150150c..eaf3db4 100644 --- a/lips/scoring/scoring.py +++ b/lips/scoring/scoring.py @@ -1,34 +1,105 @@ # Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) # See AUTHORS.txt # This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. -# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, -# you can obtain one at http://mozilla.org/MPL/2.0/. # SPDX-License-Identifier: MPL-2.0 -# This file is part of LIPS, LIPS is a python platform for power networks benchmarking -from abc import ABC, abstractmethod -from typing import Union + +import bisect +from abc import ABC +from typing import Union, List, Dict, Any from ..config import ConfigManager from ..logger import CustomLogger +# Constants +VALID_COMPARISONS = {"minimize", "maximize"} + class Scoring(ABC): + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the Scoring instance, loading configuration and setting up logger. + + Args: + config (ConfigManager, optional): A ConfigManager instance. Defaults to None. + config_path (str, optional): Path to the configuration file. Defaults to None. + config_section (str, optional): Section of the configuration file. Defaults to None. + log_path (str, optional): Path to the log file. Defaults to None. + """ + self.config = config if config else ConfigManager(section_name=config_section, path=config_path) + self.logger = CustomLogger(__class__.__name__, log_path).logger + + self.thresholds = self.config.get_option("thresholds") + self.value_by_color = self.config.get_option("valuebycolor") + + self._validate_thresholds_config() + + def calculate_score_color(self, metrics: Dict[str, Any]) -> Dict[str, Any]: + """ + Recursively calculates the score color for each metric in a tree metrics dictionary. + + Args: + metrics: A dictionary of metric names and their corresponding values. + + Returns: + A dictionary with the same structure as `metrics` but with colorized values. + """ + tree = {} + + for key, value in metrics.items(): + if isinstance(value, dict): + tree[key] = self.calculate_score_color(value) + else: + threshold_data = self.thresholds[key] + comparison_type = threshold_data["comparison_type"] + metric_thresholds = threshold_data["thresholds"] + discrete_metric = self.colorize_metric_value(metric_value=value, comparison_type=comparison_type, + thresholds=metric_thresholds) + tree[key] = discrete_metric + + return tree + + def colorize_metric_value(self, metric_value: float, comparison_type: str, thresholds: List[float]) -> str: + """ + Assigns a color based on the metric value, its comparison type (minimize or maximize), and thresholds. + + Args: + metric_value (float): The value of the metric to be colorized. + comparison_type (str): Either 'minimize' or 'maximize' to indicate how the threshold comparison is performed. + thresholds (List[float]): A sorted list of threshold values. + + Returns: + str: The color corresponding to the metric value based on its threshold position. + + Raises: + ValueError: If the comparison_type is not 'minimize' or 'maximize'. + """ + if comparison_type not in VALID_COMPARISONS: + raise ValueError("comparison_type must be 'minimize' or 'maximize'") + + value_position = bisect.bisect_left(thresholds, metric_value) + + color_by_threshold = list(self.value_by_color.keys()) + + return color_by_threshold[value_position] if comparison_type == "minimize" else color_by_threshold[ + - (value_position + 1)] + + def _validate_thresholds_config(self) -> None: + """ + Validates the thresholds configuration against the value_by_color length. + + Raises: + ValueError: If the thresholds configuration is invalid or missing required data. + """ + if not self.thresholds: + raise ValueError("Thresholds configuration is missing.") + if not self.value_by_color: + raise ValueError("Value by color configuration is missing.") - def __init__(self, - config: Union[ConfigManager, None] = None, - config_path: Union[str, None] = None, - config_section: Union[str, None] = None, - log_path: Union[str, None] = None - ): - if config is None: - self.config = ConfigManager(section_name=config_section, path=config_path) - else: - self.config = config - - # logger - self.log_path = log_path - self.logger = CustomLogger(__class__.__name__, self.log_path).logger - - @abstractmethod - def scoring(self): - pass + expected_len = len(self.value_by_color) - 1 + for metric_name, threshold_data in self.thresholds.items(): + if not isinstance(threshold_data, dict) or "thresholds" not in threshold_data: + raise ValueError(f"Invalid format for thresholds data for metric '{metric_name}'.") + if len(threshold_data["thresholds"]) != expected_len: + raise ValueError( + f"Metric '{metric_name}': Thresholds list length must be equal to ValueByColor length -1: i.e: {expected_len}.") diff --git a/lips/tests/scoring/test_scoring.py b/lips/tests/scoring/test_scoring.py new file mode 100644 index 0000000..9368894 --- /dev/null +++ b/lips/tests/scoring/test_scoring.py @@ -0,0 +1,51 @@ +import unittest +from unittest.mock import MagicMock +from lips.scoring import Scoring + + +class TestScoring(unittest.TestCase): + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + }, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + }[key] + self.scoring = Scoring(config=self.mock_config) + + def test_colorize_metric_value_minimize(self): + self.assertEqual(self.scoring.colorize_metric_value(0.01, "minimize", [0.02, 0.05]), "green") + self.assertEqual(self.scoring.colorize_metric_value(0.03, "minimize", [0.02, 0.05]), "orange") + self.assertEqual(self.scoring.colorize_metric_value(0.06, "minimize", [0.02, 0.05]), "red") + + def test_colorize_metric_value_maximize(self): + self.assertEqual(self.scoring.colorize_metric_value(0.95, "maximize", [0.8, 0.9]), "green") + self.assertEqual(self.scoring.colorize_metric_value(0.85, "maximize", [0.8, 0.9]), "orange") + self.assertEqual(self.scoring.colorize_metric_value(0.75, "maximize", [0.8, 0.9]), "red") + + def test_calculate_score_color(self): + metrics = {"a_or": 0.03, "spearman_correlation_drag": 0.85, "inference_time": 600} + expected_output = {"a_or": "orange", "spearman_correlation_drag": "orange", "inference_time": "orange"} + self.assertEqual(self.scoring.calculate_score_color(metrics), expected_output) + + def test_invalid_comparison_type(self): + with self.assertRaises(ValueError): + self.scoring.colorize_metric_value(0.5, "invalid_type", [0.1, 0.2]) + + def test_validate_thresholds_config_missing(self): + self.mock_config.get_option.side_effect = lambda key: None if key == "thresholds" else {"green": 2, "orange": 1, + "red": 0} + with self.assertRaises(ValueError): + Scoring(config=self.mock_config) + + def test_validate_thresholds_config_mismatch(self): + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + }[key] + with self.assertRaises(ValueError): + Scoring(config=self.mock_config) + From 2eeffa8fec0ae7152d908163677fbcc388894ce0 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Fri, 21 Feb 2025 14:18:18 +0100 Subject: [PATCH 09/18] add new global score functions to the Scoring class --- .../powergrid/scoring/ScoreConfig.ini | 51 ++--- lips/scoring/powergrid_scoring.py | 4 +- lips/scoring/scoring.py | 131 ++++++++----- lips/tests/scoring/test_scoring.py | 177 ++++++++++++++---- 4 files changed, 258 insertions(+), 105 deletions(-) diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index d86a305..8fbe462 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,34 +1,35 @@ [DEFAULT] ValueByColor = {"green": 2, "orange": 1, "red": 0} +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} + Thresholds = { - "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, - "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, - "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, - "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, - "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, - "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, - "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, - "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, - "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, - "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, - "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, - "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, - "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, - "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, - "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, - "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, - "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, - "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, - "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}} + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}} [USE CASE 1] -Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} SpeedConfig = {"speed_score_formula" :"PowerGrid Competition", "max_speed_ratio_allowed" : 10000, "reference_mean_simulation_time": 1500 diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 843cfca..687dfa4 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -41,7 +41,7 @@ def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = for key in self.coefficients: if key in score_color: flat_dict = utils.flatten_dict(score_color[key]) - score_values[key] = self._calculate_sub_score(flat_dict.values()) + score_values[key] = self._calculate_leaf_score(flat_dict.values()) score_values["Speed"] = speed_score # calculate global score value @@ -63,7 +63,7 @@ def _calculate_score_color(metrics, thresholds): tree[key] = discrete_metric return tree - def _calculate_sub_score(self, colors: List[str]): + def _calculate_leaf_score(self, colors: List[str]): s = sum([self.value_by_color[color] for color in colors]) return s / (len(colors) * max(self.value_by_color.values())) diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py index eaf3db4..d2d1bb5 100644 --- a/lips/scoring/scoring.py +++ b/lips/scoring/scoring.py @@ -15,91 +15,136 @@ class Scoring(ABC): + """ + Abstract base class for calculating scores based on metrics and thresholds. + """ + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, config_section: Union[str, None] = None, log_path: Union[str, None] = None): """ - Initializes the Scoring instance, loading configuration and setting up logger. + Initializes the Scoring instance with configuration and logger. Args: - config (ConfigManager, optional): A ConfigManager instance. Defaults to None. - config_path (str, optional): Path to the configuration file. Defaults to None. - config_section (str, optional): Section of the configuration file. Defaults to None. - log_path (str, optional): Path to the log file. Defaults to None. + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. """ self.config = config if config else ConfigManager(section_name=config_section, path=config_path) self.logger = CustomLogger(__class__.__name__, log_path).logger self.thresholds = self.config.get_option("thresholds") self.value_by_color = self.config.get_option("valuebycolor") + self.coefficients = self.config.get_option("coefficients") + self._validate_configuration() - self._validate_thresholds_config() - - def calculate_score_color(self, metrics: Dict[str, Any]) -> Dict[str, Any]: + def colorize_metrics(self, metrics: Dict[str, Any]) -> Dict[str, Any]: """ - Recursively calculates the score color for each metric in a tree metrics dictionary. + Recursively colorizes metric values based on thresholds. Args: - metrics: A dictionary of metric names and their corresponding values. + metrics: A dictionary of metric names and their corresponding values (can be nested). Returns: A dictionary with the same structure as `metrics` but with colorized values. """ - tree = {} - + colorized_metrics = {} for key, value in metrics.items(): if isinstance(value, dict): - tree[key] = self.calculate_score_color(value) + colorized_metrics[key] = self.colorize_metrics(value) else: - threshold_data = self.thresholds[key] - comparison_type = threshold_data["comparison_type"] - metric_thresholds = threshold_data["thresholds"] - discrete_metric = self.colorize_metric_value(metric_value=value, comparison_type=comparison_type, - thresholds=metric_thresholds) - tree[key] = discrete_metric - - return tree + colorized_metrics[key] = self._colorize_metric_value(key, value) + return colorized_metrics - def colorize_metric_value(self, metric_value: float, comparison_type: str, thresholds: List[float]) -> str: + def _colorize_metric_value(self, metric_name: str, metric_value: float) -> str: """ - Assigns a color based on the metric value, its comparison type (minimize or maximize), and thresholds. + Assigns a color to a single metric value based on its threshold. Args: - metric_value (float): The value of the metric to be colorized. - comparison_type (str): Either 'minimize' or 'maximize' to indicate how the threshold comparison is performed. - thresholds (List[float]): A sorted list of threshold values. + metric_name: The name of the metric. + metric_value: The value of the metric. Returns: - str: The color corresponding to the metric value based on its threshold position. + The color corresponding to the metric value. Raises: - ValueError: If the comparison_type is not 'minimize' or 'maximize'. + ValueError: If the comparison type is invalid. """ - if comparison_type not in VALID_COMPARISONS: - raise ValueError("comparison_type must be 'minimize' or 'maximize'") - - value_position = bisect.bisect_left(thresholds, metric_value) + threshold_data = self.thresholds[metric_name] + comparison_type = threshold_data["comparison_type"] + thresholds = threshold_data["thresholds"] - color_by_threshold = list(self.value_by_color.keys()) + if comparison_type not in VALID_COMPARISONS: + raise ValueError(f"Invalid comparison type: {comparison_type}. Must be 'minimize' or 'maximize'.") - return color_by_threshold[value_position] if comparison_type == "minimize" else color_by_threshold[ - - (value_position + 1)] + index = bisect.bisect_left(thresholds, metric_value) + colors = list(self.value_by_color.keys()) + return colors[index] if comparison_type == "minimize" else colors[-(index + 1)] - def _validate_thresholds_config(self) -> None: + def _validate_configuration(self) -> None: """ - Validates the thresholds configuration against the value_by_color length. + Validates the thresholds and value_by_color configurations. Raises: - ValueError: If the thresholds configuration is invalid or missing required data. + ValueError: If the configuration is invalid. """ if not self.thresholds: raise ValueError("Thresholds configuration is missing.") if not self.value_by_color: raise ValueError("Value by color configuration is missing.") - expected_len = len(self.value_by_color) - 1 + expected_threshold_count = len(self.value_by_color) - 1 for metric_name, threshold_data in self.thresholds.items(): - if not isinstance(threshold_data, dict) or "thresholds" not in threshold_data: - raise ValueError(f"Invalid format for thresholds data for metric '{metric_name}'.") - if len(threshold_data["thresholds"]) != expected_len: + if not isinstance(threshold_data, + dict) or "thresholds" not in threshold_data or "comparison_type" not in threshold_data: raise ValueError( - f"Metric '{metric_name}': Thresholds list length must be equal to ValueByColor length -1: i.e: {expected_len}.") + f"Invalid thresholds data for metric '{metric_name}'. Must be a dict with 'thresholds' and 'comparison_type' keys.") + if len(threshold_data["thresholds"]) != expected_threshold_count: + raise ValueError( + f"Metric '{metric_name}': Thresholds count must be {expected_threshold_count} (length of ValueByColor - 1).") + + def _calculate_leaf_score(self, colors: List[str]) -> float: + """Calculates the score for a leaf node (set of colorized metrics).""" + return sum(self.value_by_color[color] for color in colors) / (len(colors) * max(self.value_by_color.values())) + + def calculate_sub_scores(self, node: Dict[str, Any]) -> Union[float, Dict[str, Any]]: + """ + Calculates sub-scores recursively for a node in the metrics tree. + + Args: + node: A node in the metrics tree (can be a leaf or a sub-tree). + + Returns: + The sub-score for the node (float for leaf, dict for sub-tree). + + Raises: + ValueError: If the input JSON is not a dictionary or if a parent node is inconsistently branched. + """ + if not isinstance(node, dict): + raise ValueError("Input must be a dictionary.") + + if all(isinstance(value, str) for value in node.values()): # Leaf node + return self._calculate_leaf_score(list(node.values())) + elif any(isinstance(value, str) for value in node.values()): # Inconsistent branching + raise ValueError("Parent node is not uniformly branched (mix of leaf and sub-tree children).") + else: # Sub-tree node + return {key: self.calculate_sub_scores(value) for key, value in node.items()} + + def calculate_global_score(self, tree: Union[float, Dict[str, Any]]) -> float: + """ + Calculates the global score for the entire metrics tree. + + Args: + tree: a pre-calculated sub-score tree + + Returns: + The global score. + """ + if isinstance(tree, (int, float)): # Base case: already a sub-score + return tree + + global_score = 0 + for key, subtree in tree.items(): + weight = self.coefficients.get(key, 1) # Default weight is 1 + global_score += weight * self.calculate_global_score(subtree) + return global_score diff --git a/lips/tests/scoring/test_scoring.py b/lips/tests/scoring/test_scoring.py index 9368894..09713b1 100644 --- a/lips/tests/scoring/test_scoring.py +++ b/lips/tests/scoring/test_scoring.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import MagicMock + from lips.scoring import Scoring @@ -7,45 +8,151 @@ class TestScoring(unittest.TestCase): def setUp(self): self.mock_config = MagicMock() self.mock_config.get_option.side_effect = lambda key: { - "thresholds": { - "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, - "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, - "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, - }, + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, }, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, - }[key] + "coefficients": {"ML": 0.3, "OOD": 0.3, "Physics": 0.3, "Speed": 0.1}}[key] self.scoring = Scoring(config=self.mock_config) - def test_colorize_metric_value_minimize(self): - self.assertEqual(self.scoring.colorize_metric_value(0.01, "minimize", [0.02, 0.05]), "green") - self.assertEqual(self.scoring.colorize_metric_value(0.03, "minimize", [0.02, 0.05]), "orange") - self.assertEqual(self.scoring.colorize_metric_value(0.06, "minimize", [0.02, 0.05]), "red") - - def test_colorize_metric_value_maximize(self): - self.assertEqual(self.scoring.colorize_metric_value(0.95, "maximize", [0.8, 0.9]), "green") - self.assertEqual(self.scoring.colorize_metric_value(0.85, "maximize", [0.8, 0.9]), "orange") - self.assertEqual(self.scoring.colorize_metric_value(0.75, "maximize", [0.8, 0.9]), "red") - - def test_calculate_score_color(self): - metrics = {"a_or": 0.03, "spearman_correlation_drag": 0.85, "inference_time": 600} - expected_output = {"a_or": "orange", "spearman_correlation_drag": "orange", "inference_time": "orange"} - self.assertEqual(self.scoring.calculate_score_color(metrics), expected_output) - - def test_invalid_comparison_type(self): - with self.assertRaises(ValueError): - self.scoring.colorize_metric_value(0.5, "invalid_type", [0.1, 0.2]) - - def test_validate_thresholds_config_missing(self): - self.mock_config.get_option.side_effect = lambda key: None if key == "thresholds" else {"green": 2, "orange": 1, - "red": 0} - with self.assertRaises(ValueError): + def test_colorize_metrics(self): + metrics = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, + "OOD": {"a_or": 0.03, "inference_time": 400}, + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.8}, "Speed": {"inference_time": 600}} + expected_colorized_metrics = {"ML": {"a_or": "green", "spearman_correlation_drag": "green"}, + "OOD": {"a_or": "orange", "inference_time": "green"}, + "Physics": {"a_or": "red", "spearman_correlation_drag": "red"}, + "Speed": {"inference_time": "orange"}} + colorized_metrics = self.scoring.colorize_metrics(metrics) + self.assertEqual(colorized_metrics, expected_colorized_metrics) + + def test__colorize_metric_value(self): + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.01), "green") + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.04), "orange") + self.assertEqual(self.scoring._colorize_metric_value("a_or", 0.06), "red") + + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.92), "green") + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.85), "orange") + self.assertEqual(self.scoring._colorize_metric_value("spearman_correlation_drag", 0.7), "red") + + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 450), "green") + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 750), "orange") + self.assertEqual(self.scoring._colorize_metric_value("inference_time", 1100), "red") + + def test__colorize_metric_value_invalid_comparison_type(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "invalid", "thresholds": [0.02, 0.05]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[ + key] # empty coefficients to avoid other errors. + self.scoring = Scoring(config=self.mock_config) # recreate scoring object with invalid config. + with self.assertRaises(ValueError) as context: + self.scoring._colorize_metric_value("a_or", 0.01) + self.assertIn("Invalid comparison type", str(context.exception)) + + def test__validate_configuration_missing_thresholds(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": None, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: Scoring(config=self.mock_config) + self.assertIn("Thresholds configuration is missing", str(context.exception)) - def test_validate_thresholds_config_mismatch(self): - self.mock_config.get_option.side_effect = lambda key: { - "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02]}}, - "valuebycolor": {"green": 2, "orange": 1, "red": 0}, - }[key] - with self.assertRaises(ValueError): + def test__validate_configuration_missing_valuebycolor(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}}, "valuebycolor": None, + "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn("Value by color configuration is missing", str(context.exception)) + + def test__validate_configuration_invalid_threshold_data(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"thresholds": [0.02, 0.05]}}, # Missing comparison_type + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: Scoring(config=self.mock_config) + self.assertIn( + "Invalid thresholds data for metric 'a_or'. Must be a dict with 'thresholds' and 'comparison_type' keys.", + str(context.exception)) + + def test__validate_configuration_invalid_threshold_count(self): + self.mock_config.get_option.side_effect = lambda key: \ + {"thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05, 0.1]}}, + # Too many thresholds + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {}}[key] + with self.assertRaises(ValueError) as context: + Scoring(config=self.mock_config) + self.assertIn("Metric 'a_or': Thresholds count must be 2 (length of ValueByColor - 1).", str(context.exception)) + + def test__calculate_leaf_score(self): + colors = ["green", "orange", "red"] + expected_score = (2 + 1 + 0) / (3 * 2) # (sum of values) / (number of colors * max value) + self.assertEqual(self.scoring._calculate_leaf_score(colors), expected_score) + + def test_calculate_sub_scores_leaf(self): + node = {"a_or": "green", "spearman_correlation_drag": "orange"} + expected_score = self.scoring._calculate_leaf_score(list(node.values())) + self.assertEqual(self.scoring.calculate_sub_scores(node), expected_score) + + def test_calculate_sub_scores_subtree(self): + node = {"ML": {"a_or": "green", "spearman_correlation_drag": "orange"}, "OOD": {"inference_time": "red"}} + expected_scores = {"ML": self.scoring._calculate_leaf_score(list(node["ML"].values())), + "OOD": self.scoring._calculate_leaf_score(list(node["OOD"].values()))} + self.assertEqual(self.scoring.calculate_sub_scores(node), expected_scores) + + def test_calculate_sub_scores_invalid_input(self): + with self.assertRaises(ValueError) as context: + self.scoring.calculate_sub_scores("not a dict") + self.assertIn("Input must be a dictionary.", str(context.exception)) + + def test_calculate_sub_scores_inconsistent_branching(self): + node = {"a": "green", "b": {"c": "red"}} + with self.assertRaises(ValueError) as context: + self.scoring.calculate_sub_scores(node) + self.assertIn("Parent node is not uniformly branched", str(context.exception)) + + def test_calculate_global_score_subtree(self): + tree = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, "OOD": {"inference_time": 400}, + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.75}, "Speed": {"inference_time": 600}, } + tree_score = {"ML": 1, "OOD": 1, "Physics": 0, "Speed": 0.5} + + # Calculate expected score manually based on coefficients and leaf scores + ml_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["ML"]).values())) + ood_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["OOD"]).values())) + physics_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["Physics"]).values())) + speed_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["Speed"]).values())) + + expected_global_score = (0.3 * ml_score + 0.3 * ood_score + 0.3 * physics_score + 0.1 * speed_score) + + global_score = self.scoring.calculate_global_score(tree_score) # Operate on the original tree + + self.assertAlmostEqual(global_score, expected_global_score, places=6) # Use assertAlmostEqual + + def test_calculate_global_score_subtree_with_missing_coefficient(self): + tree = {"ML": {"a_or": 0.01, "spearman_correlation_drag": 0.95}, # Original float values + "OOD": {"inference_time": 400}, # Original float values + "Physics": {"a_or": 0.06, "spearman_correlation_drag": 0.75}, # Original float values + "Speed": {"inference_time": 600}, # Original float values + "NewComponent": {"a_or": 0.03} # Original float values + } + tree_score = {"ML": 1, "OOD": 1, "Physics": 0, "Speed": 0.5, "NewComponent": 0.5} + + ml_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["ML"]).values())) + ood_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["OOD"]).values())) + physics_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["Physics"]).values())) + speed_score = self.scoring._calculate_leaf_score(list(self.scoring.colorize_metrics(tree["Speed"]).values())) + new_component_score = self.scoring._calculate_leaf_score( + list(self.scoring.colorize_metrics(tree["NewComponent"]).values())) + + expected_global_score = ( + 0.3 * ml_score + 0.3 * ood_score + 0.3 * physics_score + 0.1 * speed_score + new_component_score + # Default weight of 1 + ) + + global_score = self.scoring.calculate_global_score(tree_score) # Operate on the original tree + self.assertAlmostEqual(global_score, expected_global_score, places=6) + +if __name__ == '__main__': + unittest.main() From 449141eed605c2e936edd305c8dff8430cf49138 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Mon, 24 Feb 2025 18:14:38 +0100 Subject: [PATCH 10/18] score compute functionality for Airfoil competition --- .gitignore | 5 +- .../powergrid/scoring/ScoreConfig.ini | 6 +- lips/scoring/__init__.py | 3 +- lips/scoring/airfoil_powergrid_scoring.py | 173 ++++++++++++++++++ lips/scoring/scoring.py | 3 +- lips/scoring/utils.py | 4 +- lips/tests/scoring/test_scoring.py | 5 +- 7 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 lips/scoring/airfoil_powergrid_scoring.py diff --git a/.gitignore b/.gitignore index 1d40201..ef7fcd8 100644 --- a/.gitignore +++ b/.gitignore @@ -212,4 +212,7 @@ lips/tests/data/powergrid/l2rpn_idf_2023 logs.log lips_logs.log -trained_models/ \ No newline at end of file +trained_models/ +/draft.py +/score_scripts/NeurIPS_scoreV2_8/ +/score_scripts/PowerGrid_score/ diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index 8fbe462..f89f45d 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,7 +1,7 @@ [DEFAULT] ValueByColor = {"green": 2, "orange": 1, "red": 0} -Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"Speed": 0.1} +Coefficients = {"ML": 0.4, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.25, "Accuracy": 0.75} Thresholds = { "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, @@ -27,7 +27,9 @@ Thresholds = { "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, - "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}} + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}} [USE CASE 1] SpeedConfig = {"speed_score_formula" :"PowerGrid Competition", diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py index 6bb86c9..02d0a93 100644 --- a/lips/scoring/__init__.py +++ b/lips/scoring/__init__.py @@ -2,8 +2,9 @@ from lips.scoring.scoring import Scoring from lips.scoring.powergrid_scoring import PowerGridScoring +from lips.scoring.airfoil_powergrid_scoring import AirfoilPowerGridScoring __all__ = [ - "Scoring", "PowerGridScoring" + "Scoring", "PowerGridScoring", "AirfoilPowerGridScoring" ] \ No newline at end of file diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py new file mode 100644 index 0000000..ec84961 --- /dev/null +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -0,0 +1,173 @@ +import math +from typing import Union, Dict, List + +from .scoring import Scoring +from .utils import get_nested_value, filter_metrics, read_json +from ..config import ConfigManager + + +class AirfoilPowerGridScoring(Scoring): + """ + Class responsible for calculating the score of the AirFoil Power Grid competition : https://www.codabench.org/competitions/3282/ + """ + + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the AirfoilPowerGridScoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ + super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) + + def _reconstruct_ml_metrics(self, raw_metrics: Dict, ml_key_path: List[str]) -> Dict: + """ + Construct ML metrics by retrieving and filtering data from the raw-JSON metrics. + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_key_path: List of keys representing the path to the ML metrics + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered ML metrics. + + Raises: + ValueError: If the specified path is invalid or ML metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + all_ml_metrics = get_nested_value(raw_metrics, ml_key_path) + if all_ml_metrics is None: + raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") + + if not isinstance(all_ml_metrics, dict): + raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") + + ml_metrics = {"ML": filter_metrics(all_ml_metrics, self.thresholds.keys())} + + pressure_surfacic_value_path = ml_key_path[:-1] + ["MSE_normalized_surfacic", "pressure"] + ml_metrics["ML"]["pressure_surfacic"] = get_nested_value(raw_metrics, pressure_surfacic_value_path) + + return ml_metrics + + def _reconstruct_physic_metrics(self, raw_metrics: Dict, physic_key_path: List[str]) -> Dict: + """ + Construct Physic metrics by retrieving and filtering data from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + physic_key_path: List of keys representing the path to the physics metrics + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered physics metrics. + + Raises: + ValueError: If the specified path is invalid or physics metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + all_physic_metrics = get_nested_value(raw_metrics, physic_key_path) + if all_physic_metrics is None: + raise ValueError(f"Invalid path {physic_key_path}. Could not retrieve Physic metrics.") + + if not isinstance(all_physic_metrics, dict): + raise TypeError(f"Expected a dictionary at {physic_key_path}, but got {type(all_physic_metrics).__name__}.") + + physic_metrics = {"Physics": filter_metrics(all_physic_metrics, self.thresholds.keys())} + return physic_metrics + + def _reconstruct_ood_metrics(self, raw_metrics: Dict, ml_ood_key_path: List[str], + physic_ood_key_path: List[str]) -> Dict: + """ + Construct OOD metrics by retrieving and Combining ML and Physic OOD-metrics from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_ood_key_path: Path to the ML OOD metrics. + physic_ood_key_path: Path to the Physics OOD metrics. + + Returns: + Dictionary containing the combined OOD metrics. + """ + ml_ood_metrics = self._reconstruct_ml_metrics(raw_metrics, ml_ood_key_path)["ML"] + physic_ood_metrics = self._reconstruct_physic_metrics(raw_metrics, physic_ood_key_path)["Physics"] + + return {"OOD": {**ml_ood_metrics, **physic_ood_metrics}} + + def compute_speed_score(self, time_inference: float) -> float: + """ + Computes the speed score based on: + + Score_Speed = min( (log10(SpeedUp) / log10(SpeedUpMax)), 1) + + Where : SpeedUp = time_ClassicalSolver / time_Inference + + Args: + time_inference: Inference time in seconds. + + Returns: + The speed score (between 0 and 1). + """ + + speed_up = self._calculate_speed_up(time_inference) + max_speed_ratio_allowed = self.thresholds["max_speed_ratio_allowed"]["thresholds"][0] + res = min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1) + return max(res, 0) + + def _calculate_speed_up(self, time_inference: float) -> float: + """Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + """ + + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] + return time_classical_solver / time_inference + + def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + """ + Computes the competition score based on the provided metrics in metrics_dict or metrics_path + + Args: + metrics_dict: Dictionary containing the raw metrics data. + metrics_path: Path to the JSON file containing the raw metrics data. + + Returns: + Dictionary containing the score colors, the score values, and the global score. + Raises: + ValueError: If both metrics_dict and metrics_path are None. + """ + + if metrics_dict is not None: + metrics = metrics_dict.copy() + elif metrics_path != "": + metrics = read_json(json_path=metrics_path, json_object=metrics_dict) + else: + raise ValueError("metrics_path and metrics_dict cant' both be None") + + time_inference = metrics_dict["test_mean_simulation_time"] + + ml_metrics = self._reconstruct_ml_metrics(metrics, + ml_key_path=["fc_metrics_test", "test", "ML", "MSE_normalized"]) + physic_metrics = self._reconstruct_physic_metrics(metrics, + physic_key_path=["fc_metrics_test", "test", "Physics"]) + ood_metrics = self._reconstruct_ood_metrics(metrics, ml_ood_key_path=['fc_metrics_test_ood', 'test_ood', 'ML', + 'MSE_normalized'], + physic_ood_key_path=['fc_metrics_test_ood', 'test_ood', 'Physics']) + + metrics = {**ml_metrics, **physic_metrics, **ood_metrics} + + sub_scores_color = self.colorize_metrics(metrics) + sub_scores_values = self.calculate_sub_scores(sub_scores_color) + + speed_score = self.compute_speed_score(time_inference) + sub_scores_values["ML"] = {"Accuracy": sub_scores_values["ML"], "SpeedUP": speed_score} + sub_scores_values["OOD"] = {"Accuracy": sub_scores_values["OOD"], "SpeedUP": speed_score} + + global_score = self.calculate_global_score(sub_scores_values) + + return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py index d2d1bb5..e983181 100644 --- a/lips/scoring/scoring.py +++ b/lips/scoring/scoring.py @@ -99,7 +99,8 @@ def _validate_configuration(self) -> None: dict) or "thresholds" not in threshold_data or "comparison_type" not in threshold_data: raise ValueError( f"Invalid thresholds data for metric '{metric_name}'. Must be a dict with 'thresholds' and 'comparison_type' keys.") - if len(threshold_data["thresholds"]) != expected_threshold_count: + if (threshold_data["comparison_type"] in VALID_COMPARISONS) and ( + len(threshold_data["thresholds"]) != expected_threshold_count): raise ValueError( f"Metric '{metric_name}': Thresholds count must be {expected_threshold_count} (length of ValueByColor - 1).") diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index a28abe9..d660294 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -89,6 +89,6 @@ def get_nested_value(data, keys): data = data[key] return data -def filter_metrics(data, metric_list): +def filter_metrics(data, metrics): """Filter the data dictionary to include only the specified metrics.""" - return {key: value for key, value in data.items() if key in metric_list} \ No newline at end of file + return {key: value for key, value in data.items() if key in metrics} \ No newline at end of file diff --git a/lips/tests/scoring/test_scoring.py b/lips/tests/scoring/test_scoring.py index 09713b1..64a6ab5 100644 --- a/lips/tests/scoring/test_scoring.py +++ b/lips/tests/scoring/test_scoring.py @@ -10,7 +10,10 @@ def setUp(self): self.mock_config.get_option.side_effect = lambda key: { "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, - "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, }, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]} + }, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, "coefficients": {"ML": 0.3, "OOD": 0.3, "Physics": 0.3, "Speed": 0.1}}[key] self.scoring = Scoring(config=self.mock_config) From 18af92a88d03fbf4358ea4b84975815f0e5da781 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Tue, 25 Feb 2025 12:53:16 +0100 Subject: [PATCH 11/18] airfoil_powergrid_scoring.py unnitest --- .../powergrid/scoring/ScoreConfig.ini | 2 +- lips/scoring/airfoil_powergrid_scoring.py | 6 +- lips/scoring/utils.py | 2 + .../scoring/test_airfoil_powergrid_scoring.py | 105 ++++++++++++++++++ 4 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 lips/tests/scoring/test_airfoil_powergrid_scoring.py diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index f89f45d..457b5be 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -1,7 +1,7 @@ [DEFAULT] ValueByColor = {"green": 2, "orange": 1, "red": 0} -Coefficients = {"ML": 0.4, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.25, "Accuracy": 0.75} +Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.25, "Accuracy": 0.75} Thresholds = { "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py index ec84961..b541381 100644 --- a/lips/scoring/airfoil_powergrid_scoring.py +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -117,7 +117,7 @@ def compute_speed_score(self, time_inference: float) -> float: speed_up = self._calculate_speed_up(time_inference) max_speed_ratio_allowed = self.thresholds["max_speed_ratio_allowed"]["thresholds"][0] - res = min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1) + res = min((math.log10(speed_up) / math.log10(max_speed_ratio_allowed)), 1) return max(res, 0) def _calculate_speed_up(self, time_inference: float) -> float: @@ -141,7 +141,7 @@ def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: s Raises: ValueError: If both metrics_dict and metrics_path are None. """ - + if metrics_dict is not None: metrics = metrics_dict.copy() elif metrics_path != "": @@ -149,7 +149,7 @@ def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: s else: raise ValueError("metrics_path and metrics_dict cant' both be None") - time_inference = metrics_dict["test_mean_simulation_time"] + time_inference = metrics["test_mean_simulation_time"] ml_metrics = self._reconstruct_ml_metrics(metrics, ml_key_path=["fc_metrics_test", "test", "ML", "MSE_normalized"]) diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index d660294..8cf2f89 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -1,6 +1,7 @@ import json import math from typing import Union, Dict +import logging def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): @@ -85,6 +86,7 @@ def get_nested_value(data, keys): """Retrieve a nested value from a dictionary using a list of keys.""" for key in keys: if key not in data: + logging.warning(f"Path '{keys}' not found in data. Returning None.") return None data = data[key] return data diff --git a/lips/tests/scoring/test_airfoil_powergrid_scoring.py b/lips/tests/scoring/test_airfoil_powergrid_scoring.py new file mode 100644 index 0000000..629447d --- /dev/null +++ b/lips/tests/scoring/test_airfoil_powergrid_scoring.py @@ -0,0 +1,105 @@ +import json +import unittest +from unittest.mock import MagicMock, patch, mock_open + +from lips.scoring import AirfoilPowerGridScoring + + +class TestAirfoilPowerGridScoring(unittest.TestCase): + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "pressure": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}}, + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + "coefficients": {"ML": 0.3, "OOD": 0.3, "Physics": 0.3, "Speed": 0.1}}[key] + self.scoring = AirfoilPowerGridScoring(config=self.mock_config) + + def test__reconstruct_ml_metrics(self): + raw_metrics = {"fc_metrics_test": {"test": { + "ML": {"MSE_normalized": {"a_or": 0.4, "spearman_correlation_drag": 0.6}, + "MSE_normalized_surfacic": {"pressure": 0.2}}}}} + ml_key_path = ["fc_metrics_test", "test", "ML", "MSE_normalized"] + ml_metrics = self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + self.assertEqual(ml_metrics["ML"]["a_or"], 0.4) + self.assertEqual(ml_metrics["ML"]["pressure_surfacic"], 0.2) + + def test__reconstruct_ml_metrics_invalid_path(self): + raw_metrics = {} + ml_key_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + + def test__reconstruct_ml_metrics_invalid_type(self): + raw_metrics = {"fc_metrics_test": {"test": {"ML": {"MSE_normalized": "not a dict"}}}} + ml_key_path = ["fc_metrics_test", "test", "ML", "MSE_normalized"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_key_path) + + def test__reconstruct_physic_metrics(self): + raw_metrics = {"fc_metrics_test": { + "test": {"Physics": {"spearman_correlation_drag": 0.7, "spearman_correlation_lift": 0.8}}}} + physic_key_path = ["fc_metrics_test", "test", "Physics"] + physic_metrics = self.scoring._reconstruct_physic_metrics(raw_metrics, physic_key_path) + self.assertEqual(physic_metrics["Physics"]["spearman_correlation_drag"], 0.7) + + def test__reconstruct_ood_metrics(self): + raw_metrics = {"fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}}, "Physics": {"spearman_correlation_drag": 0.9}}}, + "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"some_metric": 0.4}}, "Physics": {"some_metric": 0.7}}}} + ml_ood_key_path = ["fc_metrics_test_ood", "test_ood", "ML", "MSE_normalized"] + physic_ood_key_path = ["fc_metrics_test_ood", "test_ood", "Physics"] + ood_metrics = self.scoring._reconstruct_ood_metrics(raw_metrics, ml_ood_key_path, physic_ood_key_path) + self.assertEqual(ood_metrics["OOD"]["spearman_correlation_drag"], 0.9) + + def test_compute_speed_score(self): + time_inference = 5.0 + speed_score = self.scoring.compute_speed_score(time_inference) + self.assertAlmostEqual(speed_score, 0.61928031) + + def test_compute_speed_score_max_speed(self): + time_inference = 1600 + speed_score = self.scoring.compute_speed_score(time_inference) + self.assertAlmostEqual(speed_score, 0) + + @patch("builtins.open", new_callable=mock_open) + def test_compute_scores_from_path(self, mock_file): + mock_json_data = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"a_or": 0.4}, "MSE_normalized_surfacic": {"pressure": 0.008}}, + "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, + "Physics": {"spearman_correlation_lift": 0.9}}}} + mock_file.return_value.read.return_value = json.dumps(mock_json_data) + + scores = self.scoring.compute_scores(metrics_path="dummy.json") + self.assertAlmostEqual(scores["Global Score"], 0.521568188) + + def test_compute_scores_from_dict(self): + metrics_dict = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"a_or": 0.4}, "MSE_normalized_surfacic": {"pressure": 0.008}}, + "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { + "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, + "Physics": {"spearman_correlation_lift": 0.9}}}} + scores = self.scoring.compute_scores(metrics_dict=metrics_dict) + self.assertAlmostEqual(scores["Global Score"], 0.521568188) + + def test_compute_scores_invalid_input(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + def test_compute_scores_missing_time(self): + metrics_dict = {"fc_metrics_test": { + "test": {"ML": {"MSE_normalized": {"some_metric": 0.4}}, "Physics": {"some_metric": 0.7}}}, + "fc_metrics_test_ood": {"test_ood": {"ML": {"MSE_normalized": {"some_metric": 0.3}}, + "Physics": {"some_metric": 0.9}}}} # Dummy metrics + with self.assertRaises(KeyError): + self.scoring.compute_scores(metrics_dict=metrics_dict) + +if __name__ == '__main__': + unittest.main() From 170b93b0bd983ba3414fcc856d242cb6599a2470 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Tue, 25 Feb 2025 14:06:33 +0100 Subject: [PATCH 12/18] add ml4physim_powergrid_socring.py --- lips/scoring/ml4physim_powergrid_socring.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lips/scoring/ml4physim_powergrid_socring.py diff --git a/lips/scoring/ml4physim_powergrid_socring.py b/lips/scoring/ml4physim_powergrid_socring.py new file mode 100644 index 0000000..e69de29 From bfc184f3f20228157c645b31d9ca5df6f55d850b Mon Sep 17 00:00:00 2001 From: Seif Attoui Date: Tue, 25 Feb 2025 14:07:43 +0100 Subject: [PATCH 13/18] Update setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index f2a92d1..dd15fc1 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,8 @@ # SPDX-License-Identifier: MPL-2.0 # This file is part of leap_net, leap_net a keras implementation of the LEAP Net model. + +import time import os import setuptools from setuptools import setup From 8c780739488b95075738d23108be2ca0c1f387fb Mon Sep 17 00:00:00 2001 From: Seif Attoui Date: Tue, 25 Feb 2025 14:15:16 +0100 Subject: [PATCH 14/18] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index dd15fc1..aaa7ea0 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ # This file is part of leap_net, leap_net a keras implementation of the LEAP Net model. -import time import os import setuptools from setuptools import setup From ea2d5e91b08afbdc213b1b713181a5b0963b6d34 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Thu, 27 Feb 2025 13:43:09 +0100 Subject: [PATCH 15/18] add ml4physim_powergrid_socring.py --- .../powergrid/scoring/ScoreConfig.ini | 95 +++++--- lips/scoring/__init__.py | 3 +- lips/scoring/airfoil_powergrid_scoring.py | 12 +- lips/scoring/ml4physim_powergrid_socring.py | 175 ++++++++++++++ lips/scoring/powergrid_scoring.py | 226 ++---------------- lips/scoring/scoring.py | 33 ++- lips/scoring/utils.py | 44 +--- .../scoring/test_airfoil_powergrid_scoring.py | 17 +- .../test_ml4physim_powergrid_socring.py | 162 +++++++++++++ lips/tests/scoring/test_scoring.py | 6 +- 10 files changed, 476 insertions(+), 297 deletions(-) create mode 100644 lips/tests/scoring/test_ml4physim_powergrid_socring.py diff --git a/configurations/powergrid/scoring/ScoreConfig.ini b/configurations/powergrid/scoring/ScoreConfig.ini index 457b5be..6c4fcfb 100644 --- a/configurations/powergrid/scoring/ScoreConfig.ini +++ b/configurations/powergrid/scoring/ScoreConfig.ini @@ -3,6 +3,40 @@ ValueByColor = {"green": 2, "orange": 1, "red": 0} Coefficients = {"ML": 0.3, "OOD": 0.3, "Physics":0.3 ,"SpeedUP": 0.25, "Accuracy": 0.75} +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}} +[AirfoilCompetition] +ValueByColor = {"green": 2, "orange": 1, "red": 0} + +Coefficients = {"ML": {"value": 0.4, "Accuracy": {"value": 0.75}, "SpeedUP": {"value": 0.25}}, + "OOD": {"value": 0.3, "Accuracy": {"value": 0.75}, "SpeedUP": {"value": 0.25}}, + "Physics": {"value": 0.3}} + Thresholds = { "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, @@ -31,33 +65,36 @@ Thresholds = { "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}} -[USE CASE 1] -SpeedConfig = {"speed_score_formula" :"PowerGrid Competition", - "max_speed_ratio_allowed" : 10000, - "reference_mean_simulation_time": 1500 - } +[ML4PhysimCompetition] +ValueByColor = {"green": 2, "orange": 1, "red": 0} -Thresholds = {"a_or":(0.02,0.05,"min"), - "a_ex":(0.02,0.05,"min"), - "p_or":(0.02,0.05,"min"), - "p_ex":(0.02,0.05,"min"), - "v_or":(0.2,0.5,"min"), - "v_ex":(0.2,0.5,"min"), - "CURRENT_POS":(1., 5.,"min"), - "VOLTAGE_POS":(1.,5.,"min"), - "LOSS_POS":(1.,5.,"min"), - "DISC_LINES":(1.,5.,"min"), - "CHECK_LOSS":(1.,5.,"min"), - "CHECK_GC":(0.05,0.10,"min"), - "CHECK_LC":(0.05,0.10,"min"), - "CHECK_JOULE_LAW":(1.,5.,"min"), - "x-velocity":(0.01,0.02,"min"), - "y-velocity":(0.01,0.02,"min"), - "pressure":(0.002,0.01,"min"), - "pressure_surfacic":(0.008,0.02,"min"), - "turbulent_viscosity":(0.05,0.1,"min"), - "mean_relative_drag":(0.4,5.0,"min"), - "mean_relative_lift":(0.1,0.3,"min"), - "spearman_correlation_drag":(0.8,0.9,"max"), - "spearman_correlation_lift":(0.96,0.99,"max") - } \ No newline at end of file +Coefficients = {"ID": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "OOD": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "SpeedUP": {"value": 0.4}} +Thresholds = { + "a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.2, 0.5]}, + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.05, 0.10]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [1.0, 5.0]}, + "x-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "y-velocity": {"comparison_type": "minimize", "thresholds": [0.01, 0.02]}, + "pressure": {"comparison_type": "minimize", "thresholds": [0.002, 0.01]}, + "pressure_surfacic": {"comparison_type": "minimize", "thresholds": [0.008, 0.02]}, + "turbulent_viscosity": {"comparison_type": "minimize", "thresholds": [0.05, 0.1]}, + "mean_relative_drag": {"comparison_type": "minimize", "thresholds": [0.4, 5.0]}, + "mean_relative_lift": {"comparison_type": "minimize", "thresholds": [0.1, 0.3]}, + "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, + "spearman_correlation_lift": {"comparison_type": "maximize", "thresholds": [0.96, 0.99]}, + "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [32.79]}, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [50]}} \ No newline at end of file diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py index 02d0a93..28395cd 100644 --- a/lips/scoring/__init__.py +++ b/lips/scoring/__init__.py @@ -3,8 +3,9 @@ from lips.scoring.scoring import Scoring from lips.scoring.powergrid_scoring import PowerGridScoring from lips.scoring.airfoil_powergrid_scoring import AirfoilPowerGridScoring +from lips.scoring.ml4physim_powergrid_socring import ML4PhysimPowerGridScoring __all__ = [ - "Scoring", "PowerGridScoring", "AirfoilPowerGridScoring" + "Scoring", "PowerGridScoring", "AirfoilPowerGridScoring", "ML4PhysimPowerGridScoring" ] \ No newline at end of file diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py index b541381..6900668 100644 --- a/lips/scoring/airfoil_powergrid_scoring.py +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -1,12 +1,12 @@ import math from typing import Union, Dict, List -from .scoring import Scoring +from .powergrid_scoring import PowerGridScoring from .utils import get_nested_value, filter_metrics, read_json from ..config import ConfigManager -class AirfoilPowerGridScoring(Scoring): +class AirfoilPowerGridScoring(PowerGridScoring): """ Class responsible for calculating the score of the AirFoil Power Grid competition : https://www.codabench.org/competitions/3282/ """ @@ -120,14 +120,6 @@ def compute_speed_score(self, time_inference: float) -> float: res = min((math.log10(speed_up) / math.log10(max_speed_ratio_allowed)), 1) return max(res, 0) - def _calculate_speed_up(self, time_inference: float) -> float: - """Calculates the speedup factor based on: - SpeedUp = time_ClassicalSolver / time_Inference - """ - - time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] - return time_classical_solver / time_inference - def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: """ Computes the competition score based on the provided metrics in metrics_dict or metrics_path diff --git a/lips/scoring/ml4physim_powergrid_socring.py b/lips/scoring/ml4physim_powergrid_socring.py index e69de29..63de569 100644 --- a/lips/scoring/ml4physim_powergrid_socring.py +++ b/lips/scoring/ml4physim_powergrid_socring.py @@ -0,0 +1,175 @@ +import math +from typing import Union, Dict, List + +from .powergrid_scoring import PowerGridScoring +from .utils import get_nested_value, read_json +from ..config import ConfigManager + + +class ML4PhysimPowerGridScoring(PowerGridScoring): + + def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the ML4PhysimPowerGridScoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ + super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) + + def _reconstruct_ml_metrics(self, raw_metrics: Dict, ml_section_path: List[str]) -> Dict: + """ + Construct ML metrics by retrieving data from the raw-JSON metrics. + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_section_path: List of keys representing the path to the ML section + within the raw_metrics dictionary. + + Returns: + Dictionary containing the desired ML metrics. + + Raises: + ValueError: If the specified path is invalid or ML metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + ml_section = get_nested_value(raw_metrics, ml_section_path) + + if ml_section is None: + raise ValueError(f"Invalid path {ml_section_path}. Could not retrieve ML metrics.") + + if not isinstance(ml_section, dict): + raise TypeError(f"Expected a dictionary at {ml_section_path}, but got {type(ml_section).__name__}.") + + ml_metrics = {} + + ml_metrics["a_or"] = ml_section["MAPE_90_avg"]["a_or"] + ml_metrics["a_ex"] = ml_section["MAPE_90_avg"]["a_ex"] + ml_metrics["p_or"] = ml_section["MAPE_10_avg"]["p_or"] + ml_metrics["p_ex"] = ml_section["MAPE_10_avg"]["p_ex"] + ml_metrics["v_or"] = ml_section["MAE_avg"]["v_or"] + ml_metrics["v_ex"] = ml_section["MAE_avg"]["v_ex"] + + return {"ML": ml_metrics} + + def _reconstruct_physic_metrics(self, raw_metrics: Dict, physic_section_path: List[str]) -> Dict: + """ + Construct Physic metrics by retrieving and filtering data from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + physic_section_path: List of keys representing the path to the physics section + within the raw_metrics dictionary. + + Returns: + Dictionary containing the filtered physics metrics. + + Raises: + ValueError: If the specified path is invalid or physics metrics are not found. + TypeError: If the value at the specified path is not a dictionary. + """ + + physic_section = get_nested_value(raw_metrics, physic_section_path) + + if physic_section is None: + raise ValueError(f"Invalid path {physic_section_path}. Could not retrieve Physic metrics.") + + if not isinstance(physic_section, dict): + raise TypeError(f"Expected a dictionary at {physic_section_path}, but got {type(physic_section).__name__}.") + + physic_metrics = {} + + physic_metrics["CURRENT_POS"] = physic_section["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100. + physic_metrics["VOLTAGE_POS"] = physic_section["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100. + physic_metrics["LOSS_POS"] = physic_section["LOSS_POS"]["violation_proportion"] * 100. + physic_metrics["DISC_LINES"] = physic_section["DISC_LINES"]["violation_proportion"] * 100. + physic_metrics["CHECK_LOSS"] = physic_section["CHECK_LOSS"]["violation_percentage"] + physic_metrics["CHECK_GC"] = physic_section["CHECK_GC"]["violation_percentage"] + physic_metrics["CHECK_LC"] = physic_section["CHECK_LC"]["violation_percentage"] + physic_metrics["CHECK_JOULE_LAW"] = physic_section["CHECK_JOULE_LAW"]["violation_proportion"] * 100. + + return {"Physics": physic_metrics} + + def _reconstruct_ood_metrics(self, raw_metrics: Dict, ml_ood_section_path: List[str], + physic_ood_section_path: List[str]) -> Dict: + """ + Construct OOD metrics by retrieving and Combining ML and Physic OOD-metrics from the raw-JSON metrics . + + Args: + raw_metrics: Dictionary containing the raw metrics data. + ml_ood_section_path: Path to the ML OOD section. + physic_ood_section_path: Path to the Physics OOD section. + + Returns: + Dictionary containing the combined OOD metrics. + """ + ml_ood_metrics = self._reconstruct_ml_metrics(raw_metrics, ml_ood_section_path) + physic_ood_metrics = self._reconstruct_physic_metrics(raw_metrics, physic_ood_section_path) + + return {"OOD": {**ml_ood_metrics, **physic_ood_metrics}} + + def compute_speed_score(self, time_inference: float) -> float: + """ + Computes the speed score based on: + + Score_Speed = min( weibull(SpeedUp), 1) + + Where : SpeedUp = time_ClassicalSolver / time_Inference + + Args: + time_inference: Inference time in seconds. + + Returns: + The speed score (between 0 and 1). + """ + speed_up = self._calculate_speed_up(time_inference) + res = min(self._weibull(5, 1.7, speed_up), 1) + return max(res, 0) + + def _weibull(self, c, b, x): + a = c * ((-math.log(0.9)) ** (-1 / b)) + return 1. - math.exp(-(x / a) ** b) + + def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + """ + Computes the competition score based on the provided metrics in metrics_dict or metrics_path + + Args: + metrics_dict: Dictionary containing the raw metrics data. + metrics_path: Path to the JSON file containing the raw metrics data. + + Returns: + Dictionary containing the score colors, the score values, and the global score. + Raises: + ValueError: If both metrics_dict and metrics_path are None. + """ + + if metrics_dict is not None: + metrics = metrics_dict.copy() + elif metrics_path != "": + metrics = read_json(json_path=metrics_path, json_object=metrics_dict) + else: + raise ValueError("metrics_path and metrics_dict cant' both be None") + + time_inference = metrics["test"]["ML"]["TIME_INF"] + + ml_metrics = self._reconstruct_ml_metrics(metrics, ["test", "ML"]) + physic_metrics = self._reconstruct_physic_metrics(metrics, ["test", "Physics"]) + ood_metrics = self._reconstruct_ood_metrics(metrics, ["test_ood_topo", "ML"], ["test_ood_topo", "Physics"]) + + metrics = {"ID": {**ml_metrics, **physic_metrics}, **ood_metrics} + sub_scores_color = self.colorize_metrics(metrics) + + sub_scores_values = self.calculate_sub_scores(sub_scores_color) + + speed_score = self.compute_speed_score(time_inference) + sub_scores_values["SpeedUP"] = speed_score + + global_score = self.calculate_global_score(sub_scores_values) + + return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 687dfa4..8d0eafb 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,219 +1,39 @@ -import math -from typing import Union, Dict, List +from abc import abstractmethod +from typing import Union, Dict from lips.config import ConfigManager -from lips.logger import CustomLogger from lips.scoring import Scoring -from lips.scoring import utils -from lips.scoring.utils import get_nested_value, filter_metrics class PowerGridScoring(Scoring): def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, - scenario: Union[str, None] = None, log_path: Union[str, None] = None): - super().__init__(config=config, config_path=config_path, config_section=scenario, log_path=log_path) - self.logger = CustomLogger(__class__.__name__, self.log_path).logger + config_section: Union[str, None] = None, log_path: Union[str, None] = None): + super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) - self.thresholds = self.config.get_option("thresholds") - self.coefficients = self.config.get_option("coefficients") - self.value_by_color = self.config.get_option("valuebycolor") + @abstractmethod + def _reconstruct_ml_metrics(self, **kwargs) -> Dict: + pass - self.speed_config = self.config.get_option("speedconfig") + @abstractmethod + def _reconstruct_physic_metrics(self, **kwargs) -> Dict: + pass - def scoring(self, metrics_path: str = "", metrics_dict: Union[Dict, str, None] = None): + @abstractmethod + def _reconstruct_ood_metrics(self, **kwargs) -> Dict: + pass - if metrics_dict is not None: - metrics = metrics_dict.copy() - elif metrics_path != "": - metrics = utils.read_json(json_path=metrics_path, json_object=metrics_dict) - else: - raise ValueError("metrics_path and metrics_dict cant' both be None") + @abstractmethod + def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + pass - # calculate speed score - time_inference = metrics.pop("Speed")["inference_time"] - speed_score = self._calculate_speed_score(time_inference) - # score discretize - score_color = PowerGridScoring._calculate_score_color(metrics, self.thresholds) + @abstractmethod + def compute_speed_score(self, time_inference: float) -> float: + pass - score_values = dict() - - for key in self.coefficients: - if key in score_color: - flat_dict = utils.flatten_dict(score_color[key]) - score_values[key] = self._calculate_leaf_score(flat_dict.values()) - score_values["Speed"] = speed_score - - # calculate global score value - global_score = self._calculate_global_score(score_values) - score_values["Global Score"] = global_score - - return {"Score Colors": score_color, "Score Values": score_values} - - @staticmethod - def _calculate_score_color(metrics, thresholds): - tree = {} - for key, value in metrics.items(): - if isinstance(value, dict): - tree[key] = PowerGridScoring._calculate_score_color(value, thresholds) - else: - - discrete_metric = PowerGridScoring._discretize_metric(metric_name=key, metric_value=value, - thresholds=thresholds) - tree[key] = discrete_metric - return tree - - def _calculate_leaf_score(self, colors: List[str]): - s = sum([self.value_by_color[color] for color in colors]) - return s / (len(colors) * max(self.value_by_color.values())) - - @staticmethod - def _discretize_metric(metric_name, metric_value, thresholds): - """ - Discretize a metric value into a qualitative evaluation (g, o, r). - - :param metric_name: Name of the metric to evaluate - :param metric_value: The value of the metric to be evaluated - :param thresholds: Dictionary with thresholds for each metric. Format: - { - "metric_name": (threshold_min, threshold_max, eval_type) - } - eval_type can be "min" or "max". - :return: Evaluation string ("g", "o", or "r") - :raises ValueError: If the metric_name is not in thresholds or eval_type is invalid + def _calculate_speed_up(self, time_inference: float) -> float: + """Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference """ - # Ensure the metric name exists in thresholds - if metric_name not in thresholds: - available_metrics = ", ".join(thresholds.keys()) - raise ValueError( - f"Metric '{metric_name}' not found in thresholds. Available metrics in thresholds: {available_metrics}") - - # Extract thresholds and evaluation type - threshold_min, threshold_max, eval_type = thresholds[metric_name] - - # Validation for eval_type - if eval_type not in {"min", "max"}: - raise ValueError(f"Invalid eval_type '{eval_type}' for metric '{metric_name}'. Must be 'min' or 'max'.") - - # Determine evaluation based on thresholds and eval_type - if eval_type == "min": - # "min" means smaller values are better - evaluation = "g" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "r" - else: - # "max" means larger values are better - evaluation = "r" if metric_value <= threshold_min else "o" if metric_value < threshold_max else "g" - - return evaluation - - # separate competions from powergrid_scoring - def _calculate_speed_score(self, time_inference): - - time_classical_solver = self.speed_config["reference_mean_simulation_time"] - speed_up = PowerGridScoring._calculate_speed_up(time_classical_solver, time_inference) - - if self.speed_config["speed_score_formula"] == "PowerGrid Competition": - return PowerGridScoring._calculate_speed_score_powergrid_competition_formula(speed_up) - elif self.speed_config["speed_score_formula"] == "AirFoil Competition": - - max_speed_ratio_allowed = self.speed_config["max_speed_ratio_allowed"] - - return PowerGridScoring._calculate_speed_score_airfoil_competition_formula(speed_up, - max_speed_ratio_allowed) - else: - raise ValueError(f'{self.speed_config["speed_score_formula"]} formula not found, please implement it first') - - @staticmethod - def _calculate_speed_up(time_classical_solver, time_inference): + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] return time_classical_solver / time_inference - - @staticmethod - def _calculate_speed_score_airfoil_competition_formula(speed_up, max_speed_ratio_allowed): - res = min(math.log10(speed_up) / math.log10(max_speed_ratio_allowed), 1) - return max(res, 0) - - @staticmethod - def _calculate_speed_score_powergrid_competition_formula(speed_up): - res = utils.weibull(5, 1.7, speed_up) - return max(min(res, 1), 0) - - def _calculate_global_score(self, sub_scores): - global_score = 0 - for coef in self.coefficients.keys(): - global_score += self.coefficients[coef] * sub_scores[coef] - return global_score - - @staticmethod - def reconstruct_ml_metrics(input_json, ml_key_path, used_metric_list): - """ - Construct ML metrics by retrieving and filtering data from the given JSON. - - Parameters: - - input_json (dict): The input JSON containing the ML metrics. - - ml_key_path (list): Path to the ML section in the JSON as a list of keys. - - used_metric_list (list): List of metrics to include in the output. - - Returns: - - dict: Filtered ML metrics containing only the specified metrics. - """ - all_ml_metrics = get_nested_value(input_json, ml_key_path) - if all_ml_metrics is None: - raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") - - if not isinstance(all_ml_metrics, dict): - raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") - - return {"ML": filter_metrics(all_ml_metrics, used_metric_list)} - - @staticmethod - def reconstruct_speed_metric(input_json, speed_key_path): - """ - Construct a dictionary containing the speed metric. - - Parameters: - - input_json (dict): The input JSON containing the speed metric. - - speed_key_path (list): Path to the inference time in the JSON as a list of keys. - - Returns: - - dict: A dictionary with the speed metric in the format {"Speed": {"inference_time": value}}. - - Raises: - - ValueError: If the specified path does not exist or the value is None. - """ - inference_time = get_nested_value(input_json, speed_key_path) - - if inference_time is None: - raise ValueError(f"Invalid path {speed_key_path}. Could not retrieve inference time.") - - if not isinstance(inference_time, (int, float)): - raise TypeError(f"Inference time must be a numeric value, but got {type(inference_time).__name__}.") - - return {"Speed": {"inference_time": inference_time}} - - @staticmethod - def reconstruct_physic_metrics(input_json, physic_key_path, competition_name, used_metric_list=None): - - all_physic_metrics = get_nested_value(input_json, physic_key_path) - if all_physic_metrics is None: - raise ValueError(f"Invalid path {physic_key_path}. Could not retrieve physic metrics.") - - if not isinstance(all_physic_metrics, dict): - raise TypeError(f"Expected a dictionary at {physic_key_path}, but got {type(all_physic_metrics).__name__}.") - - if competition_name == "AirFoil Competition": - physic_metrics = filter_metrics(all_physic_metrics, used_metric_list) - - elif competition_name == "PowerGrid Competition": - physic_metrics = {"CURRENT_POS": all_physic_metrics["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100., - "VOLTAGE_POS": all_physic_metrics["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100., - "LOSS_POS": all_physic_metrics["LOSS_POS"]["violation_proportion"] * 100., - "DISC_LINES": all_physic_metrics["DISC_LINES"]["violation_proportion"] * 100., - "CHECK_LOSS": all_physic_metrics["CHECK_LOSS"]["violation_percentage"], - "CHECK_GC": all_physic_metrics["CHECK_GC"]["violation_percentage"], - "CHECK_LC": all_physic_metrics["CHECK_LC"]["violation_percentage"], - "CHECK_JOULE_LAW": all_physic_metrics["CHECK_JOULE_LAW"]["violation_proportion"] * 100.} - - else: - raise ValueError(f'{competition_name} not in options') - - return {"Physics": physic_metrics} - diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py index e983181..bf7656f 100644 --- a/lips/scoring/scoring.py +++ b/lips/scoring/scoring.py @@ -131,21 +131,48 @@ def calculate_sub_scores(self, node: Dict[str, Any]) -> Union[float, Dict[str, A else: # Sub-tree node return {key: self.calculate_sub_scores(value) for key, value in node.items()} - def calculate_global_score(self, tree: Union[float, Dict[str, Any]]) -> float: + def calculate_global_score(self, tree: Union[float, Dict[str, Any]], key_path: List[str] = None) -> float: """ Calculates the global score for the entire metrics tree. Args: tree: a pre-calculated sub-score tree + coefficients: a tree containing coefficient values + key_path: the path to the current node in the tree Returns: The global score. + """ + if isinstance(tree, (int, float)): # Base case: already a sub-score return tree + key_path = key_path or [] global_score = 0 for key, subtree in tree.items(): - weight = self.coefficients.get(key, 1) # Default weight is 1 - global_score += weight * self.calculate_global_score(subtree) + new_path = key_path + [key] + + weight = self._get_coefficient(new_path) or 1 # Default weight is 1 + global_score += weight * self.calculate_global_score(subtree, new_path) + return global_score + + def _get_coefficient(self, key_path: List[str]) -> Union[float, None]: + """ + Retrieves the coefficient value from a nested dictionary based on a given path. + Args: + key_path: A list of keys representing the path to the desired coefficient. + + Returns: + The coefficient value if found, otherwise None. + """ + current = self.coefficients + for key in key_path: + if isinstance(current, dict) and key in current: + current = current[key] + else: + self.logger.warning(f"Coefficient not found for path: {' -> '.join(key_path)}. Using default value 1.") + return None + return current.get("value") + diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index 8cf2f89..822a69d 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -1,7 +1,6 @@ import json -import math -from typing import Union, Dict import logging +from typing import Union, Dict def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): @@ -44,44 +43,6 @@ def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): raise ValueError("Both json_path and json_object are empty. Provide at least one.") -def flatten_dict(input_dict, parent_key=""): - """ - Flatten a nested dictionary structure. - - :param input_dict: Dictionary to flatten - :param parent_key: Key to prepend (used for recursion) - :return: Flattened dictionary - """ - flattened = {} - for key, value in input_dict.items(): - if isinstance(value, dict): - # Recursively flatten if the value is a dictionary - flattened.update(flatten_dict(value, parent_key)) - else: - # Add to flattened dictionary - flattened[key] = value - return flattened - - -def weibull(c, b, x): - a = c * ((-math.log(0.9)) ** (-1 / b)) - return 1. - math.exp(-(x / a) ** b) - -def merge_dicts(dict_list): - """ - Merges a list of dictionaries into a single dictionary. - - Parameters: - - dict_list (list): A list of dictionaries to merge. - - Returns: - - dict: A single dictionary containing all key-value pairs. - """ - merged_dict = {} - for d in dict_list: - merged_dict.update(d) # Update merged_dict with each dictionary in the list - return merged_dict - def get_nested_value(data, keys): """Retrieve a nested value from a dictionary using a list of keys.""" for key in keys: @@ -91,6 +52,7 @@ def get_nested_value(data, keys): data = data[key] return data + def filter_metrics(data, metrics): """Filter the data dictionary to include only the specified metrics.""" - return {key: value for key, value in data.items() if key in metrics} \ No newline at end of file + return {key: value for key, value in data.items() if key in metrics} diff --git a/lips/tests/scoring/test_airfoil_powergrid_scoring.py b/lips/tests/scoring/test_airfoil_powergrid_scoring.py index 629447d..c9baf74 100644 --- a/lips/tests/scoring/test_airfoil_powergrid_scoring.py +++ b/lips/tests/scoring/test_airfoil_powergrid_scoring.py @@ -17,7 +17,9 @@ def setUp(self): "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}}, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, - "coefficients": {"ML": 0.3, "OOD": 0.3, "Physics": 0.3, "Speed": 0.1}}[key] + "coefficients": {"ML": {"value": 0.3, "Accuracy": {"value": 0.75}, "Speed": {"value": 0.25}}, + "OOD": {"value": 0.3, "Accuracy": {"value": 0.75}, "Speed": {"value": 0.25}}, + "Physics": {"value": 0.3}}}[key] self.scoring = AirfoilPowerGridScoring(config=self.mock_config) def test__reconstruct_ml_metrics(self): @@ -72,13 +74,13 @@ def test_compute_speed_score_max_speed(self): def test_compute_scores_from_path(self, mock_file): mock_json_data = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { "test": {"ML": {"MSE_normalized": {"a_or": 0.4}, "MSE_normalized_surfacic": {"pressure": 0.008}}, - "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { + "Physics": {"spearman_correlation_drag": 0.7}}}, "fc_metrics_test_ood": { "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, - "Physics": {"spearman_correlation_lift": 0.9}}}} + "Physics": {"spearman_correlation_lift": 0.9}}}} mock_file.return_value.read.return_value = json.dumps(mock_json_data) scores = self.scoring.compute_scores(metrics_path="dummy.json") - self.assertAlmostEqual(scores["Global Score"], 0.521568188) + self.assertAlmostEqual(scores["Global Score"], 0.484068188) def test_compute_scores_from_dict(self): metrics_dict = {"test_mean_simulation_time": 5.0, "fc_metrics_test": { @@ -87,7 +89,7 @@ def test_compute_scores_from_dict(self): "test_ood": {"ML": {"MSE_normalized": {"a_or": 0.3}, "MSE_normalized_surfacic": {"pressure": 0.06}}, "Physics": {"spearman_correlation_lift": 0.9}}}} scores = self.scoring.compute_scores(metrics_dict=metrics_dict) - self.assertAlmostEqual(scores["Global Score"], 0.521568188) + self.assertAlmostEqual(scores["Global Score"], 0.484068188) def test_compute_scores_invalid_input(self): with self.assertRaises(ValueError): @@ -96,10 +98,11 @@ def test_compute_scores_invalid_input(self): def test_compute_scores_missing_time(self): metrics_dict = {"fc_metrics_test": { "test": {"ML": {"MSE_normalized": {"some_metric": 0.4}}, "Physics": {"some_metric": 0.7}}}, - "fc_metrics_test_ood": {"test_ood": {"ML": {"MSE_normalized": {"some_metric": 0.3}}, - "Physics": {"some_metric": 0.9}}}} # Dummy metrics + "fc_metrics_test_ood": {"test_ood": {"ML": {"MSE_normalized": {"some_metric": 0.3}}, + "Physics": {"some_metric": 0.9}}}} # Dummy metrics with self.assertRaises(KeyError): self.scoring.compute_scores(metrics_dict=metrics_dict) + if __name__ == '__main__': unittest.main() diff --git a/lips/tests/scoring/test_ml4physim_powergrid_socring.py b/lips/tests/scoring/test_ml4physim_powergrid_socring.py new file mode 100644 index 0000000..b94e9cc --- /dev/null +++ b/lips/tests/scoring/test_ml4physim_powergrid_socring.py @@ -0,0 +1,162 @@ +import json +import unittest +from unittest.mock import patch, MagicMock, mock_open + +from lips.scoring import ML4PhysimPowerGridScoring + + +class TestML4PhysimPowerGridScoring(unittest.TestCase): + + def setUp(self): + self.mock_config = MagicMock() + self.mock_config.get_option.side_effect = lambda key: { + "thresholds": {"a_or": {"comparison_type": "minimize", "thresholds": [0.02, 0.05]}, + "a_ex": {"comparison_type": "minimize", "thresholds": [0.03, 0.06]}, + "p_or": {"comparison_type": "minimize", "thresholds": [0.04, 0.07]}, + "p_ex": {"comparison_type": "minimize", "thresholds": [0.05, 0.08]}, + "v_or": {"comparison_type": "minimize", "thresholds": [0.06, 0.09]}, + "v_ex": {"comparison_type": "minimize", "thresholds": [0.07, 0.1]}, + + "CURRENT_POS": {"comparison_type": "minimize", "thresholds": [0.1, 0.2]}, + "VOLTAGE_POS": {"comparison_type": "minimize", "thresholds": [0.2, 0.3]}, + "LOSS_POS": {"comparison_type": "minimize", "thresholds": [0.3, 0.4]}, + "DISC_LINES": {"comparison_type": "minimize", "thresholds": [0.4, 0.5]}, + "CHECK_LOSS": {"comparison_type": "minimize", "thresholds": [0.5, 0.6]}, + "CHECK_GC": {"comparison_type": "minimize", "thresholds": [0.6, 0.7]}, + "CHECK_LC": {"comparison_type": "minimize", "thresholds": [0.7, 0.8]}, + "CHECK_JOULE_LAW": {"comparison_type": "minimize", "thresholds": [0.8, 0.9]}, + + "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}}, + + "valuebycolor": {"green": 2, "orange": 1, "red": 0}, + "coefficients": {"ID": {"value": 0.3, "ML": {"value": 0.4}, "Physics": {"value": 0.6}}, + "OOD": {"value": 0.3, "ML": {"value": 0.66}, "Physics": {"value": 0.34}}, + "SpeedUP": {"value": 0.4}}}[key] + + self.scoring = ML4PhysimPowerGridScoring(config=self.mock_config) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_valid_path(self, mock_get_nested_value): + raw_metrics = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.1, "a_ex": 0.2}, "MAPE_10_avg": {"p_or": 0.3, "p_ex": 0.4}, + "MAE_avg": {"v_or": 0.5, "v_ex": 0.6}}}} + mock_get_nested_value.return_value = raw_metrics["test"]["ML"] + ml_section_path = ["test", "ML"] + expected_result = {"ML": {"a_or": 0.1, "a_ex": 0.2, "p_or": 0.3, "p_ex": 0.4, "v_or": 0.5, "v_ex": 0.6}} + result = self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + self.assertEqual(result, expected_result) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_invalid_path(self, mock_get_nested_value): + raw_metrics = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.1, "a_ex": 0.2}, "MAPE_10_avg": {"p_or": 0.3, "p_ex": 0.4}, + "MAE_avg": {"v_or": 0.5, "v_ex": 0.6}}}} + mock_get_nested_value.return_value = None + ml_section_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_ml_metrics_invalid_type(self, mock_get_nested_value): + raw_metrics = {"test": {"ML": "invalid_type"}} + mock_get_nested_value.return_value = raw_metrics["test"]["ML"] + ml_section_path = ["test", "ML"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_ml_metrics(raw_metrics, ml_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_valid_path(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.1}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.2}}, + "LOSS_POS": {"violation_proportion": 0.3}, + "DISC_LINES": {"violation_proportion": 0.4}, + "CHECK_LOSS": {"violation_percentage": 0.5}, + "CHECK_GC": {"violation_percentage": 0.6}, + "CHECK_LC": {"violation_percentage": 0.7}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.8}}}} + mock_get_nested_value.return_value = raw_metrics["test"]["Physics"] + physic_section_path = ["test", "Physics"] + expected_result = {"Physics": {"CURRENT_POS": 10.0, "VOLTAGE_POS": 20.0, "LOSS_POS": 30.0, "DISC_LINES": 40.0, + "CHECK_LOSS": 0.5, "CHECK_GC": 0.6, "CHECK_LC": 0.7, "CHECK_JOULE_LAW": 80.0}} + result = self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + self.assertEqual(result, expected_result) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_invalid_path(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.5}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.2}}, + "LOSS_POS": {"violation_proportion": 0.3}, + "DISC_LINES": {"violation_proportion": 0.4}, + "CHECK_LOSS": {"violation_percentage": 0.5}, + "CHECK_GC": {"violation_percentage": 0.6}, + "CHECK_LC": {"violation_percentage": 0.7}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.8}}}} + mock_get_nested_value.return_value = None + physic_section_path = ["invalid", "path"] + with self.assertRaises(ValueError): + self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + + @patch('lips.scoring.utils.get_nested_value') + def test_reconstruct_physic_metrics_invalid_type(self, mock_get_nested_value): + raw_metrics = {"test": {"Physics": "invalid_type"}} + mock_get_nested_value.return_value = raw_metrics["test"]["Physics"] + physic_section_path = ["test", "Physics"] + with self.assertRaises(TypeError): + self.scoring._reconstruct_physic_metrics(raw_metrics, physic_section_path) + + def test_calculate_speed_up(self): + time_inference = 0.5 + expected_speed_up = 3000.0 + result = self.scoring._calculate_speed_up(time_inference) + self.assertEqual(result, expected_speed_up) + + def test_compute_speed_score(self): + time_inference = 0.5 + expected_speed_score = 1.0 + result = self.scoring.compute_speed_score(time_inference) + self.assertEqual(result, expected_speed_score) + + def test_weibull(self): + c = 5 + b = 1.7 + x = 3000.0 + expected_weibull_value = 1.0 + result = self.scoring._weibull(c, b, x) + self.assertAlmostEqual(result, expected_weibull_value, places=5) + + @patch("builtins.open", new_callable=mock_open) + def test_compute_scores(self, mock_file): + mock_json_data = {"test": { + "ML": {"MAPE_90_avg": {"a_or": 0.15, "a_ex": 0.25}, "MAPE_10_avg": {"p_or": 0.35, "p_ex": 0.45}, + "MAE_avg": {"v_or": 0.55, "v_ex": 0.65}, "TIME_INF": 12.0}, + "Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.15}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.25}}, + "LOSS_POS": {"violation_proportion": 0.35}, "DISC_LINES": {"violation_proportion": 0.45}, + "CHECK_LOSS": {"violation_percentage": 0.55}, "CHECK_GC": {"violation_percentage": 0.65}, + "CHECK_LC": {"violation_percentage": 0.75}, "CHECK_JOULE_LAW": {"violation_proportion": 0.85}}}, + "test_ood_topo": { + "ML": {"MAPE_90_avg": {"a_or": 0.12, "a_ex": 0.22}, "MAPE_10_avg": {"p_or": 0.32, "p_ex": 0.42}, + "MAE_avg": {"v_or": 0.52, "v_ex": 0.62}}, + "Physics": {"CURRENT_POS": {"a_or": {"Violation_proportion": 0.12}}, + "VOLTAGE_POS": {"v_or": {"Violation_proportion": 0.22}}, + "LOSS_POS": {"violation_proportion": 0.32}, "DISC_LINES": {"violation_proportion": 0.42}, + "CHECK_LOSS": {"violation_percentage": 0.52}, "CHECK_GC": {"violation_percentage": 0.62}, + "CHECK_LC": {"violation_percentage": 0.72}, + "CHECK_JOULE_LAW": {"violation_proportion": 0.82}}}} + + mock_file.return_value.read.return_value = json.dumps(mock_json_data) + + scores = self.scoring.compute_scores(metrics_path="dummy.json") + self.assertAlmostEqual(scores["Global Score"], 0.452874999) + + def test_compute_scores_invalid_input(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + def test_compute_scores_no_metrics(self): + with self.assertRaises(ValueError): + self.scoring.compute_scores() + + +if __name__ == '__main__': + unittest.main() diff --git a/lips/tests/scoring/test_scoring.py b/lips/tests/scoring/test_scoring.py index 64a6ab5..16a1fd8 100644 --- a/lips/tests/scoring/test_scoring.py +++ b/lips/tests/scoring/test_scoring.py @@ -12,10 +12,10 @@ def setUp(self): "spearman_correlation_drag": {"comparison_type": "maximize", "thresholds": [0.8, 0.9]}, "inference_time": {"comparison_type": "minimize", "thresholds": [500, 1000]}, "reference_mean_simulation_time": {"comparison_type": "ratio", "thresholds": [1500]}, - "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]} - }, + "max_speed_ratio_allowed": {"comparison_type": "ratio", "thresholds": [10000]}}, "valuebycolor": {"green": 2, "orange": 1, "red": 0}, - "coefficients": {"ML": 0.3, "OOD": 0.3, "Physics": 0.3, "Speed": 0.1}}[key] + "coefficients": {"ML": {"value": 0.3}, "OOD": {"value": 0.3}, "Physics": {"value": 0.3}, + "Speed": {"value": 0.1}}}[key] self.scoring = Scoring(config=self.mock_config) def test_colorize_metrics(self): From 24fb0cc9a28cb3b661a08098e5c73840d13a3268 Mon Sep 17 00:00:00 2001 From: seifou23i Date: Thu, 27 Feb 2025 14:21:28 +0100 Subject: [PATCH 16/18] add unittests --- draft_notebook.ipynb | 365 ------------------ draft_script.py | 158 -------- lips/scoring/__init__.py | 8 + lips/scoring/airfoil_powergrid_scoring.py | 10 +- lips/scoring/ml4physim_powergrid_socring.py | 11 + lips/scoring/powergrid_scoring.py | 68 +++- lips/scoring/scoring.py | 18 +- lips/scoring/utils.py | 83 ++-- score_scripts/NeurIPS_scoreV2_8/metadata | 2 - .../NeurIPS_scoreV2_8/res/json_metrics.json | 72 ---- .../res/json_metrics_new.json | 29 -- score_scripts/NeurIPS_scoreV2_8/score.py | 241 ------------ score_scripts/NeurIPS_scoreV2_8/scores.html | 1 - score_scripts/NeurIPS_scoreV2_8/scores.txt | 4 - score_scripts/PowerGrid_score/metadata | 2 - .../PowerGrid_score/res/json_metrics.json | 197 ---------- .../PowerGrid_score/res/json_metrics_new.json | 39 -- score_scripts/PowerGrid_score/score.py | 88 ----- score_scripts/PowerGrid_score/scores.html | 0 score_scripts/PowerGrid_score/scores.txt | 0 .../PowerGrid_score/utils/__init__.py | 0 .../PowerGrid_score/utils/compute_score.py | 162 -------- 22 files changed, 162 insertions(+), 1396 deletions(-) delete mode 100644 draft_notebook.ipynb delete mode 100644 draft_script.py delete mode 100644 score_scripts/NeurIPS_scoreV2_8/metadata delete mode 100644 score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json delete mode 100644 score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json delete mode 100644 score_scripts/NeurIPS_scoreV2_8/score.py delete mode 100644 score_scripts/NeurIPS_scoreV2_8/scores.html delete mode 100644 score_scripts/NeurIPS_scoreV2_8/scores.txt delete mode 100644 score_scripts/PowerGrid_score/metadata delete mode 100644 score_scripts/PowerGrid_score/res/json_metrics.json delete mode 100644 score_scripts/PowerGrid_score/res/json_metrics_new.json delete mode 100644 score_scripts/PowerGrid_score/score.py delete mode 100644 score_scripts/PowerGrid_score/scores.html delete mode 100644 score_scripts/PowerGrid_score/scores.txt delete mode 100644 score_scripts/PowerGrid_score/utils/__init__.py delete mode 100644 score_scripts/PowerGrid_score/utils/compute_score.py diff --git a/draft_notebook.ipynb b/draft_notebook.ipynb deleted file mode 100644 index 7822b26..0000000 --- a/draft_notebook.ipynb +++ /dev/null @@ -1,365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "dbff8eac-c04b-4fe2-8462-ed3be79c8f7f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:22:59.172122Z", - "start_time": "2024-12-18T17:22:59.153473Z" - } - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:00.903459Z", - "start_time": "2024-12-18T17:23:00.884193Z" - } - }, - "outputs": [], - "source": [ - "from lips.scoring import PowerGridScoring" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "542ed9d6-5949-409a-bcb5-380abc2a441c", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:02.170728Z", - "start_time": "2024-12-18T17:23:02.155588Z" - } - }, - "outputs": [], - "source": [ - "scoring = PowerGridScoring(config_path=\"configurations/powergrid/scoring/ScoreConfig.ini\", scenario=\"DEFAULT\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "4c318392-fc30-4e95-b9b2-7d8fad33ee06", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:03.691169Z", - "start_time": "2024-12-18T17:23:03.666826Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Score Colors': {'ML': {'x-velocity': 'g',\n", - " 'y-velocity': 'g',\n", - " 'pressure': 'o',\n", - " 'turbulent_viscosity': 'g'},\n", - " 'Physics': {'spearman_correlation_drag': 'o',\n", - " 'spearman_correlation_lift': 'g',\n", - " 'mean_relative_drag': 'g',\n", - " 'mean_relative_lift': 'g'},\n", - " 'OOD': {'ML': {'x-velocity': 'g',\n", - " 'y-velocity': 'g',\n", - " 'pressure': 'r',\n", - " 'turbulent_viscosity': 'g'},\n", - " 'Physics': {'spearman_correlation_drag': 'r',\n", - " 'spearman_correlation_lift': 'g',\n", - " 'mean_relative_drag': 'g',\n", - " 'mean_relative_lift': 'g'}}},\n", - " 'Score Values': {'ML': 0.875,\n", - " 'OOD': 0.75,\n", - " 'Physics': 0.875,\n", - " 'Speed': 0.024655622815809464,\n", - " 'Global Score': 0.7524655622815809}}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scoring.scoring(metrics_path=\"./score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "44738300-76ab-4834-b8d9-bf96b6b04f8f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:05.955460Z", - "start_time": "2024-12-18T17:23:05.932678Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'test': {'ML': {'MSE_avg': {'v_or': 0.00010645988368196413,\n", - " 'v_ex': 0.00011296240700175986,\n", - " 'p_or': 0.18411502242088318,\n", - " 'p_ex': 0.1777658313512802,\n", - " 'a_or': 16.51224136352539,\n", - " 'a_ex': 16.86044692993164},\n", - " 'MAE_avg': {'v_or': 0.001553428708575666,\n", - " 'v_ex': 0.001500570448115468,\n", - " 'p_or': 0.14518952369689941,\n", - " 'p_ex': 0.14409466087818146,\n", - " 'a_or': 1.4204329252243042,\n", - " 'a_ex': 1.4652109146118164},\n", - " 'MAPE_avg': {'v_or': 9.848606168816332e-06,\n", - " 'v_ex': 1.0132289389730431e-05,\n", - " 'p_or': 63017717760.0,\n", - " 'p_ex': 22879956992.0,\n", - " 'a_or': 121335185408.0,\n", - " 'a_ex': 52108500992.0},\n", - " 'MAPE_90_avg': {'v_or': 5.658699643585033e-06,\n", - " 'v_ex': 5.919416245742559e-06,\n", - " 'p_or': 0.0033098454767013486,\n", - " 'p_ex': 0.0032964480543750145,\n", - " 'a_or': 0.007386949784782199,\n", - " 'a_ex': 0.007292245940119987},\n", - " 'MAPE_10_avg': {'v_or': 7.93336312753172e-06,\n", - " 'v_ex': 8.195988824322852e-06,\n", - " 'p_or': 0.006397687786608211,\n", - " 'p_ex': 0.006388048696331843,\n", - " 'a_or': 0.021526892545594315,\n", - " 'a_ex': 0.022162465019418712},\n", - " 'TIME_INF': 4.163906573085114},\n", - " 'Physics': {'CURRENT_POS': {'a_or': {'Error': 10040.3349609375,\n", - " 'Violation_proportion': 0.00044102150537634406},\n", - " 'a_ex': {'Error': 11018.400390625,\n", - " 'Violation_proportion': 0.0005930645161290322}},\n", - " 'VOLTAGE_POS': {'v_or': {'Violation_proportion': 0.0},\n", - " 'v_ex': {'Violation_proportion': 0.0}},\n", - " 'LOSS_POS': {'loss_criterion': 209.59898,\n", - " 'violation_proportion': 0.008733548387096774},\n", - " 'DISC_LINES': {'p_or': 0.0,\n", - " 'p_ex': 0.0,\n", - " 'a_or': 0.0,\n", - " 'a_ex': 0.0,\n", - " 'v_or': 0.0,\n", - " 'v_ex': 0.0,\n", - " 'violation_proportion': 0.0},\n", - " 'CHECK_LOSS': {'violation_percentage': 1.539},\n", - " 'CHECK_GC': {'violation_percentage': 0.0,\n", - " 'mae': 9.128214e-05,\n", - " 'wmape': 1.2492486e-06},\n", - " 'CHECK_LC': {'violation_percentage': 77.35536440677966,\n", - " 'mae': 0.10304196395080921,\n", - " 'mape': 0.002859223697836113},\n", - " 'CHECK_JOULE_LAW': {'violation_proportion': 0.0,\n", - " 'mae': 3.054519562577347e-07,\n", - " 'wmape': 2.268769940016548e-06}},\n", - " 'IndRed': {'TIME_INF': 3.9108043271116912}},\n", - " 'test_ood_topo': {'ML': {'MSE_avg': {'v_or': 0.0008741252822801471,\n", - " 'v_ex': 0.0009632128058001399,\n", - " 'p_or': 0.47769251465797424,\n", - " 'p_ex': 0.4475671350955963,\n", - " 'a_or': 40.1962890625,\n", - " 'a_ex': 41.00724411010742},\n", - " 'MAE_avg': {'v_or': 0.0023060280364006758,\n", - " 'v_ex': 0.0023030198644846678,\n", - " 'p_or': 0.18595275282859802,\n", - " 'p_ex': 0.1843600869178772,\n", - " 'a_or': 1.7880399227142334,\n", - " 'a_ex': 1.8418012857437134},\n", - " 'MAPE_avg': {'v_or': 1.5199337212834507e-05,\n", - " 'v_ex': 1.5929399523884058e-05,\n", - " 'p_or': 189589094400.0,\n", - " 'p_ex': 56932474880.0,\n", - " 'a_or': 386140471296.0,\n", - " 'a_ex': 157286449152.0},\n", - " 'MAPE_90_avg': {'v_or': 7.414690380649798e-06,\n", - " 'v_ex': 8.032887808804589e-06,\n", - " 'p_or': 0.004977200835027814,\n", - " 'p_ex': 0.004950829903700901,\n", - " 'a_or': 0.010261113576488012,\n", - " 'a_ex': 0.01012922324923192},\n", - " 'MAPE_10_avg': {'v_or': 1.0476555723965095e-05,\n", - " 'v_ex': 1.1106274117571183e-05,\n", - " 'p_or': 0.008155790825441843,\n", - " 'p_ex': 0.008147190351295156,\n", - " 'a_or': 0.027774648502433164,\n", - " 'a_ex': 0.028373457148515593}},\n", - " 'Physics': {'CURRENT_POS': {'a_or': {'Error': 21769.7578125,\n", - " 'Violation_proportion': 0.0004038709677419355},\n", - " 'a_ex': {'Error': 23943.244140625,\n", - " 'Violation_proportion': 0.0005124731182795699}},\n", - " 'VOLTAGE_POS': {'v_or': {'Violation_proportion': 0.0},\n", - " 'v_ex': {'Violation_proportion': 0.0}},\n", - " 'LOSS_POS': {'loss_criterion': 1076.4073,\n", - " 'violation_proportion': 0.00829997311827957},\n", - " 'DISC_LINES': {'p_or': 0.0,\n", - " 'p_ex': 0.0,\n", - " 'a_or': 0.0,\n", - " 'a_ex': 0.0,\n", - " 'v_or': 0.0,\n", - " 'v_ex': 0.0,\n", - " 'violation_proportion': 0.0},\n", - " 'CHECK_LOSS': {'violation_percentage': 3.2485},\n", - " 'CHECK_GC': {'violation_percentage': 0.0,\n", - " 'mae': 9.088051e-05,\n", - " 'wmape': 1.2194811e-06},\n", - " 'CHECK_LC': {'violation_percentage': 83.45940254237289,\n", - " 'mae': 0.14096058704044898,\n", - " 'mape': 0.0039999913278262085},\n", - " 'CHECK_JOULE_LAW': {'violation_proportion': 0.0,\n", - " 'mae': 2.9970232693607007e-07,\n", - " 'wmape': 2.1807441470482367e-06}},\n", - " 'IndRed': {}}}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import json\n", - "\n", - "with open(\"./score_scripts/PowerGrid_score/res/json_metrics.json\", \"r\") as f:\n", - " raw_metrics = json.load(f)\n", - "raw_metrics" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "3e05c2fd12981d8d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:16.572933Z", - "start_time": "2024-12-18T17:23:16.554868Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Speed': {'inference_time': 4.163906573085114}}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scoring.reconstruct_speed_metric(raw_metrics, [\"test\", \"ML\", \"TIME_INF\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "b40bcc1a2a0a9ad9", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:23:18.497174Z", - "start_time": "2024-12-18T17:23:18.481366Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'ML': {'v_or': 0.00010645988368196413,\n", - " 'v_ex': 0.00011296240700175986,\n", - " 'p_or': 0.18411502242088318,\n", - " 'p_ex': 0.1777658313512802,\n", - " 'a_or': 16.51224136352539,\n", - " 'a_ex': 16.86044692993164}}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scoring.reconstruct_ml_metrics(raw_metrics, [\"test\", \"ML\", \"MSE_avg\"], scoring.thresholds.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "19c352a2e2f0f9cb", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-18T17:43:48.497718Z", - "start_time": "2024-12-18T17:43:48.478015Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Physics': {'CURRENT_POS': 0.04410215053763441,\n", - " 'VOLTAGE_POS': 0.0,\n", - " 'LOSS_POS': 0.8733548387096773,\n", - " 'DISC_LINES': 0.0,\n", - " 'CHECK_LOSS': 1.539,\n", - " 'CHECK_GC': 0.0,\n", - " 'CHECK_LC': 77.35536440677966,\n", - " 'CHECK_JOULE_LAW': 0.0}}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scoring.reconstruct_physic_metrics(raw_metrics, [\"test\", \"Physics\"], competition_name=\"PowerGrid Competition\",used_metric_list=scoring.thresholds.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7e1066c2-198c-4a6a-8e89-bfc28ac42dcb", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9dabb70f-110b-4151-aea4-62521e48e04d", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.16" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/draft_script.py b/draft_script.py deleted file mode 100644 index 1d7a78c..0000000 --- a/draft_script.py +++ /dev/null @@ -1,158 +0,0 @@ -from typing import List, Dict - -from lips.scoring import PowerGridScoring -from lips.scoring import utils - - - -metrics = utils.read_json(json_path="/mnt/seif/HSA/LIPS/score_scripts/PowerGrid_score/res/json_metrics.json") - -# thresholds={"a_or":(0.02,0.05,"min"), -# "a_ex":(0.02,0.05,"min"), -# "p_or":(0.02,0.05,"min"), -# "p_ex":(0.02,0.05,"min"), -# "v_or":(0.2,0.5,"min"), -# "v_ex":(0.2,0.5,"min"), -# "CURRENT_POS":(1., 5.,"min"), -# "VOLTAGE_POS":(1.,5.,"min"), -# "LOSS_POS":(1.,5.,"min"), -# "DISC_LINES":(1.,5.,"min"), -# "CHECK_LOSS":(1.,5.,"min"), -# "CHECK_GC":(0.05,0.10,"min"), -# "CHECK_LC":(0.05,0.10,"min"), -# "CHECK_JOULE_LAW":(1.,5.,"min") -# }.keys() - - -thresholds={"x-velocity":(0.01,0.02,"min"), - "y-velocity":(0.01,0.02,"min"), - "pressure":(0.002,0.01,"min"), - "pressure_surfacic":(0.008,0.02,"min"), - "turbulent_viscosity":(0.05,0.1,"min"), - "mean_relative_drag":(0.4,5.0,"min"), - "mean_relative_lift":(0.1,0.3,"min"), - "spearman_correlation_drag":(0.8,0.9,"max"), - "spearman_correlation_lift":(0.96,0.99,"max") - }.keys() -import json - -def transform_json_generic( - input_json, - used_metric_list, - ml_metric, - in_distrubution_key_path, - ood_key_path, - inference_time -): - """ - Transforms a JSON structure into a desired format based on selected metrics and paths. - - Parameters: - - input_json (dict): The original JSON data. - - used_metric_list (list): List of metrics to include in the output (e.g., ["a_or", "a_ex"]). - - ml_metric (str): Metric key to use in the ML section. - - in_distribution_key_path (list): Path to the in-distribution (test) section. - - ood_key_path (list): Path to the OOD test section. - - inference_time (float): The inference time value. - - Returns: - - dict: Transformed JSON in the desired format. - """ - def get_nested_value(data, keys): - """Retrieve a nested value from a dictionary using a list of keys.""" - for key in keys: - if key not in data: - return None - data = data[key] - return data - - def filter_metrics(data, metric_list): - """Filter the data dictionary to include only the specified metrics.""" - return {key: value for key, value in data.items() if key in metric_list} - - def transform_section(section, metric_list, ml_metric): - """Transform a section (e.g., in-distribution or OOD) based on the desired metrics.""" - transformed = {} - for subsection, data in section.items(): - if subsection == "ML": - # Process ML section with the chosen ml_metric - transformed["ML"] = filter_metrics(data.get(ml_metric, {}), metric_list) - elif subsection == "Physics": - # Process Physics section with the chosen metrics - transformed["Physics"] = filter_metrics(data, metric_list) - else: - # Process other subsections if needed - transformed[subsection] = filter_metrics(data, metric_list) - return transformed - - # Extract in-distribution section - in_distribution_section = get_nested_value(input_json, in_distrubution_key_path) - if not in_distribution_section: - raise ValueError(f"In-distribution section not found at path: {in_distrubution_key_path}") - - in_distribution_transformed = transform_section(in_distribution_section, used_metric_list, ml_metric) - - # Extract OOD section - ood_section = get_nested_value(input_json, ood_key_path) - if not ood_section: - raise ValueError(f"OOD section not found at path: {ood_key_path}") - - ood_transformed = transform_section(ood_section, used_metric_list, ml_metric) - - # Combine into the desired format - transformed_json = { - **in_distribution_transformed, - "Speed": {"inference_time": inference_time}, - "OOD": ood_transformed, - } - - return transformed_json - -# Example usage: - -# used_metric_list = thresholds -# ml_metric = "MSE_avg" -# in_distrubution_key_path = ["test"] -# ood_key_path = ["test_ood_topo"] -# inference_time = 4.163906573085114 -# -# transformed_json = transform_json_generic( -# input_json=metrics, -# used_metric_list=used_metric_list, -# ml_metric=ml_metric, -# in_distrubution_key_path=in_distrubution_key_path, -# ood_key_path=ood_key_path, -# inference_time=inference_time -# ) - -used_metric_list = thresholds - -from lips.scoring.utils import filter_metrics, get_nested_value - - -def construct_ml_metrics(input_json, ml_key_path, used_metric_list): - """ - Construct ML metrics by retrieving and filtering data from the given JSON. - - Parameters: - - input_json (dict): The input JSON containing the ML metrics. - - ml_key_path (list): Path to the ML section in the JSON as a list of keys. - - used_metric_list (list): List of metrics to include in the output. - - Returns: - - dict: Filtered ML metrics containing only the specified metrics. - """ - all_ml_metrics = get_nested_value(input_json, ml_key_path) - if all_ml_metrics is None: - raise ValueError(f"Invalid path {ml_key_path}. Could not retrieve ML metrics.") - - if not isinstance(all_ml_metrics, dict): - raise TypeError(f"Expected a dictionary at {ml_key_path}, but got {type(all_ml_metrics).__name__}.") - - return {"ML" : filter_metrics(all_ml_metrics, used_metric_list)} - - -def construct_speed_metric(input_json, speed_key_path): - return {"Speed": {"inference_time": get_nested_value(input_json, speed_key_path)}} - -print(construct_speed_metric(metrics, ['test', 'ML', 'TIME_INF'])) \ No newline at end of file diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py index 28395cd..ddc6de8 100644 --- a/lips/scoring/__init__.py +++ b/lips/scoring/__init__.py @@ -1,3 +1,11 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + from __future__ import absolute_import from lips.scoring.scoring import Scoring diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py index 6900668..2fe7f94 100644 --- a/lips/scoring/airfoil_powergrid_scoring.py +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -1,3 +1,11 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + import math from typing import Union, Dict, List @@ -8,7 +16,7 @@ class AirfoilPowerGridScoring(PowerGridScoring): """ - Class responsible for calculating the score of the AirFoil Power Grid competition : https://www.codabench.org/competitions/3282/ + Calculates the score for the AirFoil Power Grid competition: https://www.codabench.org/competitions/3282/ """ def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, diff --git a/lips/scoring/ml4physim_powergrid_socring.py b/lips/scoring/ml4physim_powergrid_socring.py index 63de569..4486c83 100644 --- a/lips/scoring/ml4physim_powergrid_socring.py +++ b/lips/scoring/ml4physim_powergrid_socring.py @@ -1,3 +1,11 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + import math from typing import Union, Dict, List @@ -7,6 +15,9 @@ class ML4PhysimPowerGridScoring(PowerGridScoring): + """ + Calculates the score for the ML4Physim Power Grid competition: https://www.codabench.org/competitions/2378/ + """ def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, config_section: Union[str, None] = None, log_path: Union[str, None] = None): diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py index 8d0eafb..3413225 100644 --- a/lips/scoring/powergrid_scoring.py +++ b/lips/scoring/powergrid_scoring.py @@ -1,3 +1,11 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + from abc import abstractmethod from typing import Union, Dict @@ -6,34 +14,90 @@ class PowerGridScoring(Scoring): + """ + Abstract base class for calculating scores for power grid use cases. + """ def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, config_section: Union[str, None] = None, log_path: Union[str, None] = None): + """ + Initializes the PowerGridScoring instance with configuration and logger. + + Args: + config: A ConfigManager instance. Defaults to None. + config_path: Path to the configuration file. Defaults to None. + config_section: Section of the configuration file. Defaults to None. + log_path: Path to the log file. Defaults to None. + """ super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) @abstractmethod def _reconstruct_ml_metrics(self, **kwargs) -> Dict: + """ + Reconstructs the ML metrics from the raw data. + + Returns: + A dictionary containing the reconstructed ML metrics. + """ pass @abstractmethod def _reconstruct_physic_metrics(self, **kwargs) -> Dict: + """ + Reconstructs the physics metrics from the raw data. + + Returns: + A dictionary containing the reconstructed physics metrics. + """ pass @abstractmethod def _reconstruct_ood_metrics(self, **kwargs) -> Dict: + """ + Reconstructs the OOD metrics from the raw data. + + Returns: + A dictionary containing the reconstructed OOD metrics. + """ pass @abstractmethod def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: + """ + Computes the scores based on the provided metrics. + + Args: + metrics_dict: A dictionary containing the metrics data. + metrics_path: The path to a JSON file containing the metrics data. + + Returns: + A dictionary containing the calculated scores. + """ pass @abstractmethod def compute_speed_score(self, time_inference: float) -> float: + """ + Computes the speed score based on the inference time. + + Args: + time_inference: The inference time. + + Returns: + The calculated speed score. + """ pass def _calculate_speed_up(self, time_inference: float) -> float: - """Calculates the speedup factor based on: - SpeedUp = time_ClassicalSolver / time_Inference """ + Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + Args: + time_inference: The inference time in seconds. + + Returns: + The calculated speedup factor. + """ + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] return time_classical_solver / time_inference diff --git a/lips/scoring/scoring.py b/lips/scoring/scoring.py index bf7656f..a919213 100644 --- a/lips/scoring/scoring.py +++ b/lips/scoring/scoring.py @@ -1,7 +1,11 @@ # Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) # See AUTHORS.txt # This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. # SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + import bisect from abc import ABC @@ -16,7 +20,7 @@ class Scoring(ABC): """ - Abstract base class for calculating scores based on metrics and thresholds. + Base class for calculating scores based on metrics and thresholds. """ def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, @@ -105,7 +109,15 @@ def _validate_configuration(self) -> None: f"Metric '{metric_name}': Thresholds count must be {expected_threshold_count} (length of ValueByColor - 1).") def _calculate_leaf_score(self, colors: List[str]) -> float: - """Calculates the score for a leaf node (set of colorized metrics).""" + """ + Calculates the score for a leaf node (set of colorized metrics). + + Args: + colors: A list of color strings representing the colorized metrics. + + Returns: + The calculated score for the leaf node. + """ return sum(self.value_by_color[color] for color in colors) / (len(colors) * max(self.value_by_color.values())) def calculate_sub_scores(self, node: Dict[str, Any]) -> Union[float, Dict[str, Any]]: @@ -137,7 +149,6 @@ def calculate_global_score(self, tree: Union[float, Dict[str, Any]], key_path: L Args: tree: a pre-calculated sub-score tree - coefficients: a tree containing coefficient values key_path: the path to the current node in the tree Returns: @@ -175,4 +186,3 @@ def _get_coefficient(self, key_path: List[str]) -> Union[float, None]: self.logger.warning(f"Coefficient not found for path: {' -> '.join(key_path)}. Using default value 1.") return None return current.get("value") - diff --git a/lips/scoring/utils.py b/lips/scoring/utils.py index 822a69d..915bad6 100644 --- a/lips/scoring/utils.py +++ b/lips/scoring/utils.py @@ -1,58 +1,83 @@ +# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of LIPS, LIPS is a python platform for power networks benchmarking + import json import logging -from typing import Union, Dict +from typing import Union, Dict, Any, List -def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None): - """ - Reads a JSON file from the specified path or a JSON object if the path is empty. +def read_json(json_path: str = "", json_object: Union[Dict, str, None] = None) -> Any: + """Reads a JSON file from the specified path or a JSON object. - Args: - json_path (str): Path to the JSON file. If empty, the json_object is used. - json_object (Union[Dict, str, None]): A JSON object (as a dict or a JSON string). Used if json_path is empty. + Args: + json_path: Path to the JSON file. If empty, `json_object` is used. + json_object: A JSON object (as a dict or a JSON string). + Used if `json_path` is empty. - Returns: - Any: Parsed JSON data as a Python object (dict, list, etc.). + Returns: + Parsed JSON data as a Python object. - Raises: - ValueError: If both json_path and json_object are empty. - FileNotFoundError: If the json_path does not exist. - json.JSONDecodeError: If the JSON data is invalid. - """ + Raises: + ValueError: If both `json_path` and `json_object` are empty or + if `json_object` is not a valid type. + FileNotFoundError: If the file specified by `json_path` + does not exist. + json.JSONDecodeError: If the JSON data is invalid. + """ if json_path: - # Read from JSON file try: with open(json_path, "r") as file: return json.load(file) except FileNotFoundError as e: raise FileNotFoundError(f"JSON file not found at path: {json_path}") from e except json.JSONDecodeError as e: - raise json.JSONDecodeError(f"Invalid JSON format in file: {json_path}", e.doc, e.pos) + raise json.JSONDecodeError(f"Invalid JSON format in file: {json_path}", e.doc, e.pos) from e elif json_object: - # Read from provided JSON object if isinstance(json_object, str): try: - return json.loads(json_object) # Parse JSON string + return json.loads(json_object) except json.JSONDecodeError as e: - raise json.JSONDecodeError("Invalid JSON string provided.", e.doc, e.pos) + raise json.JSONDecodeError("Invalid JSON string provided.", e.doc, e.pos) from e elif isinstance(json_object, dict): - return json_object # Already a parsed dictionary + return json_object else: - raise ValueError("json_metrics must be a valid JSON string or dictionary.") + raise ValueError("`json_object` must be a valid JSON string or dictionary.") else: - raise ValueError("Both json_path and json_object are empty. Provide at least one.") + raise ValueError("Both `json_path` and `json_object` are empty. Provide at least one.") + + +def get_nested_value(data: Dict, keys: List[str]) -> Any: + """Retrieves a nested value from a dictionary using a list of keys. + Args: + data: The dictionary to retrieve the value from. + keys: A list of keys representing the path to the nested value. -def get_nested_value(data, keys): - """Retrieve a nested value from a dictionary using a list of keys.""" + Returns: + The nested value if found, otherwise None. + """ for key in keys: - if key not in data: + if isinstance(data, dict) and key in data: + data = data[key] + else: logging.warning(f"Path '{keys}' not found in data. Returning None.") return None - data = data[key] return data -def filter_metrics(data, metrics): - """Filter the data dictionary to include only the specified metrics.""" - return {key: value for key, value in data.items() if key in metrics} +def filter_metrics(data: Dict, metrics: List[str]) -> Dict: + """Filters a dictionary to include only specified keys. + + Args: + data: The dictionary to filter. + metrics: A list of keys to keep in the filtered dictionary. + + Returns: + A new dictionary containing only the specified keys and their values. + """ + return {key: value for key, value in data.items() if key in metrics} \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/metadata b/score_scripts/NeurIPS_scoreV2_8/metadata deleted file mode 100644 index 04b3b36..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/metadata +++ /dev/null @@ -1,2 +0,0 @@ -command: python $program/score.py $input $output -description: Compute scores for the competition diff --git a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json deleted file mode 100644 index 33619e5..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "total_time": 9447.697538614273, - "training_time": 6113.95232629776, - "test_evaluation_time": 699.8024816513062, - "test_mean_simulation_time": 3.499012408256531, - "test_ood_evaluation_time": 1759.7924394607544, - "test_ood_mean_simulation_time": 3.5479686279450693, - "fc_metrics_test": { - "test": { - "ML": { - "MSE_normalized": { - "x-velocity": 0.00262499232487287, - "y-velocity": 0.0012471767764183842, - "pressure": 0.0031353064368674637, - "turbulent_viscosity": 0.02861798351376769 - }, - "MSE_normalized_surfacic": { - "pressure": 0.008781626853551213 - }, - "MAPE_normalized": { - "x-velocity": 0.23535119740284544, - "y-velocity": 0.17397767012401485, - "pressure": 0.165990814219197, - "turbulent_viscosity": 0.43160989540100914 - }, - "MAPE_normalized_surfacic": { - "pressure": 0.24832149427530473 - } - }, - "Physics": { - "spearman_correlation_drag": 0.8187369684242107, - "spearman_correlation_lift": 0.9983799594989875, - "mean_relative_drag": 0.23041081909925168, - "std_relative_drag": 0.2891833638823285, - "mean_relative_lift": 0.0611844697009299, - "std_relative_lift": 0.15944521815240825 - } - } - }, - "fc_metrics_test_ood": { - "test_ood": { - "ML": { - "MSE_normalized": { - "x-velocity": 0.009187407726621506, - "y-velocity": 0.0027280858877766494, - "pressure": 0.016331825251314956, - "turbulent_viscosity": 0.049333853470072006 - }, - "MSE_normalized_surfacic": { - "pressure": 0.06845324941442453 - }, - "MAPE_normalized": { - "x-velocity": 0.4080580649129622, - "y-velocity": 0.210093600068407, - "pressure": 0.25951081418011784, - "turbulent_viscosity": 0.46848256950820355 - }, - "MAPE_normalized_surfacic": { - "pressure": 0.32298146380673 - } - }, - "Physics": { - "spearman_correlation_drag": 0.7894712360182601, - "spearman_correlation_lift": 0.9971664788339026, - "mean_relative_drag": 0.3940285291162256, - "std_relative_drag": 0.46237913032263045, - "mean_relative_lift": 0.08437221869348716, - "std_relative_lift": 0.2164953143653425 - } - } - } -} \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json b/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json deleted file mode 100644 index cdd7bc0..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/res/json_metrics_new.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "ML": { - "x-velocity": 0.00262499232487287, - "y-velocity": 0.0012471767764183842, - "pressure": 0.0031353064368674637, - "turbulent_viscosity": 0.02861798351376769 - }, - "Physics": { - "spearman_correlation_drag": 0.8187369684242107, - "spearman_correlation_lift": 0.9983799594989875, - "mean_relative_drag": 0.23041081909925168, - "mean_relative_lift": 0.0611844697009299 - }, - "Speed" : {"inference_time": 699.8024816513062}, - "OOD": { - "ML": { - "x-velocity": 0.009187407726621506, - "y-velocity": 0.0027280858877766494, - "pressure": 0.016331825251314956, - "turbulent_viscosity": 0.049333853470072006 - }, - "Physics": { - "spearman_correlation_drag": 0.7894712360182601, - "spearman_correlation_lift": 0.9971664788339026, - "mean_relative_drag": 0.3940285291162256, - "mean_relative_lift": 0.08437221869348716 - } - } -} diff --git a/score_scripts/NeurIPS_scoreV2_8/score.py b/score_scripts/NeurIPS_scoreV2_8/score.py deleted file mode 100644 index 3784999..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/score.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python -# Scoring program for the AirfRANSModel challenge. -# Use directly the outputfile from the LIPS evaluation program. -# Some libraries and options -import os -from sys import argv -import sys -import json -import math - - -# Default I/O directories: -root_dir = "/app/" -default_input_dir = root_dir + "scoring_input" -default_output_dir = root_dir + "scoring_output" -default_input_dir = "./" -default_output_dir="./" - -# Constant used for a missing score -missing_score = -0.999999 - -def _HERE(*args): - h = os.path.dirname(os.path.realpath(__file__)) - return os.path.join(h, *args) - -class ModelApiError(Exception): - """Model api error""" - - def __init__(self, msg=""): - self.msg = msg - print(msg) - - -def import_metrics(input_dir): - ## import parameters.json as a dictionary - path_submission_parameters = os.path.join(input_dir, 'res', 'json_metrics.json') - if not os.path.exists(path_submission_parameters): - raise ModelApiError("Missing json_metrics.json file") - exit_program() - with open(path_submission_parameters) as json_file: - metrics = json.load(json_file) - return metrics - -def exit_program(): - print("Error exiting") - sys.exit(0) - -def SpeedMetric(speedUp,speedMax): - return max(min(math.log10(speedUp)/math.log10(speedMax),1),0) - -# =============================== MAIN ======================================== - -if __name__ == "__main__": - - #### INPUT/OUTPUT: Get input and output directory names - if len(argv) == 1: # Use the default input and output directories if no arguments are provided - input_dir = default_input_dir - output_dir = default_output_dir - else: - input_dir = argv[1] - output_dir = argv[2] - # Create the output directory, if it does not already exist and open output files - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - score_file = open(os.path.join(output_dir, 'scores.txt'), 'w') - html_file = open(os.path.join(output_dir, 'scores.html'), 'w') - - # Init HTML - html_file.write("Scoring program output") - - ## thresholds - # First competition thresholds - # thresholds={"x-velocity":(0.1,0.2,"min"), - # "y-velocity":(0.1,0.2,"min"), - # "pressure":(0.02,0.1,"min"), - # "pressure_surfacic":(0.08,0.2,"min"), - # "turbulent_viscosity":(0.5,1.0,"min"), - # "mean_relative_drag":(1.0,10.0,"min"), - # "mean_relative_lift":(0.2,0.5,"min"), - # "spearman_correlation_drag":(0.5,0.8,"max"), - # "spearman_correlation_lift":(0.94,0.98,"max") - # } - - # NeurIPS competition thresholds - thresholds={"x-velocity":(0.01,0.02,"min"), - "y-velocity":(0.01,0.02,"min"), - "pressure":(0.002,0.01,"min"), - "pressure_surfacic":(0.008,0.02,"min"), - "turbulent_viscosity":(0.05,0.1,"min"), - "mean_relative_drag":(0.4,5.0,"min"), - "mean_relative_lift":(0.1,0.3,"min"), - "spearman_correlation_drag":(0.8,0.9,"max"), - "spearman_correlation_lift":(0.96,0.99,"max") - } - - - # scoring configuration - configuration={ - "coefficients":{"ML":0.4,"OOD":0.3,"Physics":0.3}, - "ratioRelevance":{"Speed-up":0.25,"Accuracy":0.75}, - "valueByColor":{"g":2,"o":1,"r":0}, - "maxSpeedRatioAllowed":10000, - "reference_mean_simulation_time":1500 - } - - coefficients=configuration["coefficients"] - ratioRelevance=configuration["ratioRelevance"] - valueByColor=configuration["valueByColor"] - maxSpeedRatioAllowed=configuration["maxSpeedRatioAllowed"] - - #Import metrics - metrics = import_metrics(input_dir) - fc_metrics_test = metrics["fc_metrics_test"] - fc_metrics_test_ood = metrics["fc_metrics_test_ood"] - test_mean_simulation_time = metrics["test_mean_simulation_time"] - test_ood_mean_simulation_time = metrics["test_ood_mean_simulation_time"] - - - ## Extract metrics of interest - ML_metrics = "ML" - ml_metrics = fc_metrics_test["test"][ML_metrics]["MSE_normalized"] - ml_metrics["pressure_surfacic"] = fc_metrics_test["test"][ML_metrics]["MSE_normalized_surfacic"]["pressure"] - - physic_compliances = "Physics" - phy_variables_to_keep = ["mean_relative_drag","mean_relative_lift","spearman_correlation_drag","spearman_correlation_lift"] - phy_metrics = {phy_variable:fc_metrics_test["test"][physic_compliances][phy_variable] for phy_variable in phy_variables_to_keep} - - ml_ood_metrics = fc_metrics_test_ood["test_ood"][ML_metrics]["MSE_normalized"] - ml_ood_metrics["pressure_surfacic"] = fc_metrics_test_ood["test_ood"][ML_metrics]["MSE_normalized_surfacic"]["pressure"] - phy_ood_metrics = {phy_variable:fc_metrics_test_ood["test_ood"][physic_compliances][phy_variable] for phy_variable in phy_variables_to_keep} - ood_metrics = {**ml_ood_metrics,**phy_ood_metrics} - - allmetrics={ - ML_metrics:ml_metrics, - physic_compliances:phy_metrics, - "OOD":ood_metrics - } - - print(allmetrics) - - ## Compute speed-up : - - - reference_mean_simulation_time=configuration["reference_mean_simulation_time"] - speedUp={ - ML_metrics:reference_mean_simulation_time/test_mean_simulation_time, - "OOD":reference_mean_simulation_time/test_mean_simulation_time - } - - accuracyResults=dict() - - - for subcategoryName, subcategoryVal in allmetrics.items(): - accuracyResults[subcategoryName]=[] - html_file.write("

" + subcategoryName + "

") - for variableName, variableError in subcategoryVal.items(): - thresholdMin,thresholdMax,evalType = thresholds[variableName] - if evalType=="min": - if variableError" + subcategoryName + " - " + variableName + " - " + ": "+accuracyEval+"

") - html_file.write("
") - - - - - - ## ML subscore - - - mlSubscore=0 - - #Compute accuracy - accuracyMaxPoints=ratioRelevance["Accuracy"] - accuracyResult=sum([valueByColor[color] for color in accuracyResults["ML"]]) - accuracyResult=accuracyResult*accuracyMaxPoints/(len(accuracyResults["ML"])*max(valueByColor.values())) - mlSubscore+=accuracyResult - - #Compute speed-up - speedUpMaxPoints=ratioRelevance["Speed-up"] - speedUpResult=SpeedMetric(speedUp=speedUp["ML"],speedMax=maxSpeedRatioAllowed) - speedUpResult=speedUpResult*speedUpMaxPoints - mlSubscore+=speedUpResult - - ## compute Physics subscore - # Compute accuracy - accuracyResult=sum([valueByColor[color] for color in accuracyResults["Physics"]]) - accuracyResult=accuracyResult/(len(accuracyResults["Physics"])*max(valueByColor.values())) - physicsSubscore=accuracyResult - - ## Compute OOD subscore - - oodSubscore=0 - - #Compute accuracy - accuracyMaxPoints=ratioRelevance["Accuracy"] - accuracyResult=sum([valueByColor[color] for color in accuracyResults["OOD"]]) - accuracyResult=accuracyResult*accuracyMaxPoints/(len(accuracyResults["OOD"])*max(valueByColor.values())) - oodSubscore+=accuracyResult - - #Compute speed-up - speedUpMaxPoints=ratioRelevance["Speed-up"] - speedUpResult=SpeedMetric(speedUp=speedUp["OOD"],speedMax=maxSpeedRatioAllowed) - speedUpResult=speedUpResult*speedUpMaxPoints - oodSubscore+=speedUpResult - - ## Compute global score - globalScore=100*(coefficients["ML"]*mlSubscore+coefficients["Physics"]*physicsSubscore+coefficients["OOD"]*oodSubscore) - - print(globalScore) - - # Write scores in the output file - # exemple write score - score_file.write("global_warmup" + ": %0.12f\n" % globalScore) - score_file.write("ML_warmup" + ": %0.12f\n" % mlSubscore) - score_file.write("Physics_warmup" + ": %0.12f\n" % physicsSubscore) - score_file.write("OOD_warmup" + ": %0.12f\n" % oodSubscore) - - html_file.write("
") - html_file.write("

Global score: %0.12f

" % globalScore) - html_file.write("

ML score: %0.12f

" % mlSubscore) - html_file.write("

Physics score: %0.12f

" % physicsSubscore) - html_file.write("

OOD score: %0.12f

" % oodSubscore) - html_file.write("") - - html_file.close() - score_file.close() \ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/scores.html b/score_scripts/NeurIPS_scoreV2_8/scores.html deleted file mode 100644 index 1b38d27..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/scores.html +++ /dev/null @@ -1 +0,0 @@ -Scoring program output

ML

ML - x-velocity - : g

ML - y-velocity - : g

ML - pressure - : o

ML - turbulent_viscosity - : g

ML - pressure_surfacic - : o


Physics

Physics - mean_relative_drag - : g

Physics - mean_relative_lift - : g

Physics - spearman_correlation_drag - : o

Physics - spearman_correlation_lift - : g


OOD

OOD - x-velocity - : g

OOD - y-velocity - : g

OOD - pressure - : r

OOD - turbulent_viscosity - : g

OOD - pressure_surfacic - : r

OOD - mean_relative_drag - : g

OOD - mean_relative_lift - : g

OOD - spearman_correlation_drag - : r

OOD - spearman_correlation_lift - : g



Global score: 76.765637772046

ML score: 0.764509111029

Physics score: 0.875000000000

OOD score: 0.664509111029

\ No newline at end of file diff --git a/score_scripts/NeurIPS_scoreV2_8/scores.txt b/score_scripts/NeurIPS_scoreV2_8/scores.txt deleted file mode 100644 index 81e03a4..0000000 --- a/score_scripts/NeurIPS_scoreV2_8/scores.txt +++ /dev/null @@ -1,4 +0,0 @@ -global_warmup: 76.765637772046 -ML_warmup: 0.764509111029 -Physics_warmup: 0.875000000000 -OOD_warmup: 0.664509111029 diff --git a/score_scripts/PowerGrid_score/metadata b/score_scripts/PowerGrid_score/metadata deleted file mode 100644 index 04b3b36..0000000 --- a/score_scripts/PowerGrid_score/metadata +++ /dev/null @@ -1,2 +0,0 @@ -command: python $program/score.py $input $output -description: Compute scores for the competition diff --git a/score_scripts/PowerGrid_score/res/json_metrics.json b/score_scripts/PowerGrid_score/res/json_metrics.json deleted file mode 100644 index b3bb13f..0000000 --- a/score_scripts/PowerGrid_score/res/json_metrics.json +++ /dev/null @@ -1,197 +0,0 @@ -{ - "test": { - "ML": { - "MSE_avg": { - "v_or": 0.00010645988368196413, - "v_ex": 0.00011296240700175986, - "p_or": 0.18411502242088318, - "p_ex": 0.1777658313512802, - "a_or": 16.51224136352539, - "a_ex": 16.86044692993164 - }, - "MAE_avg": { - "v_or": 0.001553428708575666, - "v_ex": 0.001500570448115468, - "p_or": 0.14518952369689941, - "p_ex": 0.14409466087818146, - "a_or": 1.4204329252243042, - "a_ex": 1.4652109146118164 - }, - "MAPE_avg": { - "v_or": 9.848606168816332e-06, - "v_ex": 1.0132289389730431e-05, - "p_or": 63017717760.0, - "p_ex": 22879956992.0, - "a_or": 121335185408.0, - "a_ex": 52108500992.0 - }, - "MAPE_90_avg": { - "v_or": 5.658699643585033e-06, - "v_ex": 5.919416245742559e-06, - "p_or": 0.0033098454767013486, - "p_ex": 0.0032964480543750145, - "a_or": 0.007386949784782199, - "a_ex": 0.007292245940119987 - }, - "MAPE_10_avg": { - "v_or": 7.93336312753172e-06, - "v_ex": 8.195988824322852e-06, - "p_or": 0.006397687786608211, - "p_ex": 0.006388048696331843, - "a_or": 0.021526892545594315, - "a_ex": 0.022162465019418712 - }, - "TIME_INF": 4.163906573085114 - }, - "Physics": { - "CURRENT_POS": { - "a_or": { - "Error": 10040.3349609375, - "Violation_proportion": 0.00044102150537634406 - }, - "a_ex": { - "Error": 11018.400390625, - "Violation_proportion": 0.0005930645161290322 - } - }, - "VOLTAGE_POS": { - "v_or": { - "Violation_proportion": 0.0 - }, - "v_ex": { - "Violation_proportion": 0.0 - } - }, - "LOSS_POS": { - "loss_criterion": 209.59898, - "violation_proportion": 0.008733548387096774 - }, - "DISC_LINES": { - "p_or": 0.0, - "p_ex": 0.0, - "a_or": 0.0, - "a_ex": 0.0, - "v_or": 0.0, - "v_ex": 0.0, - "violation_proportion": 0.0 - }, - "CHECK_LOSS": { - "violation_percentage": 1.539 - }, - "CHECK_GC": { - "violation_percentage": 0.0, - "mae": 9.128214e-05, - "wmape": 1.2492486e-06 - }, - "CHECK_LC": { - "violation_percentage": 77.35536440677966, - "mae": 0.10304196395080921, - "mape": 0.002859223697836113 - }, - "CHECK_JOULE_LAW": { - "violation_proportion": 0.0, - "mae": 3.054519562577347e-07, - "wmape": 2.268769940016548e-06 - } - }, - "IndRed": { - "TIME_INF": 3.9108043271116912 - } - }, - "test_ood_topo": { - "ML": { - "MSE_avg": { - "v_or": 0.0008741252822801471, - "v_ex": 0.0009632128058001399, - "p_or": 0.47769251465797424, - "p_ex": 0.4475671350955963, - "a_or": 40.1962890625, - "a_ex": 41.00724411010742 - }, - "MAE_avg": { - "v_or": 0.0023060280364006758, - "v_ex": 0.0023030198644846678, - "p_or": 0.18595275282859802, - "p_ex": 0.1843600869178772, - "a_or": 1.7880399227142334, - "a_ex": 1.8418012857437134 - }, - "MAPE_avg": { - "v_or": 1.5199337212834507e-05, - "v_ex": 1.5929399523884058e-05, - "p_or": 189589094400.0, - "p_ex": 56932474880.0, - "a_or": 386140471296.0, - "a_ex": 157286449152.0 - }, - "MAPE_90_avg": { - "v_or": 7.414690380649798e-06, - "v_ex": 8.032887808804589e-06, - "p_or": 0.004977200835027814, - "p_ex": 0.004950829903700901, - "a_or": 0.010261113576488012, - "a_ex": 0.01012922324923192 - }, - "MAPE_10_avg": { - "v_or": 1.0476555723965095e-05, - "v_ex": 1.1106274117571183e-05, - "p_or": 0.008155790825441843, - "p_ex": 0.008147190351295156, - "a_or": 0.027774648502433164, - "a_ex": 0.028373457148515593 - } - }, - "Physics": { - "CURRENT_POS": { - "a_or": { - "Error": 21769.7578125, - "Violation_proportion": 0.0004038709677419355 - }, - "a_ex": { - "Error": 23943.244140625, - "Violation_proportion": 0.0005124731182795699 - } - }, - "VOLTAGE_POS": { - "v_or": { - "Violation_proportion": 0.0 - }, - "v_ex": { - "Violation_proportion": 0.0 - } - }, - "LOSS_POS": { - "loss_criterion": 1076.4073, - "violation_proportion": 0.00829997311827957 - }, - "DISC_LINES": { - "p_or": 0.0, - "p_ex": 0.0, - "a_or": 0.0, - "a_ex": 0.0, - "v_or": 0.0, - "v_ex": 0.0, - "violation_proportion": 0.0 - }, - "CHECK_LOSS": { - "violation_percentage": 3.2485 - }, - "CHECK_GC": { - "violation_percentage": 0.0, - "mae": 9.088051e-05, - "wmape": 1.2194811e-06 - }, - "CHECK_LC": { - "violation_percentage": 83.45940254237289, - "mae": 0.14096058704044898, - "mape": 0.0039999913278262085 - }, - "CHECK_JOULE_LAW": { - "violation_proportion": 0.0, - "mae": 2.9970232693607007e-07, - "wmape": 2.1807441470482367e-06 - } - }, - "IndRed": {} - } -} \ No newline at end of file diff --git a/score_scripts/PowerGrid_score/res/json_metrics_new.json b/score_scripts/PowerGrid_score/res/json_metrics_new.json deleted file mode 100644 index 8a4b677..0000000 --- a/score_scripts/PowerGrid_score/res/json_metrics_new.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "ML": { - "a_ex": 0.007292245940119987, - "a_or": 0.007386949784782199, - "p_ex": 0.006388048696331843, - "p_or": 0.006397687786608211, - "v_ex": 0.001500570448115468, - "v_or": 0.001553428708575666}, - "Physics": { - "CHECK_GC": 0.0, - "CHECK_JOULE_LAW": 0.0, - "CHECK_LC": 77.35536440677966, - "CHECK_LOSS": 1.539, - "CURRENT_POS": 0.04410215053763441, - "DISC_LINES": 0.0, - "LOSS_POS": 0.8733548387096773, - "VOLTAGE_POS": 0.0}, - "Speed" : {"inference_time": 699.8024816513062}, - "OOD": { - "ML": { - "a_ex": 0.01012922324923192, - "a_or": 0.010261113576488012, - "p_ex": 0.008147190351295156, - "p_or": 0.008155790825441843, - "v_ex": 0.0023030198644846678, - "v_or": 0.0023060280364006758}, - "Physics": { - "CHECK_GC": 0.0, - "CHECK_JOULE_LAW": 0.0, - "CHECK_LC": 83.45940254237289, - "CHECK_LOSS": 3.2485, - "CURRENT_POS": 0.04038709677419355, - "DISC_LINES": 0.0, - "LOSS_POS": 0.8299973118279571, - "VOLTAGE_POS": 0.0} - } -} - - diff --git a/score_scripts/PowerGrid_score/score.py b/score_scripts/PowerGrid_score/score.py deleted file mode 100644 index e230b1f..0000000 --- a/score_scripts/PowerGrid_score/score.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# Use directly the outputfile from the LIPS evaluation program. -# Some libraries and options -import os -from sys import argv -import sys -import json -import math - -import utils.compute_score as cs - -# Default I/O directories: -root_dir = "/app/" -default_input_dir = root_dir + "scoring_input" -default_output_dir = root_dir + "scoring_output" -default_input_dir = "./" -default_output_dir="./" - - - -def _HERE(*args): - h = os.path.dirname(os.path.realpath(__file__)) - return os.path.join(h, *args) - -class ModelApiError(Exception): - """Model api error""" - - def __init__(self, msg=""): - self.msg = msg - print(msg) - - -def import_metrics(input_dir): - ## import parameters.json as a dictionary - path_submission_parameters = os.path.join(input_dir, 'res', 'json_metrics.json') - # path_submission_parameters = os.path.join(input_dir, 'json_metrics.json') - print(path_submission_parameters) - if not os.path.exists(path_submission_parameters): - raise ModelApiError("Missing json_metrics.json file") - exit_program() - with open(path_submission_parameters) as json_file: - metrics = json.load(json_file) - return metrics - - - -# =============================== MAIN ======================================== - -if __name__ == "__main__": - - #### INPUT/OUTPUT: Get input and output directory names - if len(argv) == 1: # Use the default input and output directories if no arguments are provided - input_dir = default_input_dir - output_dir = default_output_dir - else: - input_dir = argv[1] - output_dir = argv[2] - # Create the output directory, if it does not already exist and open output files - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - - - score_file = open(os.path.join(output_dir, 'scores.txt'), 'w') - html_file = open(os.path.join(output_dir, 'scores.html'), 'w') - print("Scoring file : ", os.path.join(output_dir, 'scores.txt')) - metrics = import_metrics(input_dir) - metrics["solver_time"] = 32.79 - - globalScore, test_ml_subscore, test_physics_subscore, test_ood_ml_subscore, test_ood_physics_subscore, speedup_score = cs.compute_global_score(metrics) - - print("scoring done ", globalScore, test_ml_subscore, test_physics_subscore, test_ood_ml_subscore, test_ood_physics_subscore, speedup_score) - - score_file.write("global" + ": %0.12f\n" % globalScore) - score_file.write("ML_test" + ": %0.12f\n" % test_ml_subscore) - score_file.write("Physics_test" + ": %0.12f\n" % test_physics_subscore) - score_file.write("ML_ood" + ": %0.12f\n" % test_ood_ml_subscore) - score_file.write("Physics_ood" + ": %0.12f\n" % test_ood_physics_subscore) - score_file.write("speedup" + ": %0.12f\n" % speedup_score) - - html_file.write("Scoring program output") - html_file.write("

Global Score :

") - html_file.write("

" + ": %0.12f\n" % globalScore) - html_file.write("

") - - html_file.close() - score_file.close() \ No newline at end of file diff --git a/score_scripts/PowerGrid_score/scores.html b/score_scripts/PowerGrid_score/scores.html deleted file mode 100644 index e69de29..0000000 diff --git a/score_scripts/PowerGrid_score/scores.txt b/score_scripts/PowerGrid_score/scores.txt deleted file mode 100644 index e69de29..0000000 diff --git a/score_scripts/PowerGrid_score/utils/__init__.py b/score_scripts/PowerGrid_score/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/score_scripts/PowerGrid_score/utils/compute_score.py b/score_scripts/PowerGrid_score/utils/compute_score.py deleted file mode 100644 index cea5ff6..0000000 --- a/score_scripts/PowerGrid_score/utils/compute_score.py +++ /dev/null @@ -1,162 +0,0 @@ -import math -from lips.metrics.power_grid.compute_solver_time import compute_solver_time -from lips.metrics.power_grid.compute_solver_time_grid2op import compute_solver_time_grid2op - -thresholds={"a_or":(0.02,0.05,"min"), - "a_ex":(0.02,0.05,"min"), - "p_or":(0.02,0.05,"min"), - "p_ex":(0.02,0.05,"min"), - "v_or":(0.2,0.5,"min"), - "v_ex":(0.2,0.5,"min"), - "CURRENT_POS":(1., 5.,"min"), - "VOLTAGE_POS":(1.,5.,"min"), - "LOSS_POS":(1.,5.,"min"), - "DISC_LINES":(1.,5.,"min"), - "CHECK_LOSS":(1.,5.,"min"), - "CHECK_GC":(0.05,0.10,"min"), - "CHECK_LC":(0.05,0.10,"min"), - "CHECK_JOULE_LAW":(1.,5.,"min") - } - -configuration={ - "coefficients":{"test":0.3, "test_ood":0.3, "speed_up":0.4}, - "test_ratio":{"ml": 0.66, "physics":0.34}, - "test_ood_ratio":{"ml": 0.66, "physics":0.34}, - "value_by_color":{"g":2,"o":1,"r":0}, - "max_speed_ratio_allowed":50 -} - -def evaluate_model(benchmark, model): - metrics = benchmark.evaluate_simulator(augmented_simulator=model, - eval_batch_size=128, - dataset="all", - shuffle=False, - save_path=None, - save_predictions=False - ) - return metrics - -def compute_speed_up_metrics( metrics): - speed_up = metrics["solver_time"] / metrics["test"]["ML"]["TIME_INF"] - return speed_up - -def compute_speed_up(config, metrics): - # solver_time = compute_solver_time(nb_samples=int(1e5), config=config) - solver_time = compute_solver_time_grid2op(config_path=config.path_config, benchmark_name=config.section_name, nb_samples=int(1e5)) - speed_up = solver_time / metrics["test"]["ML"]["TIME_INF"] - return speed_up - -def reconstruct_metric_dict(metrics, dataset: str="test"): - rec_metrics = dict() - rec_metrics["ML"] = dict() - rec_metrics["Physics"] = dict() - - rec_metrics["ML"]["a_or"] = metrics[dataset]["ML"]["MAPE_90_avg"]["a_or"] - rec_metrics["ML"]["a_ex"] = metrics[dataset]["ML"]["MAPE_90_avg"]["a_ex"] - rec_metrics["ML"]["p_or"] = metrics[dataset]["ML"]["MAPE_10_avg"]["p_or"] - rec_metrics["ML"]["p_ex"] = metrics[dataset]["ML"]["MAPE_10_avg"]["p_ex"] - rec_metrics["ML"]["v_or"] = metrics[dataset]["ML"]["MAE_avg"]["v_or"] - rec_metrics["ML"]["v_ex"] = metrics[dataset]["ML"]["MAE_avg"]["v_ex"] - - rec_metrics["Physics"]["CURRENT_POS"] = metrics[dataset]["Physics"]["CURRENT_POS"]["a_or"]["Violation_proportion"] * 100. - rec_metrics["Physics"]["VOLTAGE_POS"] = metrics[dataset]["Physics"]["VOLTAGE_POS"]["v_or"]["Violation_proportion"] * 100. - rec_metrics["Physics"]["LOSS_POS"] = metrics[dataset]["Physics"]["LOSS_POS"]["violation_proportion"] * 100. - rec_metrics["Physics"]["DISC_LINES"] = metrics[dataset]["Physics"]["DISC_LINES"]["violation_proportion"] * 100. - rec_metrics["Physics"]["CHECK_LOSS"] = metrics[dataset]["Physics"]["CHECK_LOSS"]["violation_percentage"] - rec_metrics["Physics"]["CHECK_GC"] = metrics[dataset]["Physics"]["CHECK_GC"]["violation_percentage"] - rec_metrics["Physics"]["CHECK_LC"] = metrics[dataset]["Physics"]["CHECK_LC"]["violation_percentage"] - rec_metrics["Physics"]["CHECK_JOULE_LAW"] = metrics[dataset]["Physics"]["CHECK_JOULE_LAW"]["violation_proportion"] * 100. - - return rec_metrics - -def discretize_results(metrics): - results=dict() - for subcategoryName, subcategoryVal in metrics.items(): - results[subcategoryName]=[] - for variableName, variableError in subcategoryVal.items(): - thresholdMin,thresholdMax,evalType=thresholds[variableName] - if evalType=="min": - if variableError Date: Mon, 10 Mar 2025 10:17:15 +0100 Subject: [PATCH 17/18] clean code logic --- lips/scoring/__init__.py | 3 +- lips/scoring/airfoil_powergrid_scoring.py | 18 +++- lips/scoring/ml4physim_powergrid_socring.py | 18 +++- lips/scoring/powergrid_scoring.py | 103 -------------------- 4 files changed, 33 insertions(+), 109 deletions(-) delete mode 100644 lips/scoring/powergrid_scoring.py diff --git a/lips/scoring/__init__.py b/lips/scoring/__init__.py index ddc6de8..199bfad 100644 --- a/lips/scoring/__init__.py +++ b/lips/scoring/__init__.py @@ -9,11 +9,10 @@ from __future__ import absolute_import from lips.scoring.scoring import Scoring -from lips.scoring.powergrid_scoring import PowerGridScoring from lips.scoring.airfoil_powergrid_scoring import AirfoilPowerGridScoring from lips.scoring.ml4physim_powergrid_socring import ML4PhysimPowerGridScoring __all__ = [ - "Scoring", "PowerGridScoring", "AirfoilPowerGridScoring", "ML4PhysimPowerGridScoring" + "Scoring", "AirfoilPowerGridScoring", "ML4PhysimPowerGridScoring" ] \ No newline at end of file diff --git a/lips/scoring/airfoil_powergrid_scoring.py b/lips/scoring/airfoil_powergrid_scoring.py index 2fe7f94..f119f98 100644 --- a/lips/scoring/airfoil_powergrid_scoring.py +++ b/lips/scoring/airfoil_powergrid_scoring.py @@ -9,12 +9,12 @@ import math from typing import Union, Dict, List -from .powergrid_scoring import PowerGridScoring +from .scoring import Scoring from .utils import get_nested_value, filter_metrics, read_json from ..config import ConfigManager -class AirfoilPowerGridScoring(PowerGridScoring): +class AirfoilPowerGridScoring(Scoring): """ Calculates the score for the AirFoil Power Grid competition: https://www.codabench.org/competitions/3282/ """ @@ -171,3 +171,17 @@ def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: s global_score = self.calculate_global_score(sub_scores_values) return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} + + def _calculate_speed_up(self, time_inference: float) -> float: + """ + Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + Args: + time_inference: The inference time in seconds. + + Returns: + The calculated speedup factor. + """ + + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] + return time_classical_solver / time_inference \ No newline at end of file diff --git a/lips/scoring/ml4physim_powergrid_socring.py b/lips/scoring/ml4physim_powergrid_socring.py index 4486c83..b7cda3b 100644 --- a/lips/scoring/ml4physim_powergrid_socring.py +++ b/lips/scoring/ml4physim_powergrid_socring.py @@ -9,12 +9,12 @@ import math from typing import Union, Dict, List -from .powergrid_scoring import PowerGridScoring +from .scoring import Scoring from .utils import get_nested_value, read_json from ..config import ConfigManager -class ML4PhysimPowerGridScoring(PowerGridScoring): +class ML4PhysimPowerGridScoring(Scoring): """ Calculates the score for the ML4Physim Power Grid competition: https://www.codabench.org/competitions/2378/ """ @@ -184,3 +184,17 @@ def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: s global_score = self.calculate_global_score(sub_scores_values) return {"Score Colors": sub_scores_color, "Score values": sub_scores_values, "Global Score": global_score} + + def _calculate_speed_up(self, time_inference: float) -> float: + """ + Calculates the speedup factor based on: + SpeedUp = time_ClassicalSolver / time_Inference + Args: + time_inference: The inference time in seconds. + + Returns: + The calculated speedup factor. + """ + + time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] + return time_classical_solver / time_inference \ No newline at end of file diff --git a/lips/scoring/powergrid_scoring.py b/lips/scoring/powergrid_scoring.py deleted file mode 100644 index 3413225..0000000 --- a/lips/scoring/powergrid_scoring.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2021, IRT SystemX (https://www.irt-systemx.fr/en/) -# See AUTHORS.txt -# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. -# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, -# you can obtain one at http://mozilla.org/MPL/2.0/. -# SPDX-License-Identifier: MPL-2.0 -# This file is part of LIPS, LIPS is a python platform for power networks benchmarking - -from abc import abstractmethod -from typing import Union, Dict - -from lips.config import ConfigManager -from lips.scoring import Scoring - - -class PowerGridScoring(Scoring): - """ - Abstract base class for calculating scores for power grid use cases. - """ - - def __init__(self, config: Union[ConfigManager, None] = None, config_path: Union[str, None] = None, - config_section: Union[str, None] = None, log_path: Union[str, None] = None): - """ - Initializes the PowerGridScoring instance with configuration and logger. - - Args: - config: A ConfigManager instance. Defaults to None. - config_path: Path to the configuration file. Defaults to None. - config_section: Section of the configuration file. Defaults to None. - log_path: Path to the log file. Defaults to None. - """ - super().__init__(config=config, config_path=config_path, config_section=config_section, log_path=log_path) - - @abstractmethod - def _reconstruct_ml_metrics(self, **kwargs) -> Dict: - """ - Reconstructs the ML metrics from the raw data. - - Returns: - A dictionary containing the reconstructed ML metrics. - """ - pass - - @abstractmethod - def _reconstruct_physic_metrics(self, **kwargs) -> Dict: - """ - Reconstructs the physics metrics from the raw data. - - Returns: - A dictionary containing the reconstructed physics metrics. - """ - pass - - @abstractmethod - def _reconstruct_ood_metrics(self, **kwargs) -> Dict: - """ - Reconstructs the OOD metrics from the raw data. - - Returns: - A dictionary containing the reconstructed OOD metrics. - """ - pass - - @abstractmethod - def compute_scores(self, metrics_dict: Union[Dict, None] = None, metrics_path: str = "") -> Dict: - """ - Computes the scores based on the provided metrics. - - Args: - metrics_dict: A dictionary containing the metrics data. - metrics_path: The path to a JSON file containing the metrics data. - - Returns: - A dictionary containing the calculated scores. - """ - pass - - @abstractmethod - def compute_speed_score(self, time_inference: float) -> float: - """ - Computes the speed score based on the inference time. - - Args: - time_inference: The inference time. - - Returns: - The calculated speed score. - """ - pass - - def _calculate_speed_up(self, time_inference: float) -> float: - """ - Calculates the speedup factor based on: - SpeedUp = time_ClassicalSolver / time_Inference - Args: - time_inference: The inference time in seconds. - - Returns: - The calculated speedup factor. - """ - - time_classical_solver = self.thresholds["reference_mean_simulation_time"]["thresholds"][0] - return time_classical_solver / time_inference From bd8eefaadf7dcd15efe810299d1c22a9570cf9be Mon Sep 17 00:00:00 2001 From: seifou23i Date: Tue, 11 Mar 2025 10:32:13 +0100 Subject: [PATCH 18/18] scoring getting started notebook --- .../PowerGridUsecase/05_Scoring.ipynb | 309 ++++++++++++++++++ 1 file changed, 309 insertions(+) create mode 100644 getting_started/PowerGridUsecase/05_Scoring.ipynb diff --git a/getting_started/PowerGridUsecase/05_Scoring.ipynb b/getting_started/PowerGridUsecase/05_Scoring.ipynb new file mode 100644 index 0000000..b1438b6 --- /dev/null +++ b/getting_started/PowerGridUsecase/05_Scoring.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5648efb1-f90f-4261-a1cc-dba902b1b173", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-10T09:47:19.828698Z", + "start_time": "2025-03-10T09:47:19.823375Z" + } + }, + "source": [ + "# Getting Started with the Scoring Feature" + ] + }, + { + "cell_type": "markdown", + "id": "4da8963b9847353", + "metadata": {}, + "source": [ + "This notebook demonstrates how to use the scoring feature from the LIPS package. We will cover the following topics:\n", + "\n", + "- **Defining Scoring Based on Configuration File**: We will show how to define scoring configurations and explain the structure of the configuration file.\n", + "- **Application to Competitions**: We will apply the scoring feature to two competitions: the airfoil competition and the powergrid competition." + ] + }, + { + "cell_type": "markdown", + "id": "35507c90-988e-4954-8351-9655d172503f", + "metadata": {}, + "source": [ + "## 1. Configuration file" + ] + }, + { + "cell_type": "markdown", + "id": "ee8cad8d-560d-4741-9274-038dbc2d86e4", + "metadata": {}, + "source": [ + "The core configuration for scoring is located in **configurations/powergrid/scoring/ScoreConfig.ini**. This file defines the scoring logic and allows for flexible customization of your score evaluation process. Three key sections are essential within this configuration: Thresholds, ValueByColor, and Coefficients." + ] + }, + { + "cell_type": "markdown", + "id": "4a9daea4-1b29-4d96-9bd8-7879cd561f54", + "metadata": {}, + "source": [ + "### 1. Thresholds: Defining Performance Boundaries\n", + "\n", + "The `Thresholds` section specifies the performance benchmarks for individual metrics. It includes both the threshold values and the comparison type, indicating whether to minimize, maximize, or evaluate a ratio.\n", + "\n", + "* **Comparison Types:**\n", + " * `minimize`: Lower values are considered better.\n", + " * `maximize`: Higher values are considered better.\n", + " * `ratio`: used to compare two values, if the ratio is under or above the thresholds.\n", + "* **Example:**\n", + " ```ini\n", + " \"a_or\": {\"comparison_type\": \"minimize\", \"thresholds\": [0.02, 0.05]}\n", + " ```\n", + " * **Explanation:** This entry defines thresholds for the metric \"a_or\". If the \"a_or\" value is less than 0.02, it falls within the best (green) range. If it's between 0.02 and 0.05, it falls within the middle (orange) range. If it's greater than 0.05, it falls within the worst (red) range. Because the comparison type is minimize, the lower the a_or value is the better.\n", + "\n", + "### 2. ValueByColor: Assigning Scores to Performance Ranges\n", + "\n", + "The `ValueByColor` section maps performance ranges (represented by colors) to numerical scores. This allows you to quantify the qualitative assessment of your metrics.\n", + "\n", + "* **Example:**\n", + " ```ini\n", + " {\"green\": 2, \"orange\": 1, \"red\": 0}\n", + " ```\n", + " * **Explanation:** In this example, a metric falling within the \"green\" range receives a score of 2, \"orange\" receives 1, and \"red\" receives 0.\n", + "\n", + "### 3. Coefficients: Weighting Metric Contributions\n", + "\n", + "The `Coefficients` section defines the relative importance of different metrics and sub-metrics in the overall score. This allows you to prioritize certain aspects of your model's performance.\n", + "\n", + "* **Hierarchical Structure:** Coefficients can be organized hierarchically, allowing you to assign weights to both top-level metrics (e.g., \"ML\", \"OOD\", \"Physics\") and their sub-metrics (e.g., \"Accuracy\", \"SpeedUP\").\n", + "* **Example:**\n", + " ```ini\n", + " {\"ML\": {\"value\": 0.4, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}},\n", + " \"OOD\": {\"value\": 0.3, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}},\n", + " \"Physics\": {\"value\": 0.3}}\n", + " ```\n", + " * **Explanation:**\n", + " * The \"ML\" metric contributes 40% (0.4) to the overall score.\n", + " * Within \"ML\", \"Accuracy\" contributes 75% (0.75) and \"SpeedUP\" contributes 25% (0.25) to the \"ML\" sub-score.\n", + " * The \"OOD\" metric contributes 30% to the overall score, and also uses the same accuracy and speedup sub metric weights.\n", + " * The \"Physics\" metric contributes 30% to the overall score, and has no submetrics.\n", + " * This structure allows for fine-grained control over the scoring process, ensuring that the final score reflects the relative importance of different aspects of your model's performance.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1d9eda4-5953-4fda-b3fd-a8c73bab433d", + "metadata": {}, + "source": [ + "## 2. Metric format" + ] + }, + { + "cell_type": "markdown", + "id": "94f32854-1470-4274-b5b0-dbfdb3715cef", + "metadata": {}, + "source": [ + " The scoring feature expects metrics data in a nested dictionary format. The structure should represent a tree-like hierarchy, where each node\n", + " contains either sub-metrics or leaf metrics. Leaf metrics are the actual values that will be colorized and scored.\n", + "\n", + " Here's an example of the expected format:\n", + "\n", + " ```json\n", + " {\n", + " \"ML\": {\n", + " \"metric1\": 0.85,\n", + " \"metric2\": 0.20\n", + " },\n", + " \"OOD\": {\n", + " \"metric3\": 0.92,\n", + " \"metric4\": 0.15\n", + " },\n", + " \"Physics\": {\n", + " \"metric5\": 0.78,\n", + " \"metric6\": 0.30\n", + " }\n", + " }\n", + " ```\n", + "\n", + " In this example:\n", + " - The top-level keys (`ML`, `OOD`, `Physics`) represent different categories or components.\n", + " - Each category contains several metrics (e.g., `metric1`, `metric2`).\n", + " - The values associated with each metric are numerical values.\n", + "\n", + " The metrics can also be structured in deeper hierarchies:\n", + "\n", + " ```json\n", + " {\n", + " \"Category1\": {\n", + " \"SubCategory1\": {\n", + " \"metric1\": 0.75,\n", + " \"metric2\": 0.25\n", + " },\n", + " \"SubCategory2\": {\n", + " \"metric3\": 0.60,\n", + " \"metric4\": 0.40\n", + " }\n", + " },\n", + " \"Category2\": {\n", + " \"metric5\": 0.90,\n", + " \"metric6\": 0.10\n", + " }\n", + " }\n", + " ```\n", + "\n", + " It's important to maintain a consistent branching structure throughout the metrics data. This means that if one sub-category contains further nested\n", + " categories, all other sub-categories at the same level should also have the same structure.\n", + "\n", + " The keys of the metrics should match the keys defined in the configuration file. For example, if the configuration file defines thresholds for `metric1`, the metrics data must also contain `metric1`." + ] + }, + { + "cell_type": "markdown", + "id": "9fd4161c-5ed3-48af-b2ee-9e1fe7e7725a", + "metadata": {}, + "source": [ + "## End-to-End Scoring Example\n", + "This section demonstrates an end-to-end example of how to use the scoring feature. We will load a configuration file, read a metrics file, colorize the metrics, calculate sub-scores, and then calculate the global score." + ] + }, + { + "cell_type": "markdown", + "id": "233f97d0-f68c-4fea-88c1-e67b0583c2a3", + "metadata": {}, + "source": [ + "### 1. Define Example Configuration File" + ] + }, + { + "cell_type": "markdown", + "id": "985c202e-dc1d-4307-8a81-1df47cee3d57", + "metadata": {}, + "source": [ + "### 1. Define Example Configuration File\n", + "\n", + "First, let's define an example configuration file (`config.ini`).\n", + "This file contains the thresholds, value_by_color mappings, and coefficients needed for scoring." + ] + }, + { + "cell_type": "markdown", + "id": "783fb53d-6f20-49cc-89fc-0fc028ae6145", + "metadata": {}, + "source": [ + "```ini\n", + "[thresholds]\n", + "a_or = {\"comparison_type\": \"minimize\", \"thresholds\": [0.02, 0.05]}\n", + "spearman_correlation_drag = {\"comparison_type\": \"maximize\", \"thresholds\": [0.7, 0.9]}\n", + "inference_time = {\"comparison_type\": \"minimize\", \"thresholds\": [500, 700]}\n", + "[valuebycolor]\n", + "green = 2\n", + "orange = 1\n", + "red = 0\n", + "[coefficients]\n", + "ML = {\"value\": 0.4, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}}\n", + "OOD = {\"value\": 0.3, \"Accuracy\": {\"value\": 0.75}, \"SpeedUP\": {\"value\": 0.25}}\n", + "Physics = {\"value\": 0.3}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5dc2a889-8f22-4a4c-838c-9cf56c3fde2e", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#%% [markdown]\n", + "# ### 2. Define Example Metrics File\n", + "#\n", + "# Next, let's define an example metrics file (`metrics.json`).\n", + "# This file contains the metric values that we want to score.\n", + "#\n", + "# ```json\n", + "# {\n", + "# \"ML\": {\n", + "# \"a_or\": 0.01,\n", + "# \"spearman_correlation_drag\": 0.95\n", + "# },\n", + "# \"OOD\": {\n", + "# \"inference_time\": 400\n", + "# },\n", + "# \"Physics\": {\n", + "# \"a_or\": 0.06,\n", + "# \"spearman_correlation_drag\": 0.75\n", + "# },\n", + "# \"Speed\": {\n", + "# \"inference_time\": 600\n", + "# }\n", + "# }\n", + "# ```\n", + "\n", + "#%%\n", + "import json\n", + "from scoring import Scoring\n", + "from utils import read_json\n", + "\n", + "# Initialize the Scoring class with the path to the configuration file\n", + "scoring = Scoring(config_path=\"config.ini\")\n", + "\n", + "# Load metrics data from the JSON file\n", + "metrics_path = \"metrics.json\"\n", + "metrics_data = read_json(json_path=metrics_path)\n", + "\n", + "#%% [markdown]\n", + "# ### 3. Colorize Metrics\n", + "#\n", + "# Now, let's colorize the metrics using the `colorize_metrics` function.\n", + "# This will convert the numerical metric values into color strings based on the\n", + "# thresholds defined in the configuration file.\n", + "\n", + "#%%\n", + "# Colorize the metrics data\n", + "colorized_metrics = scoring.colorize_metrics(metrics_data)\n", + "print(\"Colorized Metrics:\", json.dumps(colorized_metrics, indent=4))\n", + "\n", + "#%% [markdown]\n", + "# ### 4. Calculate Sub-Scores\n", + "#\n", + "# Next, let's calculate the sub-scores using the `calculate_sub_scores` function.\n", + "# This will calculate the score for each sub-tree in the metrics data.\n", + "\n", + "#%%\n", + "# Calculate sub-scores\n", + "sub_scores = scoring.calculate_sub_scores(colorized_metrics)\n", + "print(\"Sub-Scores:\", json.dumps(sub_scores, indent=4))\n", + "\n", + "#%% [markdown]\n", + "# ### 5. Calculate Global Score\n", + "#\n", + "# Finally, let's calculate the global score using the `calculate_global_score` function.\n", + "# This will calculate the overall score for the entire metrics tree.\n", + "\n", + "#%%\n", + "# Calculate global score\n", + "global_score = scoring.calculate_global_score(sub_scores)\n", + "print(\"Global Score:\", global_score)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}