From c28e994e2c403f55df3228e49005c39acee25f72 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Sat, 23 Dec 2023 16:14:14 +0100 Subject: [PATCH 01/14] benchopt bindinds --- adastop/cli.py | 94 ++++++++++++++++++++++++++++---------- adastop/compare_agents.py | 3 +- adastop/data_processing.py | 15 ++++++ 3 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 adastop/data_processing.py diff --git a/adastop/cli.py b/adastop/cli.py index 18b29af..e3fc78b 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -2,40 +2,23 @@ import pickle import os from pathlib import Path +import subprocess import pandas as pd import numpy as np import matplotlib.pyplot as plt - +from .data_processing import process_benchopt from .compare_agents import MultipleAgentsComparator + LITTER_FILE = ".adastop_comparator.pkl" -@click.group() -@click.pass_context -def adastop(ctx): - """ - Program to perform adaptive stopping algorithm using csv file intput_file. - Use adastop sub-command --help to have help for a specific sub-command - """ - pass - -@adastop.command() -@click.option("--n-groups", default=5, show_default=True, help="Number of groups.") -@click.option("--n-permutations", default=10000, show_default=True, help="Number of random permutations.") -@click.option("--alpha", default=0.05, show_default=True, help="Type I error.") -@click.option("--beta", default=0.0, show_default=True, help="early accept parameter.") -@click.option("--seed", default=None, type=int, show_default=True, help="Random seed.") -@click.option("--compare-to-first", is_flag=True, show_default=True, default=False, help="Compare all algorithms to the first algorithm.") -@click.argument('input_file',required = True, type=str) -@click.pass_context -def compare(ctx, input_file, n_groups, n_permutations, alpha, beta, seed, compare_to_first): +def compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compare_to_first): """ - Perform one step of adaptive stopping algorithm using csv file intput_file. + Perform one step of adaptive stopping algorithm using the dataframe df. At first call, the comparator will be initialized with the arguments passed and then it will be saved to a save file in `.adastop_comparator.pkl`. """ - path_lf = Path(input_file).parent.absolute() / LITTER_FILE - df = pd.read_csv(input_file, index_col=0) + n_fits_per_group = len(df) n_agents = len(df.columns) if compare_to_first: @@ -86,6 +69,71 @@ def compare(ctx, input_file, n_groups, n_permutations, alpha, beta, seed, compar pickle.dump(comparator, fp) click.echo("Comparator Saved") + +@click.group() +@click.pass_context +def adastop(ctx): + """ + Program to perform adaptive stopping algorithm using csv file intput_file. + + Use adastop sub-command --help to have help for a specific sub-command + """ + pass + +@adastop.command() +@click.option("--n-groups", default=5, show_default=True, help="Number of groups.") +@click.option("--n-permutations", default=10000, show_default=True, help="Number of random permutations.") +@click.option("--alpha", default=0.05, show_default=True, help="Type I error.") +@click.option("--beta", default=0.0, show_default=True, help="early accept parameter.") +@click.option("--seed", default=None, type=int, show_default=True, help="Random seed.") +@click.option("--compare-to-first", is_flag=True, show_default=True, default=False, help="Compare all algorithms to the first algorithm.") +@click.argument('input_file',required = True, type=str) +@click.pass_context +def compare(ctx, input_file, n_groups, n_permutations, alpha, beta, seed, compare_to_first): + """ + Perform one step of adaptive stopping algorithm using csv file intput_file. + At first call, the comparator will be initialized with the arguments passed and then it will be saved to a save file in `.adastop_comparator.pkl`. + """ + path_lf = Path(input_file).parent.absolute() / LITTER_FILE + df = pd.read_csv(input_file, index_col=0) + compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compare_to_first) + + +@adastop.command() +@click.option("--n-groups", default=5, show_default=True, help="Number of groups.") +@click.option("--n-permutations", default=10000, show_default=True, help="Number of random permutations.") +@click.option("--alpha", default=0.05, show_default=True, help="Type I error.") +@click.option("--beta", default=0.0, show_default=True, help="early accept parameter.") +@click.option("--seed", default=None, type=int, show_default=True, help="Random seed.") +@click.option("--compare-to-first", is_flag=True, show_default=True, default=False, help="Compare all algorithms to the first algorithm.") +@click.option("--size-group", default=6, show_default=True, help="Number of groups.") +@click.argument('config_file',required = True, type=str) +@click.pass_context +def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alpha, beta, seed, compare_to_first): + """ + Perform one step of computing benchmark and then adaptive stopping algorithm. + The benchmark is supposed to be in the current directory. + """ + path_lf = Path(config_file).parent.absolute() / LITTER_FILE + + # if this is not first group, load data for comparator. + if os.path.isfile(path_lf): + with open(path_lf, 'rb') as fp: + comparator = pickle.load(fp) + k = comparator.k + else: + k = 0 + + subprocess.check_output(["benchopt", "run", ".", "--config", + config_file, "--env", "-r", str(size_group), + "--output", "adastop_result_file_"+str(k)]) + + df = process_benchopt("outputs/adastop_result_file_"+str(k)+".parquet") + df.to_csv("outputs/adastop_result_file_"+str(k)+".csv") + + compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compare_to_first) + + @adastop.command() @click.argument('folder',required = True, type=str) @click.pass_context diff --git a/adastop/compare_agents.py b/adastop/compare_agents.py index f9c81bd..551dcd0 100644 --- a/adastop/compare_agents.py +++ b/adastop/compare_agents.py @@ -270,6 +270,8 @@ def partial_compare(self, eval_values, verbose=True): admissible_values_sup = values[ self.level_spent + icumulative_probas <= clevel ] + if len(np.unique(values)) < 1/clevel: + print("WARNING: too many values are equal, or size of group too small, the test may not be precise.") if len(admissible_values_sup) > 0: bk_sup = admissible_values_sup[0] # the minimum admissible value @@ -324,7 +326,6 @@ def partial_compare(self, eval_values, verbose=True): - np.mean(Z[comp[1]][: ((k + 1) * self.n[comp[1]])] ) ) - if Tmax > bk_sup: id_reject = np.arange(len(current_decisions))[current_decisions== "continue"][imax] current_decisions[id_reject] = "reject" diff --git a/adastop/data_processing.py b/adastop/data_processing.py new file mode 100644 index 0000000..28bed52 --- /dev/null +++ b/adastop/data_processing.py @@ -0,0 +1,15 @@ +import pandas as pd + + +def process_benchopt(file): + """ + For now, suppose that there is only one dataset + """ + df = pd.read_parquet(file) + df= df[["solver_name",'objective_value','idx_rep']] + df_ret = { name : [] for name in df["solver_name"].unique()} + for rep in df["idx_rep"].unique(): + for solver in df["solver_name"].unique(): + df_rep_solver = df.loc[ (df["solver_name"]==solver) & (df["idx_rep"]==rep)] + df_ret[solver].append(df_rep_solver['objective_value'].iloc[-1]) + return pd.DataFrame(df_ret) From ab0876612a12d764b54559f16ce1c2a39cd37f50 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Tue, 26 Dec 2023 17:46:24 +0100 Subject: [PATCH 02/14] fix data handling in cli --- adastop/cli.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/adastop/cli.py b/adastop/cli.py index e3fc78b..991df49 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -116,17 +116,28 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp """ path_lf = Path(config_file).parent.absolute() / LITTER_FILE - # if this is not first group, load data for comparator. + if os.path.isfile(path_lf): with open(path_lf, 'rb') as fp: comparator = pickle.load(fp) - k = comparator.k + k = comparator.k else: k = 0 - subprocess.check_output(["benchopt", "run", ".", "--config", - config_file, "--env", "-r", str(size_group), - "--output", "adastop_result_file_"+str(k)]) + # if this is not first group, load data for comparator. + if os.path.isfile( "outputs/adastop_result_file_"+str(k)+".csv"): + df = pd.read_csv("outputs/adastop_result_file_"+str(k)+".csv", index_col=0) + else: + if k > 0: + solvers = comparator.agent_names + arg_solver = " -s "+" -s ".join(solvers) + subprocess.check_output(["benchopt", "run", ".", "--config", + config_file, "--env", "-r", str(size_group), + "--output", "adastop_result_file_"+str(k)]) + else: + subprocess.check_output(["benchopt", "run", ".", "--config", + config_file, "--env", "-r", str(size_group), + "--output", "adastop_result_file_"+str(k)]) df = process_benchopt("outputs/adastop_result_file_"+str(k)+".parquet") df.to_csv("outputs/adastop_result_file_"+str(k)+".csv") From bc4600a60d8b303d7293c3802395e9c3746c92b4 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Tue, 26 Dec 2023 17:54:17 +0100 Subject: [PATCH 03/14] add logging of solver --- adastop/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/adastop/cli.py b/adastop/cli.py index 991df49..11b8c2f 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -131,6 +131,7 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp if k > 0: solvers = comparator.agent_names arg_solver = " -s "+" -s ".join(solvers) + print("Doing comparisons for "+ ", ".join(solver)) subprocess.check_output(["benchopt", "run", ".", "--config", config_file, "--env", "-r", str(size_group), "--output", "adastop_result_file_"+str(k)]) From 93114384cb6e93e9cfcd958fd582e02ef6cd11e8 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 27 Dec 2023 10:08:20 +0100 Subject: [PATCH 04/14] handle limit cases of dataframe overpopulated --- adastop/cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/adastop/cli.py b/adastop/cli.py index 11b8c2f..36caba8 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -30,8 +30,9 @@ def compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compa if os.path.isfile(path_lf): with open(path_lf, 'rb') as fp: comparator = pickle.load(fp) + names = comparator.agent_names - Z = [np.hstack([comparator.eval_values[agent], df[agent]]) for agent in df.columns] + Z = [np.hstack([comparator.eval_values[agent], df[agent]]) for agent in names] if len(Z[0]) > comparator.K * n_fits_per_group: raise ValueError('Error: you tried to use more group than what was initially declared, this is not allowed by the theory.') assert "continue" in list(comparator.decisions.values()), "Test finished at last iteration." @@ -40,9 +41,11 @@ def compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compa comparator = MultipleAgentsComparator(n_fits_per_group, n_groups, n_permutations, comparisons, alpha, beta, seed) - Z = [df[agent].values for agent in df.columns] + names = df.columns - data = {df.columns[i] : Z[i] for i in range(len(df.columns))} + Z = [df[agent].values for agent in names] + + data = {names[i] : Z[i] for i in range(len(names))} # recover also the data of agent that were decided. if comparator.agent_names is not None: for agent in comparator.agent_names: From 0d2edae7820681d882f2c078ef829ace511e51c3 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 27 Dec 2023 10:11:50 +0100 Subject: [PATCH 05/14] print message --- adastop/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adastop/cli.py b/adastop/cli.py index 36caba8..ba7fdc2 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -134,7 +134,7 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp if k > 0: solvers = comparator.agent_names arg_solver = " -s "+" -s ".join(solvers) - print("Doing comparisons for "+ ", ".join(solver)) + print("Doing comparisons for "+str(len(solvers))+ "solvers: "+", ".join(solvers)) subprocess.check_output(["benchopt", "run", ".", "--config", config_file, "--env", "-r", str(size_group), "--output", "adastop_result_file_"+str(k)]) From 4727f4ccb1cb380b898e0989f3ace47e72f5420c Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 27 Dec 2023 10:17:47 +0100 Subject: [PATCH 06/14] keep only undecided for next interim --- adastop/cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/adastop/cli.py b/adastop/cli.py index ba7fdc2..04f6b61 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -132,9 +132,13 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp df = pd.read_csv("outputs/adastop_result_file_"+str(k)+".csv", index_col=0) else: if k > 0: - solvers = comparator.agent_names - arg_solver = " -s "+" -s ".join(solvers) - print("Doing comparisons for "+str(len(solvers))+ "solvers: "+", ".join(solvers)) + undecided_solvers = [] + for i in range(len(comparator.agent_names)): + if i in comparator.current_comparisons.ravel(): + undecided_solvers.append(comparator.agent_names[i]) + + arg_solver = " -s "+" -s ".join(undecided_solvers) + print("Doing comparisons for "+str(len(undecided_solvers))+ " solvers: "+", ".join(undecided_solvers)) subprocess.check_output(["benchopt", "run", ".", "--config", config_file, "--env", "-r", str(size_group), "--output", "adastop_result_file_"+str(k)]) From ae3634632276d3f13a40994868e94be81db88035 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:31:07 +0100 Subject: [PATCH 07/14] change for python api --- adastop/benchopt.py | 34 ++++++++++++++++++++++++++++++++++ adastop/cli.py | 18 +++++++++++++----- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 adastop/benchopt.py diff --git a/adastop/benchopt.py b/adastop/benchopt.py new file mode 100644 index 0000000..d50b3fa --- /dev/null +++ b/adastop/benchopt.py @@ -0,0 +1,34 @@ +from benchopt import run_benchmark +from benchopt.benchmark import Benchmark +import pandas as pd + + +def run_benchopt(solver, dataset, n_repetitions, output_name, forced_solvers, timeout=100, max_runs=10): + # load benchmark + BENCHMARK_PATH = "./" + benchmark = Benchmark(BENCHMARK_PATH) + + # run benchmark + run_benchmark( + benchmark, + solver_names=solver, + dataset_names=dataset, + n_repetitions=n_repetitions, + timeout=timeout, + max_runs=max_runs, + output_name=output_name, + forced_solvers = forced_solvers + ) + +def process_benchopt(file): + """ + For now, suppose that there is only one dataset + """ + df = pd.read_parquet(file) + df= df[["solver_name",'objective_value','idx_rep']] + df_ret = { name : [] for name in df["solver_name"].unique()} + for rep in df["idx_rep"].unique(): + for solver in df["solver_name"].unique(): + df_rep_solver = df.loc[ (df["solver_name"]==solver) & (df["idx_rep"]==rep)] + df_ret[solver].append(df_rep_solver['objective_value'].iloc[-1]) + return pd.DataFrame(df_ret) diff --git a/adastop/cli.py b/adastop/cli.py index 04f6b61..dcca359 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -1,12 +1,13 @@ import click import pickle +import yaml import os from pathlib import Path import subprocess import pandas as pd import numpy as np import matplotlib.pyplot as plt -from .data_processing import process_benchopt +from .benchopt import process_benchopt, run_benchopt from .compare_agents import MultipleAgentsComparator @@ -137,12 +138,19 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp if i in comparator.current_comparisons.ravel(): undecided_solvers.append(comparator.agent_names[i]) - arg_solver = " -s "+" -s ".join(undecided_solvers) + with open(config_file, 'r') as file: + config = yaml.safe_load(file) + + config['solver']=undecided_solvers + config['n_repetitions']=size_group + config['output_name'] = "adastop_result_file_"+str(k) + config['forced_solvers'] = undecided_solvers + print("Doing comparisons for "+str(len(undecided_solvers))+ " solvers: "+", ".join(undecided_solvers)) - subprocess.check_output(["benchopt", "run", ".", "--config", - config_file, "--env", "-r", str(size_group), - "--output", "adastop_result_file_"+str(k)]) + run_benchopt(**config) + else: + # initially, run everything subprocess.check_output(["benchopt", "run", ".", "--config", config_file, "--env", "-r", str(size_group), "--output", "adastop_result_file_"+str(k)]) From a3664ab6d37c6b57156e53caed342828498d2501 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:35:41 +0100 Subject: [PATCH 08/14] objective --- adastop/benchopt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adastop/benchopt.py b/adastop/benchopt.py index d50b3fa..1559e84 100644 --- a/adastop/benchopt.py +++ b/adastop/benchopt.py @@ -3,7 +3,7 @@ import pandas as pd -def run_benchopt(solver, dataset, n_repetitions, output_name, forced_solvers, timeout=100, max_runs=10): +def run_benchopt(solver, dataset, n_repetitions, output_name, forced_solvers, objective_filter, timeout=100, max_runs=10): # load benchmark BENCHMARK_PATH = "./" benchmark = Benchmark(BENCHMARK_PATH) @@ -17,7 +17,8 @@ def run_benchopt(solver, dataset, n_repetitions, output_name, forced_solvers, ti timeout=timeout, max_runs=max_runs, output_name=output_name, - forced_solvers = forced_solvers + forced_solvers = forced_solvers, + objective_filters=objective_filter ) def process_benchopt(file): From 8555301948fb7e9168885f34206a57d917f73b9d Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:49:48 +0100 Subject: [PATCH 09/14] roll back to cli --- adastop/benchopt.py | 20 -------------------- adastop/cli.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/adastop/benchopt.py b/adastop/benchopt.py index 1559e84..28bed52 100644 --- a/adastop/benchopt.py +++ b/adastop/benchopt.py @@ -1,26 +1,6 @@ -from benchopt import run_benchmark -from benchopt.benchmark import Benchmark import pandas as pd -def run_benchopt(solver, dataset, n_repetitions, output_name, forced_solvers, objective_filter, timeout=100, max_runs=10): - # load benchmark - BENCHMARK_PATH = "./" - benchmark = Benchmark(BENCHMARK_PATH) - - # run benchmark - run_benchmark( - benchmark, - solver_names=solver, - dataset_names=dataset, - n_repetitions=n_repetitions, - timeout=timeout, - max_runs=max_runs, - output_name=output_name, - forced_solvers = forced_solvers, - objective_filters=objective_filter - ) - def process_benchopt(file): """ For now, suppose that there is only one dataset diff --git a/adastop/cli.py b/adastop/cli.py index dcca359..b7b757f 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -7,7 +7,7 @@ import pandas as pd import numpy as np import matplotlib.pyplot as plt -from .benchopt import process_benchopt, run_benchopt +from .benchopt import process_benchopt from .compare_agents import MultipleAgentsComparator @@ -140,14 +140,19 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp with open(config_file, 'r') as file: config = yaml.safe_load(file) + config['solver']=undecided_solvers - config['n_repetitions']=size_group - config['output_name'] = "adastop_result_file_"+str(k) config['forced_solvers'] = undecided_solvers + + with open("/tmp/config_benchopt.yml", 'w') as file: + config = yaml.dump(config, file) + print("Doing comparisons for "+str(len(undecided_solvers))+ " solvers: "+", ".join(undecided_solvers)) - run_benchopt(**config) + subprocess.check_output(["benchopt", "run", ".", "--config /tmp/config_benchopt.yml", + "--env", "-r", str(size_group), + "--output", "adastop_result_file_"+str(k)]) else: # initially, run everything From dafa15358e74639721fc0ce9055328f88ae99a42 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:50:58 +0100 Subject: [PATCH 10/14] typo --- adastop/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adastop/cli.py b/adastop/cli.py index b7b757f..18c3b3d 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -150,7 +150,7 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp print("Doing comparisons for "+str(len(undecided_solvers))+ " solvers: "+", ".join(undecided_solvers)) - subprocess.check_output(["benchopt", "run", ".", "--config /tmp/config_benchopt.yml", + subprocess.check_output(["benchopt", "run", ".", "--config", "/tmp/config_benchopt.yml", "--env", "-r", str(size_group), "--output", "adastop_result_file_"+str(k)]) From 825504f21f22fea937622f32c1e10046400ebc25 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:53:35 +0100 Subject: [PATCH 11/14] typo --- adastop/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adastop/cli.py b/adastop/cli.py index 18c3b3d..2368a08 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -143,7 +143,7 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp config['solver']=undecided_solvers - config['forced_solvers'] = undecided_solvers + config['forced-solvers'] = undecided_solvers with open("/tmp/config_benchopt.yml", 'w') as file: config = yaml.dump(config, file) From f95f8cb3e05f507acdc88b0ff2988d3732dfa93a Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Thu, 28 Dec 2023 09:54:21 +0100 Subject: [PATCH 12/14] typo --- adastop/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/adastop/cli.py b/adastop/cli.py index 2368a08..654b597 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -143,7 +143,6 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp config['solver']=undecided_solvers - config['forced-solvers'] = undecided_solvers with open("/tmp/config_benchopt.yml", 'w') as file: config = yaml.dump(config, file) From 59790e29aa83ccdda6849b5b19088abd4cdfb0c5 Mon Sep 17 00:00:00 2001 From: Timothee Mathieu Date: Sat, 30 Dec 2023 13:46:44 +0100 Subject: [PATCH 13/14] fix compare_benchopt --- adastop/cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/adastop/cli.py b/adastop/cli.py index 654b597..48ef6f8 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -31,7 +31,12 @@ def compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compa if os.path.isfile(path_lf): with open(path_lf, 'rb') as fp: comparator = pickle.load(fp) - names = comparator.agent_names + + names = [] + for i in range(len(comparator.agent_names)): + if i in comparator.current_comparisons.ravel(): + names.append(comparator.agent_names[i]) + Z = [np.hstack([comparator.eval_values[agent], df[agent]]) for agent in names] if len(Z[0]) > comparator.K * n_fits_per_group: From 2828e170dcafa461b322d09cad8f3681296b81da Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Sun, 14 Jan 2024 10:45:20 +0100 Subject: [PATCH 14/14] working proof of concept --- adastop/benchopt.py | 2 +- adastop/cli.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/adastop/benchopt.py b/adastop/benchopt.py index 28bed52..204b398 100644 --- a/adastop/benchopt.py +++ b/adastop/benchopt.py @@ -11,5 +11,5 @@ def process_benchopt(file): for rep in df["idx_rep"].unique(): for solver in df["solver_name"].unique(): df_rep_solver = df.loc[ (df["solver_name"]==solver) & (df["idx_rep"]==rep)] - df_ret[solver].append(df_rep_solver['objective_value'].iloc[-1]) + df_ret[solver].append(df_rep_solver['objective_test_loss'].iloc[-1]) return pd.DataFrame(df_ret) diff --git a/adastop/cli.py b/adastop/cli.py index 48ef6f8..3b53b2a 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -55,7 +55,7 @@ def compare_data(path_lf, df, n_groups, n_permutations, alpha, beta, seed, compa # recover also the data of agent that were decided. if comparator.agent_names is not None: for agent in comparator.agent_names: - if agent not in df.columns: + if agent not in data.keys(): data[agent]=comparator.eval_values[agent] comparator.partial_compare(data, False) @@ -122,6 +122,8 @@ def compare_benchopt(ctx, config_file, size_group, n_groups, n_permutations, alp """ Perform one step of computing benchmark and then adaptive stopping algorithm. The benchmark is supposed to be in the current directory. + + WARNING: still experimental. """ path_lf = Path(config_file).parent.absolute() / LITTER_FILE