From ca21f7381e2f762b01f5dc27d5031e3eaec6b184 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 13 Aug 2025 09:56:42 +0200 Subject: [PATCH 1/3] remove beta in code --- adastop/cli.py | 5 ++--- adastop/compare_agents.py | 28 +--------------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/adastop/cli.py b/adastop/cli.py index 1668420..f37c210 100644 --- a/adastop/cli.py +++ b/adastop/cli.py @@ -27,12 +27,11 @@ def adastop(ctx): @click.option("-N", "--size-group", default=5,type=int, show_default=True, help="Number of groups.") @click.option("-B", "--n-permutations", default=10000,type=int, show_default=True, help="Number of random permutations.") @click.option("--alpha", default=0.05,type=float, show_default=True, help="Type I error.") -@click.option("--beta", default=0.0,type=float, show_default=True, help="early accept parameter.") @click.option("--seed", default=None, type=int, show_default=True, help="Random seed.") @click.option("--compare-to-first", is_flag=True, show_default=True, default=False, help="Compare all algorithms to the first algorithm.") @click.argument('input_file',required = True, type=str) @click.pass_context -def compare(ctx, input_file, n_groups, size_group, n_permutations, alpha, beta, seed, compare_to_first): +def compare(ctx, input_file, n_groups, size_group, n_permutations, alpha, seed, compare_to_first): """ Perform one step of adaptive stopping algorithm using csv file intput_file. The csv file must be of size `size_group`. @@ -70,7 +69,7 @@ def compare(ctx, input_file, n_groups, size_group, n_permutations, alpha, beta, else: comparator = MultipleAgentsComparator(n_fits_per_group, n_groups, n_permutations, comparisons, - alpha, beta, seed) + alpha, seed) names = df.columns Z = [df[agent].values for agent in names] diff --git a/adastop/compare_agents.py b/adastop/compare_agents.py index a32d743..f579342 100644 --- a/adastop/compare_agents.py +++ b/adastop/compare_agents.py @@ -42,9 +42,6 @@ class MultipleAgentsComparator: alpha: float, default=0.01 level of the test - beta: float, default=0 - power spent in early accept. - seed: int or None, default = None Attributes @@ -84,19 +81,16 @@ def __init__( B=10000, comparisons = None, alpha=0.01, - beta=0, seed=None, ): self.n = n self.K = K self.B = B self.alpha = alpha - self.beta = beta self.comparisons = comparisons self.boundary = [] self.k = 0 self.level_spent = 0 - self.power_spent = 0 self.seed = seed self.rng = np.random.RandomState(seed) self.rejected_decision = [] @@ -235,7 +229,6 @@ def partial_compare(self, eval_values, verbose=True): k = self.k clevel = self.alpha*(k + 1) / self.K - dlevel = self.beta*(k + 1) / self.K mean_diffs = self.compute_mean_diffs(k, Z) @@ -270,19 +263,6 @@ def partial_compare(self, eval_values, verbose=True): bk_sup = np.inf level_to_add = 0 - cumulative_probas = np.arange(len(values)) / self.normalization # corresponds to P(T < t) - admissible_values_inf = values[ - self.power_spent + cumulative_probas < dlevel - ] - - if len(admissible_values_inf) > 0: - bk_inf = admissible_values_inf[-1] # the maximum admissible value - power_to_add = cumulative_probas[ - self.power_spent + cumulative_probas <= dlevel - ][-1] - else: - bk_inf = -np.inf - power_to_add = 0 # Test statistic, step-down Tmax = 0 @@ -323,19 +303,13 @@ def partial_compare(self, eval_values, verbose=True): self.decisions[str(self.current_comparisons[id_reject])] = "smaller" if verbose: print("reject") - elif Tmin < bk_inf: - id_accept = np.arange(len(current_decisions))[current_decisions == "continue"][imin] - current_decisions[id_accept] = "accept" - self.decisions[str(self.current_comparisons[id_accept])] = "equal" else: break - self.boundary.append((bk_inf, bk_sup)) - + self.boundary.append(bk_sup) self.level_spent += level_to_add # level effectively used at this point - self.power_spent += power_to_add if k == self.K - 1: for c in self.comparisons: From f0a6caca3e62286451339115b53763aea8ca5d05 Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 13 Aug 2025 10:05:33 +0200 Subject: [PATCH 2/3] remove more betas --- adastop/compare_agents.py | 2 +- tests/test_compare_agents.py | 27 ++++----------------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/adastop/compare_agents.py b/adastop/compare_agents.py index 9e2c6e4..bd1b9c7 100644 --- a/adastop/compare_agents.py +++ b/adastop/compare_agents.py @@ -139,7 +139,7 @@ def compute_mean_diffs(self, k, Z): zval = [] for comp in self.mean_diffs: zval.append(self.mean_diffs[str(comp)][i]) - if np.max(np.abs(zval)) <= boundary[-1][1]: + if np.max(np.abs(zval)) <= boundary[-1]: for comp in self.mean_diffs: mean_diffs[str(comp)].append(self.mean_diffs[str(comp)][i]) diff --git a/tests/test_compare_agents.py b/tests/test_compare_agents.py index 2998d89..29fe0fd 100644 --- a/tests/test_compare_agents.py +++ b/tests/test_compare_agents.py @@ -10,13 +10,13 @@ def test_partial_compare(): rng = np.random.RandomState(seed) idxs = [] - comparator = MultipleAgentsComparator(n=3, K=3, B=B, alpha=alpha, seed=42, beta = 0.01) + comparator = MultipleAgentsComparator(n=3, K=3, B=B, alpha=alpha, seed=42) evals = {"Agent "+str(k): rng.normal(size=3) for k in range(3)} comparator.partial_compare(evals) def test_partial_compare_not_enough_points(): - comparator = MultipleAgentsComparator(n=3, K=3, B=5000, alpha=-1e-5, seed=42, beta = 0.01) + comparator = MultipleAgentsComparator(n=3, K=3, B=5000, alpha=-1e-5, seed=42) evals = {"Agent 1":np.array([0,0,0]),"Agent 2":np.array([0,0,0]),"Agent 3":np.array([0,0,0])} comparator.partial_compare(evals) @@ -29,7 +29,7 @@ def test_type1(K,n): idxs = [] n_agents = 3 for M in range(n_runs): - comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=M, beta = 0.01) + comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=M) evals = {} while not comparator.is_finished: if len(evals) >0: @@ -42,26 +42,7 @@ def test_type1(K,n): print(comparator.get_results()) assert np.mean(idxs) < 2*alpha + 1/4/(np.sqrt(n_runs)), "type 1 error seems to be too large." -@pytest.mark.parametrize("K,n", [(5,3), (3, 5), (1, 15)]) -def test_type1_large_beta(K,n): - rng = np.random.RandomState(seed) - idxs = [] - n_agents = 3 - for M in range(n_runs): - comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=M, beta = 0.1) - evals = {} - while not comparator.is_finished: - if len(evals) >0: - for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , rng.normal(size=n)]) - else: - evals = {"Agent "+str(k): rng.normal(size=n) for k in range(n_agents)} - comparator.partial_compare(evals) - idxs.append(not("equal" in comparator.decisions.values())) - print(comparator.get_results()) - assert np.mean(idxs) < 2*alpha + 1/4/(np.sqrt(n_runs)), "type 1 error seems to be too large." - @pytest.mark.parametrize("K,n", [(3, 5), (1, 15)]) def test_type2(K,n): rng = np.random.RandomState(seed) @@ -69,7 +50,7 @@ def test_type2(K,n): idxs = [] n_agents = 2 for M in range(n_runs): - comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=M, beta = 0.01) + comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=M) evals = {} while not comparator.is_finished: if len(evals) >0: From 78be80609a7738d6627fa96dcd57eedadc432dca Mon Sep 17 00:00:00 2001 From: TimotheeMathieu Date: Wed, 13 Aug 2025 10:29:25 +0200 Subject: [PATCH 3/3] beta in test --- tests/test_plot.py | 48 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/test_plot.py b/tests/test_plot.py index e36303d..fb6bf23 100644 --- a/tests/test_plot.py +++ b/tests/test_plot.py @@ -7,17 +7,19 @@ n_runs = 10 K = 5 n = 4 +seed = 42 def test_plot(): n_agents = 3 - comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=42, beta = 0.01) + comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, seed=42) evals = {} + rng = np.random.RandomState(seed) while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,np.random.normal(size=n)]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,rng.normal(size=n)]) else: - evals = {"Agent "+str(k): np.random.normal(size=n) for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=n) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results() @@ -28,16 +30,18 @@ def test_plot(): def test_plot_sota(): n_agents = 3 + rng = np.random.RandomState(seed) + comparisons = np.array([(0,i) for i in [1,2]]) comparator = MultipleAgentsComparator(n=n, K=K, B=B, alpha=alpha, - comparisons=comparisons, seed=42, beta = 0.01) + comparisons=comparisons, seed=42) evals = {} while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,np.random.normal(size=n)]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,rng.normal(size=n)]) else: - evals = {"Agent "+str(k): np.random.normal(size=n) for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=n) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() # plt.savefig('fig2.pdf') @@ -46,14 +50,16 @@ def test_plot_sota(): def test_plot_noteq(): n_agents = 3 - comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, seed=42, beta = 0.01) + comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, seed=42) + rng = np.random.RandomState(seed) + evals = {} while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , k+np.random.normal(size=10)]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , k+ rng.normal(size=10)]) else: - evals = {"Agent "+str(k): np.random.normal(size=10)+k for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=10)+k for k in range(n_agents)} comparator.partial_compare(evals) # plt.savefig('fig2.pdf') fig, axes= plt.subplots(1,2) @@ -61,16 +67,18 @@ def test_plot_noteq(): def test_plot_sota_noteq(): n_agents = 3 + rng = np.random.RandomState(seed) + comparisons = np.array([(0,i) for i in [1,2]]) comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, - comparisons=comparisons, seed=42, beta = 0.01) + comparisons=comparisons, seed=42) evals = {} while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,np.random.normal(size=10)+k]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , rng.normal(size=10)+k]) else: - evals = {"Agent "+str(k): np.random.normal(size=10) for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=10) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() # plt.savefig('fig2.pdf') @@ -81,14 +89,16 @@ def test_plot_sota_noteq(): def test_plot_noteq2(): n_agents = 3 - comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, seed=42, beta = 0.01) + comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, seed=42) + rng = np.random.RandomState(seed) + evals = {} while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , np.abs(2*K-k)+np.random.normal(size=10)]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , np.abs(2*K-k)+ rng.normal(size=10)]) else: - evals = {"Agent "+str(k): np.random.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} comparator.partial_compare(evals) # plt.savefig('fig2.pdf') fig, axes= plt.subplots(1,2) @@ -98,14 +108,16 @@ def test_plot_sota_noteq2(): n_agents = 3 comparisons = np.array([(0,i) for i in [1,2]]) comparator = MultipleAgentsComparator(n=10, K=K, B=B, alpha=alpha, - comparisons=comparisons, seed=42, beta = 0.01) + comparisons=comparisons, seed=42) + rng = np.random.RandomState(seed) + evals = {} while not comparator.is_finished: if len(evals) >0: for k in range(n_agents): - evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] ,np.random.normal(size=10)+np.abs(2*K-k)]) + evals["Agent "+str(k)] = np.hstack([evals["Agent "+str(k)] , rng.normal(size=10)+np.abs(2*K-k)]) else: - evals = {"Agent "+str(k): np.random.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} + evals = {"Agent "+str(k): rng.normal(size=10)+np.abs(2*K-k) for k in range(n_agents)} comparator.partial_compare(evals) comparator.plot_results_sota() # plt.savefig('fig2.pdf')