From f880b18f39e5073b0891fae1c46dec69b4c5be6c Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Wed, 7 Jan 2026 14:19:57 +0800 Subject: [PATCH 1/6] fix --- .gitignore | 6 ++++++ bigcodebench/provider/openai.py | 30 ++++++++++++++++++++++++++---- bigcodebench/provider/utility.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index abe84432..52714671 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ parts/ sdist/ var/ wheels/ +temp/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -170,3 +171,8 @@ bigcodebench/_version.py *.jsonl inspect/ *.zip + + +# custome files +execution_coverage_analysis.md +function_call_layers.md \ No newline at end of file diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index ff1459f9..d9ddbc24 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -6,7 +6,7 @@ from bigcodebench.gen.util.openai_request import make_auto_request from bigcodebench.provider.utility import make_raw_chat_prompt from bigcodebench.provider.base import DecoderBase -from bigcodebench.provider.utility import concurrent_call +from bigcodebench.provider.utility import concurrent_call, concurrent_map class OpenAIChatDecoder(DecoderBase): def __init__(self, name: str, base_url=None, reasoning_effort="medium", **kwargs) -> None: @@ -38,8 +38,8 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] api_key=os.getenv("OPENAI_API_KEY", "none"), base_url=self.base_url ) - all_outputs = [] - for message in tqdm(messages): + # Helper function to process a single message with its index + def process_message(index, message): ret = make_auto_request( client, message=message, @@ -52,8 +52,30 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] outputs = [] for item in ret.choices: outputs.append(item.message.content) - all_outputs.append(outputs) + return outputs + + # Process messages in parallel using utility function + all_outputs = concurrent_map(messages, process_message) + return all_outputs + + # Original sequential implementation (commented out) + # all_outputs = [] + # for message in tqdm(messages): + # ret = make_auto_request( + # client, + # message=message, + # model=self.name, + # max_tokens=self.max_new_tokens, + # temperature=self.temperature, + # reasoning_effort=self.reasoning_effort, + # n=num_samples, + # ) + # outputs = [] + # for item in ret.choices: + # outputs.append(item.message.content) + # all_outputs.append(outputs) + # return all_outputs def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int) -> List[str]: batches = concurrent_call( diff --git a/bigcodebench/provider/utility.py b/bigcodebench/provider/utility.py index ad091591..cc4b2cea 100644 --- a/bigcodebench/provider/utility.py +++ b/bigcodebench/provider/utility.py @@ -1,6 +1,7 @@ from typing import List from transformers import AutoTokenizer -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, as_completed +from tqdm import tqdm EOS = [ "<|endoftext|>", @@ -80,4 +81,31 @@ def make_raw_chat_prompt( def concurrent_call(n, callback, /, *args, **kwargs): with ThreadPoolExecutor(max_workers=n) as executor: futures = [executor.submit(callback, *args, **kwargs) for _ in range(n)] - return [future.result() for future in futures] \ No newline at end of file + return [future.result() for future in futures] + + +# Parallel processing utility for processing multiple different items concurrently. +# Unlike concurrent_call which runs the same operation n times, concurrent_map +# processes n different items in parallel (similar to map() but concurrent). +# Automatically tracks indices to preserve output order and shows progress with tqdm. +def concurrent_map(items, callback, /, *args, **kwargs): + """ + Process a list of items in parallel using a callback function. + + Args: + items: List of items to process + callback: Function that takes (index, item, *args, **kwargs) and returns a result + *args, **kwargs: Additional arguments passed to callback + + Returns: + List of results in the same order as input items + """ + with ThreadPoolExecutor() as executor: + futures = {executor.submit(callback, i, item, *args, **kwargs): i + for i, item in enumerate(items)} + results = [None] * len(items) + # Collect results as they complete with progress bar + for future in tqdm(as_completed(futures), total=len(items)): + index = futures[future] + results[index] = future.result() + return results \ No newline at end of file From 06b1c9726133009d7dce1238d246f0b1c316b5de Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Wed, 7 Jan 2026 14:39:28 +0800 Subject: [PATCH 2/6] roll back gitignore change --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index 52714671..abeb7677 100644 --- a/.gitignore +++ b/.gitignore @@ -171,8 +171,3 @@ bigcodebench/_version.py *.jsonl inspect/ *.zip - - -# custome files -execution_coverage_analysis.md -function_call_layers.md \ No newline at end of file From 344870b2e3db57c6d072f55cfcd724ed2c6248b1 Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Wed, 7 Jan 2026 14:40:00 +0800 Subject: [PATCH 3/6] roll back gitignore change --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index abeb7677..abe84432 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ parts/ sdist/ var/ wheels/ -temp/ share/python-wheels/ *.egg-info/ .installed.cfg From da5befeed689bad48bc4b3531c7ace2f0adbfd7f Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Wed, 7 Jan 2026 14:41:26 +0800 Subject: [PATCH 4/6] remove comment out origin code --- bigcodebench/provider/openai.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index d9ddbc24..caf52a62 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -58,24 +58,6 @@ def process_message(index, message): all_outputs = concurrent_map(messages, process_message) return all_outputs - - # Original sequential implementation (commented out) - # all_outputs = [] - # for message in tqdm(messages): - # ret = make_auto_request( - # client, - # message=message, - # model=self.name, - # max_tokens=self.max_new_tokens, - # temperature=self.temperature, - # reasoning_effort=self.reasoning_effort, - # n=num_samples, - # ) - # outputs = [] - # for item in ret.choices: - # outputs.append(item.message.content) - # all_outputs.append(outputs) - # return all_outputs def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int) -> List[str]: batches = concurrent_call( From c4672820d6b555008078d579db1f1b4401458468 Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Wed, 7 Jan 2026 17:42:21 +0800 Subject: [PATCH 5/6] remove unused param --- bigcodebench/provider/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index caf52a62..36365754 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -39,7 +39,7 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] ) # Helper function to process a single message with its index - def process_message(index, message): + def process_message(message): ret = make_auto_request( client, message=message, From 02482ffba926b881f8ff3ea1351ffd9891fafbd6 Mon Sep 17 00:00:00 2001 From: oldjeffspectator Date: Thu, 8 Jan 2026 09:54:02 +0800 Subject: [PATCH 6/6] fix bug --- bigcodebench/provider/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py index 36365754..caf52a62 100644 --- a/bigcodebench/provider/openai.py +++ b/bigcodebench/provider/openai.py @@ -39,7 +39,7 @@ def _codegen_api_batch(self, messages: List[str], num_samples: int) -> List[str] ) # Helper function to process a single message with its index - def process_message(message): + def process_message(index, message): ret = make_auto_request( client, message=message,