diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark/base.py b/browsergym/experiments/src/browsergym/experiments/benchmark/base.py index f51f7803..8efd6af3 100644 --- a/browsergym/experiments/src/browsergym/experiments/benchmark/base.py +++ b/browsergym/experiments/src/browsergym/experiments/benchmark/base.py @@ -1,4 +1,5 @@ import fnmatch +import importlib.metadata import logging import random import typing @@ -64,6 +65,31 @@ def make_action_set(self): "knows", ] +_BACKEND_TO_DISTRIBUTIONS: dict[BenchmarkBackend, tuple[str, ...]] = { + "miniwob": ("browsergym.miniwob", "browsergym-miniwob"), + "workarena": ("browsergym.workarena", "browsergym-workarena"), + "webarena": ("browsergym.webarena", "browsergym-webarena"), + "webarena_verified": ("browsergym.webarena_verified", "browsergym-webarena-verified"), + "visualwebarena": ("browsergym.visualwebarena", "browsergym-visualwebarena"), + "assistantbench": ("browsergym.assistantbench", "browsergym-assistantbench"), + "weblinx": ("weblinx_browsergym", "weblinx-browsergym"), + "timewarp": ("browsergym.timewarp", "browsergym-timewarp"), + "knows": ("browsergym.knows", "browsergym-knows"), +} + +_PREFIX_TO_DISTRIBUTIONS: list[tuple[str, tuple[str, ...]]] = [ + ("webarena_verified", ("browsergym.webarena_verified", "browsergym-webarena-verified")), + ("visualwebarena", ("browsergym.visualwebarena", "browsergym-visualwebarena")), + ("webarena", ("browsergym.webarena", "browsergym-webarena")), + ("webarenalite", ("browsergym.webarena", "browsergym-webarena")), + ("miniwob", ("browsergym.miniwob", "browsergym-miniwob")), + ("workarena", ("browsergym.workarena", "browsergym-workarena")), + ("assistantbench", ("browsergym.assistantbench", "browsergym-assistantbench")), + ("weblinx", ("weblinx_browsergym", "weblinx-browsergym")), + ("timewarp", ("browsergym.timewarp", "browsergym-timewarp")), + ("knows", ("browsergym.knows", "browsergym-knows")), +] + @dataclass class Benchmark(DataClassJsonMixin): @@ -104,6 +130,43 @@ def prepare_backends(self): prepare_backend(backend) logger.info(f"{backend} backend ready") + def get_version(self) -> str: + """ + Return the version of the benchmark based on its backend or name. + """ + dist_names = None + is_weblinx = False + + for backend in self.backends: + if backend in _BACKEND_TO_DISTRIBUTIONS: + dist_names = _BACKEND_TO_DISTRIBUTIONS[backend] + if backend == "weblinx": + is_weblinx = True + break + + if dist_names is None: + name_lower = self.name.lower() + for prefix, dists in _PREFIX_TO_DISTRIBUTIONS: + if name_lower.startswith(prefix): + dist_names = dists + if prefix == "weblinx": + is_weblinx = True + break + + if dist_names is None: + raise ValueError(f"Unknown benchmark {self.name}") + + for dist_name in dist_names: + try: + return importlib.metadata.distribution(dist_name).version + except importlib.metadata.PackageNotFoundError: + continue + + if is_weblinx: + return "0.0.1rc1" + + raise importlib.metadata.PackageNotFoundError(dist_names[0]) + def subset_from_split(self, split: Literal["train", "valid", "test"]): split_column = "browsergym_split" diff --git a/tests/experiments/test_benchmark.py b/tests/experiments/test_benchmark.py index c7f88544..0df1d24b 100644 --- a/tests/experiments/test_benchmark.py +++ b/tests/experiments/test_benchmark.py @@ -1,4 +1,5 @@ import dataclasses +import importlib.metadata import os import random import re @@ -353,3 +354,74 @@ def test_dependency_graphs(): assert [{k: set(v) for k, v in deps.items()} for deps in env_args_dependencies] == [ {k: set(v) for k, v in {0: [], 1: [], 2: [1], 3: [0, 2], 4: [3], 5: [3, 4]}.items()} ] + + +def test_benchmark_get_version(): + from unittest.mock import MagicMock, patch + + mock_dist = MagicMock() + mock_dist.version = "1.2.3" + + # Test standard benchmarks with mocked distributions + with patch("importlib.metadata.distribution", return_value=mock_dist) as mock_metadata: + miniwob_bench = DEFAULT_BENCHMARKS["miniwob"]() + assert miniwob_bench.get_version() == "1.2.3" + mock_metadata.assert_called_with("browsergym.miniwob") + + with patch("importlib.metadata.distribution", return_value=mock_dist) as mock_metadata: + webarena_bench = DEFAULT_BENCHMARKS["webarena"]() + assert webarena_bench.get_version() == "1.2.3" + mock_metadata.assert_called_with("browsergym.webarena") + + workarena_bench = Benchmark( + name="workarena_l1", + high_level_action_set_args=DEFAULT_BENCHMARKS["miniwob_tiny_test"]().high_level_action_set_args, + is_multi_tab=False, + supports_parallel_seeds=True, + env_args_list=DEFAULT_BENCHMARKS["miniwob_tiny_test"]().env_args_list, + backends=["workarena"], + ) + with patch("importlib.metadata.distribution", return_value=mock_dist) as mock_metadata: + assert workarena_bench.get_version() == "1.2.3" + mock_metadata.assert_called_with("browsergym.workarena") + + with patch("importlib.metadata.distribution", return_value=mock_dist) as mock_metadata: + visualwebarena_bench = DEFAULT_BENCHMARKS["visualwebarena"]() + assert visualwebarena_bench.get_version() == "1.2.3" + mock_metadata.assert_called_with("browsergym.visualwebarena") + + with patch("importlib.metadata.distribution", return_value=mock_dist) as mock_metadata: + assistantbench = DEFAULT_BENCHMARKS["assistantbench"]() + assert assistantbench.get_version() == "1.2.3" + mock_metadata.assert_called_with("browsergym.assistantbench") + + # Test weblinx fallback to 0.0.1rc1 when not found + with patch( + "importlib.metadata.distribution", + side_effect=importlib.metadata.PackageNotFoundError("weblinx_browsergym"), + ): + weblinx_bench = DEFAULT_BENCHMARKS["weblinx"]() + assert weblinx_bench.get_version() == "0.0.1rc1" + + # Test weblinx returns version when package is found + with patch("importlib.metadata.distribution", return_value=mock_dist): + weblinx_bench = DEFAULT_BENCHMARKS["weblinx"]() + assert weblinx_bench.get_version() == "1.2.3" + + # Test unknown benchmark raises ValueError + fake_bench = Benchmark( + name="unknown_benchmark", + high_level_action_set_args=DEFAULT_BENCHMARKS["miniwob_tiny_test"]().high_level_action_set_args, + is_multi_tab=False, + supports_parallel_seeds=True, + env_args_list=DEFAULT_BENCHMARKS["miniwob_tiny_test"]().env_args_list, + backends=[], + ) + with pytest.raises(ValueError, match="Unknown benchmark unknown_benchmark"): + fake_bench.get_version() + + # Test benchmark subset preserves get_version + with patch("importlib.metadata.distribution", return_value=mock_dist): + sub_bench = miniwob_bench.subset_from_regexp(column="task_name", regexp="click") + assert sub_bench.get_version() == "1.2.3" +