From d09853af1c6b0e3440350817b5db3b05f11be0cf Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Fri, 24 Apr 2026 18:55:46 +0530 Subject: [PATCH] Strip trailing slash from WebArena URLs (#366) Trailing / in WA_SHOPPING etc. caused reference URLs like http://host:8082//catalogsearch/... which never matched page.url, so shopping tasks (e.g. 274, 275) scored 0 even when completed. Normalize URLs in WebArenaInstance + VisualWebArenaInstance and in webarenalite/helper_functions. Also fix the README example. --- .../src/browsergym/visualwebarena/instance.py | 2 +- browsergym/webarena/README.md | 2 +- .../src/browsergym/webarena/instance.py | 3 +- .../webarena/src/browsergym/webarena/task.py | 27 ++++-- .../src/browsergym/webarena_verified/task.py | 13 +-- .../src/browsergym/webarenalite/task.py | 13 +-- tests/webarena/test_url_substitution.py | 91 +++++++++++++++++++ 7 files changed, 116 insertions(+), 35 deletions(-) create mode 100644 tests/webarena/test_url_substitution.py diff --git a/browsergym/visualwebarena/src/browsergym/visualwebarena/instance.py b/browsergym/visualwebarena/src/browsergym/visualwebarena/instance.py index 63936e8a9..fabe43e9e 100644 --- a/browsergym/visualwebarena/src/browsergym/visualwebarena/instance.py +++ b/browsergym/visualwebarena/src/browsergym/visualwebarena/instance.py @@ -33,7 +33,7 @@ def __init__( + "Please set the following environment variables to use VisualWebArena through BrowserGym:\n" + "\n".join([append_vwa(x) for x in ENV_VARS]) ) - os.environ[key] = os.environ[append_vwa(key)] + os.environ[key] = os.environ[append_vwa(key)].rstrip("/") # import webarena on instantiation from visualwebarena.browser_env.env_config import ( diff --git a/browsergym/webarena/README.md b/browsergym/webarena/README.md index cb1416dc7..44d37b099 100644 --- a/browsergym/webarena/README.md +++ b/browsergym/webarena/README.md @@ -29,7 +29,7 @@ python -c "import nltk; nltk.download('punkt_tab')" BASE_URL= # example: "http://myazuremachine.eastus.cloudapp.azure.com" # webarena environment variables (change ports as needed) -export WA_SHOPPING="$BASE_URL:8082/" +export WA_SHOPPING="$BASE_URL:8082" export WA_SHOPPING_ADMIN="$BASE_URL:8083/admin" export WA_REDDIT="$BASE_URL:8080" export WA_GITLAB="$BASE_URL:9001" diff --git a/browsergym/webarena/src/browsergym/webarena/instance.py b/browsergym/webarena/src/browsergym/webarena/instance.py index 8e4c9c8c9..4f1f047c6 100644 --- a/browsergym/webarena/src/browsergym/webarena/instance.py +++ b/browsergym/webarena/src/browsergym/webarena/instance.py @@ -30,7 +30,8 @@ def __init__( + "Please set the following environment variables to use WebArena through BrowserGym:\n" + "\n".join([append_wa(x) for x in ENV_VARS]) ) - os.environ[key] = os.environ[append_wa(key)] + # no trailing slash, or "__SHOPPING__/path" becomes "host//path" (#366) + os.environ[key] = os.environ[append_wa(key)].rstrip("/") # import webarena on instanciation from webarena.browser_env.env_config import ( diff --git a/browsergym/webarena/src/browsergym/webarena/task.py b/browsergym/webarena/src/browsergym/webarena/task.py index b8275820b..37d76cf38 100644 --- a/browsergym/webarena/src/browsergym/webarena/task.py +++ b/browsergym/webarena/src/browsergym/webarena/task.py @@ -17,6 +17,22 @@ logger = logging.getLogger(__name__) +URL_PATTERNS = { + "__GITLAB__": "gitlab", + "__REDDIT__": "reddit", + "__SHOPPING__": "shopping", + "__SHOPPING_ADMIN__": "shopping_admin", + "__WIKIPEDIA__": "wikipedia", + "__MAP__": "map", +} + + +def substitute_urls(configs_str: str, urls: dict) -> str: + for pattern, url_key in URL_PATTERNS.items(): + configs_str = configs_str.replace(pattern, urls[url_key]) + return configs_str + + class GenericWebArenaTask(AbstractBrowserTask): """ Base class for all WebArena tasks. @@ -54,16 +70,7 @@ def __init__( all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text() - # substitute URLs - for pattern, url_key in { - "__GITLAB__": "gitlab", - "__REDDIT__": "reddit", - "__SHOPPING__": "shopping", - "__SHOPPING_ADMIN__": "shopping_admin", - "__WIKIPEDIA__": "wikipedia", - "__MAP__": "map", - }.items(): - all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key]) + all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls) # load all task configs to JSON all_configs = json.loads(all_configs_str) diff --git a/browsergym/webarena_verified/src/browsergym/webarena_verified/task.py b/browsergym/webarena_verified/src/browsergym/webarena_verified/task.py index f887b5b78..c612a1175 100644 --- a/browsergym/webarena_verified/src/browsergym/webarena_verified/task.py +++ b/browsergym/webarena_verified/src/browsergym/webarena_verified/task.py @@ -10,7 +10,7 @@ import playwright._impl._errors as playwright_errors import playwright.sync_api -from browsergym.webarena.task import GenericWebArenaTask +from browsergym.webarena.task import GenericWebArenaTask, substitute_urls from browsergym.webarena_verified.evaluators import WebArenaVerifiedEvaluator from webarena_verified.types import FinalAgentResponse @@ -51,16 +51,7 @@ def __init__( .read_text() ) - # substitute URLs - for pattern, url_key in { - "__GITLAB__": "gitlab", - "__REDDIT__": "reddit", - "__SHOPPING__": "shopping", - "__SHOPPING_ADMIN__": "shopping_admin", - "__WIKIPEDIA__": "wikipedia", - "__MAP__": "map", - }.items(): - all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key]) + all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls) # load all task configs to JSON all_configs = json.loads(all_configs_str) diff --git a/browsergym/webarenalite/src/browsergym/webarenalite/task.py b/browsergym/webarenalite/src/browsergym/webarenalite/task.py index a62a27a78..c05ed204f 100644 --- a/browsergym/webarenalite/src/browsergym/webarenalite/task.py +++ b/browsergym/webarenalite/src/browsergym/webarenalite/task.py @@ -8,7 +8,7 @@ import playwright.sync_api -from browsergym.webarena.task import GenericWebArenaTask +from browsergym.webarena.task import GenericWebArenaTask, substitute_urls logger = logging.getLogger(__name__) @@ -40,16 +40,7 @@ def __init__( .joinpath("test_webarena_lite.raw.json") .read_text() ) - # substitute URLs - for pattern, url_key in { - "__GITLAB__": "gitlab", - "__REDDIT__": "reddit", - "__SHOPPING__": "shopping", - "__SHOPPING_ADMIN__": "shopping_admin", - "__WIKIPEDIA__": "wikipedia", - "__MAP__": "map", - }.items(): - all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key]) + all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls) # load all task configs to JSON all_configs = json.loads(all_configs_str) diff --git a/tests/webarena/test_url_substitution.py b/tests/webarena/test_url_substitution.py new file mode 100644 index 000000000..f4a78d7c8 --- /dev/null +++ b/tests/webarena/test_url_substitution.py @@ -0,0 +1,91 @@ +import json +import os +import sys +import types + +import pytest + + +ENV_URLS = { + "WA_SHOPPING": "http://host:8082/", + "WA_SHOPPING_ADMIN": "http://host:8083/admin/", + "WA_REDDIT": "http://host:8080/", + "WA_GITLAB": "http://host:9001/", + "WA_WIKIPEDIA": "http://host:8081/wikipedia/", + "WA_MAP": "http://host:443/", + "WA_HOMEPAGE": "http://host:80/", +} + + +@pytest.fixture +def fake_webarena(monkeypatch): + for k, v in ENV_URLS.items(): + monkeypatch.setenv(k, v) + # WebArenaInstance sets SHOPPING etc., let monkeypatch undo that + monkeypatch.setenv(k.removeprefix("WA_"), "") + + pkg = types.ModuleType("webarena") + browser_env = types.ModuleType("webarena.browser_env") + env_config = types.ModuleType("webarena.browser_env.env_config") + + # read SHOPPING etc. from the env at import time, like the real module + def env_config_getattr(name): + if name in os.environ: + return os.environ[name] + raise AttributeError(name) + + env_config.__getattr__ = env_config_getattr + env_config.ACCOUNTS = {} + + monkeypatch.setitem(sys.modules, "webarena", pkg) + monkeypatch.setitem(sys.modules, "webarena.browser_env", browser_env) + monkeypatch.setitem(sys.modules, "webarena.browser_env.env_config", env_config) + + +@pytest.mark.parametrize( + "site", ["reddit", "gitlab", "shopping", "shopping_admin", "wikipedia", "map"] +) +def test_urls_stripped(fake_webarena, site): + from browsergym.webarena.instance import WebArenaInstance + + assert not WebArenaInstance().urls[site].endswith("/") + + +def test_env_vars_stripped(fake_webarena): + # webarena's evaluator helpers build f"{SHOPPING}/path" from these + from browsergym.webarena.instance import WebArenaInstance + + WebArenaInstance() + for k in ENV_URLS: + assert not os.environ[k.removeprefix("WA_")].endswith("/") + + +def test_home_url_stripped(fake_webarena): + from browsergym.webarena.instance import WebArenaInstance + + assert not WebArenaInstance().home_url.endswith("/") + + +@pytest.mark.parametrize( + "placeholder,path", + [ + ("__SHOPPING__", "/catalogsearch/result/?q=xbox"), + ("__SHOPPING_ADMIN__", "/sales/order/view/order_id/1/"), + ("__REDDIT__", "/f/AskReddit"), + ("__GITLAB__", "/root/some-project"), + ("__WIKIPEDIA__", "/wikipedia_en_all_maxi_2022-05/A/Alpha"), + ("__MAP__", "/?q=Pittsburgh"), + ], +) +def test_no_double_slash(fake_webarena, placeholder, path): + from browsergym.webarena.instance import WebArenaInstance + from browsergym.webarena.task import substitute_urls + + raw = json.dumps({"eval": {"reference_url": f"{placeholder}{path}"}}) + out = json.loads(substitute_urls(raw, WebArenaInstance().urls)) + ref = out["eval"]["reference_url"] + + first = path.lstrip("/").split("/")[0].split("?")[0] + if first: + assert f"//{first}" not in ref + assert ref.endswith(path)