Strip trailing slash from WebArena URLs (#366)

browsergym/visualwebarena/src/browsergym/visualwebarena/instance.py

@@ -33,7 +33,7 @@ def __init__(
                 + "Please set the following environment variables to use VisualWebArena through BrowserGym:\n"
                 + "\n".join([append_vwa(x) for x in ENV_VARS])
             )
-            os.environ[key] = os.environ[append_vwa(key)]
+            os.environ[key] = os.environ[append_vwa(key)].rstrip("/")
 
         # import webarena on instantiation
         from visualwebarena.browser_env.env_config import (

browsergym/webarena/README.md

@@ -29,7 +29,7 @@ python -c "import nltk; nltk.download('punkt_tab')"
 BASE_URL=<YOUR_SERVER_URL_HERE>  # example: "http://myazuremachine.eastus.cloudapp.azure.com"
 
 # webarena environment variables (change ports as needed)
-export WA_SHOPPING="$BASE_URL:8082/"
+export WA_SHOPPING="$BASE_URL:8082"
 export WA_SHOPPING_ADMIN="$BASE_URL:8083/admin"
 export WA_REDDIT="$BASE_URL:8080"
 export WA_GITLAB="$BASE_URL:9001"

browsergym/webarena/src/browsergym/webarena/instance.py

@@ -30,7 +30,8 @@ def __init__(
                 + "Please set the following environment variables to use WebArena through BrowserGym:\n"
                 + "\n".join([append_wa(x) for x in ENV_VARS])
             )
-            os.environ[key] = os.environ[append_wa(key)]
+            # no trailing slash, or "__SHOPPING__/path" becomes "host//path" (#366)
+            os.environ[key] = os.environ[append_wa(key)].rstrip("/")
 
         # import webarena on instanciation
         from webarena.browser_env.env_config import (

browsergym/webarena/src/browsergym/webarena/task.py

@@ -17,6 +17,22 @@
 logger = logging.getLogger(__name__)
 
 
+URL_PATTERNS = {
+    "__GITLAB__": "gitlab",
+    "__REDDIT__": "reddit",
+    "__SHOPPING__": "shopping",
+    "__SHOPPING_ADMIN__": "shopping_admin",
+    "__WIKIPEDIA__": "wikipedia",
+    "__MAP__": "map",
+}
+
+
+def substitute_urls(configs_str: str, urls: dict) -> str:
+    for pattern, url_key in URL_PATTERNS.items():
+        configs_str = configs_str.replace(pattern, urls[url_key])
+    return configs_str
+
+
 class GenericWebArenaTask(AbstractBrowserTask):
     """
     Base class for all WebArena tasks.
@@ -54,16 +70,7 @@ def __init__(
 
         all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text()
 
-        # substitute URLs
-        for pattern, url_key in {
-            "__GITLAB__": "gitlab",
-            "__REDDIT__": "reddit",
-            "__SHOPPING__": "shopping",
-            "__SHOPPING_ADMIN__": "shopping_admin",
-            "__WIKIPEDIA__": "wikipedia",
-            "__MAP__": "map",
-        }.items():
-            all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key])
+        all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls)
 
         # load all task configs to JSON
         all_configs = json.loads(all_configs_str)

browsergym/webarena_verified/src/browsergym/webarena_verified/task.py

@@ -10,7 +10,7 @@
 import playwright._impl._errors as playwright_errors
 import playwright.sync_api
 
-from browsergym.webarena.task import GenericWebArenaTask
+from browsergym.webarena.task import GenericWebArenaTask, substitute_urls
 from browsergym.webarena_verified.evaluators import WebArenaVerifiedEvaluator
 from webarena_verified.types import FinalAgentResponse
 
@@ -51,16 +51,7 @@ def __init__(
             .read_text()
         )
 
-        # substitute URLs
-        for pattern, url_key in {
-            "__GITLAB__": "gitlab",
-            "__REDDIT__": "reddit",
-            "__SHOPPING__": "shopping",
-            "__SHOPPING_ADMIN__": "shopping_admin",
-            "__WIKIPEDIA__": "wikipedia",
-            "__MAP__": "map",
-        }.items():
-            all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key])
+        all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls)
 
         # load all task configs to JSON
         all_configs = json.loads(all_configs_str)

browsergym/webarenalite/src/browsergym/webarenalite/task.py

@@ -8,7 +8,7 @@
 
 import playwright.sync_api
 
-from browsergym.webarena.task import GenericWebArenaTask
+from browsergym.webarena.task import GenericWebArenaTask, substitute_urls
 
 logger = logging.getLogger(__name__)
 
@@ -40,16 +40,7 @@ def __init__(
             .joinpath("test_webarena_lite.raw.json")
             .read_text()
         )
-        # substitute URLs
-        for pattern, url_key in {
-            "__GITLAB__": "gitlab",
-            "__REDDIT__": "reddit",
-            "__SHOPPING__": "shopping",
-            "__SHOPPING_ADMIN__": "shopping_admin",
-            "__WIKIPEDIA__": "wikipedia",
-            "__MAP__": "map",
-        }.items():
-            all_configs_str = all_configs_str.replace(pattern, self.webarena_instance.urls[url_key])
+        all_configs_str = substitute_urls(all_configs_str, self.webarena_instance.urls)
 
         # load all task configs to JSON
         all_configs = json.loads(all_configs_str)

tests/webarena/test_url_substitution.py

@@ -0,0 +1,91 @@
+import json
+import os
+import sys
+import types
+
+import pytest
+
+
+ENV_URLS = {
+    "WA_SHOPPING": "http://host:8082/",
+    "WA_SHOPPING_ADMIN": "http://host:8083/admin/",
+    "WA_REDDIT": "http://host:8080/",
+    "WA_GITLAB": "http://host:9001/",
+    "WA_WIKIPEDIA": "http://host:8081/wikipedia/",
+    "WA_MAP": "http://host:443/",
+    "WA_HOMEPAGE": "http://host:80/",
+}
+
+
+@pytest.fixture
+def fake_webarena(monkeypatch):
+    for k, v in ENV_URLS.items():
+        monkeypatch.setenv(k, v)
+        # WebArenaInstance sets SHOPPING etc., let monkeypatch undo that
+        monkeypatch.setenv(k.removeprefix("WA_"), "")
+
+    pkg = types.ModuleType("webarena")
+    browser_env = types.ModuleType("webarena.browser_env")
+    env_config = types.ModuleType("webarena.browser_env.env_config")
+
+    # read SHOPPING etc. from the env at import time, like the real module
+    def env_config_getattr(name):
+        if name in os.environ:
+            return os.environ[name]
+        raise AttributeError(name)
+
+    env_config.__getattr__ = env_config_getattr
+    env_config.ACCOUNTS = {}
+
+    monkeypatch.setitem(sys.modules, "webarena", pkg)
+    monkeypatch.setitem(sys.modules, "webarena.browser_env", browser_env)
+    monkeypatch.setitem(sys.modules, "webarena.browser_env.env_config", env_config)
+
+
+@pytest.mark.parametrize(
+    "site", ["reddit", "gitlab", "shopping", "shopping_admin", "wikipedia", "map"]
+)
+def test_urls_stripped(fake_webarena, site):
+    from browsergym.webarena.instance import WebArenaInstance
+
+    assert not WebArenaInstance().urls[site].endswith("/")
+
+
+def test_env_vars_stripped(fake_webarena):
+    # webarena's evaluator helpers build f"{SHOPPING}/path" from these
+    from browsergym.webarena.instance import WebArenaInstance
+
+    WebArenaInstance()
+    for k in ENV_URLS:
+        assert not os.environ[k.removeprefix("WA_")].endswith("/")
+
+
+def test_home_url_stripped(fake_webarena):
+    from browsergym.webarena.instance import WebArenaInstance
+
+    assert not WebArenaInstance().home_url.endswith("/")
+
+
+@pytest.mark.parametrize(
+    "placeholder,path",
+    [
+        ("__SHOPPING__", "/catalogsearch/result/?q=xbox"),
+        ("__SHOPPING_ADMIN__", "/sales/order/view/order_id/1/"),
+        ("__REDDIT__", "/f/AskReddit"),
+        ("__GITLAB__", "/root/some-project"),
+        ("__WIKIPEDIA__", "/wikipedia_en_all_maxi_2022-05/A/Alpha"),
+        ("__MAP__", "/?q=Pittsburgh"),
+    ],
+)
+def test_no_double_slash(fake_webarena, placeholder, path):
+    from browsergym.webarena.instance import WebArenaInstance
+    from browsergym.webarena.task import substitute_urls
+
+    raw = json.dumps({"eval": {"reference_url": f"{placeholder}{path}"}})
+    out = json.loads(substitute_urls(raw, WebArenaInstance().urls))
+    ref = out["eval"]["reference_url"]
+
+    first = path.lstrip("/").split("/")[0].split("?")[0]
+    if first:
+        assert f"//{first}" not in ref
+    assert ref.endswith(path)