scroll: add optional bid to target sub-element scrolling (#344)

browsergym/core/src/browsergym/core/action/functions.py

@@ -278,18 +278,32 @@ def drag_and_drop(from_bid: str, to_bid: str):
 
 
 # https://playwright.dev/python/docs/api/class-mouse#mouse-wheel
-def scroll(delta_x: float, delta_y: float):
+def scroll(delta_x: float, delta_y: float, bid: str | None = None):
     """
-    Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling. Dispatches a wheel event.
+    Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling.
+    If bid is given, scrolls inside that element (e.g. a listbox or a scrollable div) instead of the page.
 
     Examples:
         scroll(0, 200)
         scroll(-50.2, -100.5)
+        scroll(0, 200, bid='a17')
     """
     delta_x, delta_y = map_coordinates(
         page, delta_x, delta_y
     )  # map coordinates to page coordinates
-    page.mouse.wheel(delta_x, delta_y)
+    if bid is None:
+        page.mouse.wheel(delta_x, delta_y)
+    else:
+        # don't scroll the element into view, that would move the page
+        elem = get_elem_by_bid(page, bid, scroll_into_view=False)
+        add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False)
+        # scrollBy instead of a synthetic wheel event: untrusted wheel events
+        # don't scroll natively. Sites with custom wheel handlers (virtualized
+        # lists, carousels) won't react to this.
+        elem.evaluate(
+            "(el, deltas) => { el.scrollBy(deltas[0], deltas[1]); }",
+            [delta_x, delta_y],
+        )
 
 
 def scroll_at(x: int, y: int, dx: int, dy: int):

docs/src/core/action_space.rst

@@ -59,7 +59,7 @@ Below a list of the primitives supported by BrowserGym:
 +          +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+
 |          | goto(url)                                                 | Navigate to a url.                                                                           |
 +----------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------+
-| misc     | scroll(dx, dy)                                            |                                                                                              |
+| misc     | scroll(dx, dy, bid=None)                                  | Scroll the page; if ``bid`` is given, scroll inside that element instead.                    |
 +          +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+
 |          | send_msg_to_user(text)                                    |                                                                                              |
 +          +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+

tests/core/data/scroll_list.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <style>
+        body { margin: 0; padding: 20px; }
+        #spacer { height: 2000px; background: linear-gradient(#eef, #fee); }
+        #scrollable {
+            width: 220px;
+            height: 120px;
+            overflow-y: scroll;
+            border: 1px solid #333;
+            margin: 10px 0;
+        }
+        #scrollable div { height: 40px; padding: 4px; border-bottom: 1px solid #ccc; }
+    </style>
+</head>
+<body>
+    <p id="top">This is the top</p>
+
+    <select id="listbox" size="4">
+        <option value="a">Apple</option>
+        <option value="b">Banana</option>
+        <option value="c">Cherry</option>
+        <option value="d">Date</option>
+        <option value="e">Elderberry</option>
+        <option value="f">Fig</option>
+        <option value="g">Grape</option>
+        <option value="h">Honeydew</option>
+        <option value="i">Iyokan</option>
+        <option value="j">Jackfruit</option>
+    </select>
+
+    <div id="scrollable">
+        <div id="item-0">item 0</div>
+        <div id="item-1">item 1</div>
+        <div id="item-2">item 2</div>
+        <div id="item-3">item 3</div>
+        <div id="item-4">item 4</div>
+        <div id="item-5">item 5</div>
+        <div id="item-6">item 6</div>
+        <div id="item-7">item 7</div>
+        <div id="item-8">item 8</div>
+        <div id="item-9">item 9</div>
+    </div>
+
+    <div id="spacer"></div>
+    <p id="bottom">This is the bottom</p>
+</body>
+</html>

tests/core/test_actions_highlevel.py

@@ -30,6 +30,7 @@
 DBLCLICK_URL = f"file://{__DATA_DIR}/dblclick.html"
 INEXISTANT_FILE_URL = f"file://{__DATA_DIR}/no_file_here.html"
 LONG_PAGE_URL = f"file://{__DATA_DIR}/long_page.html"
+SCROLL_LIST_URL = f"file://{__DATA_DIR}/scroll_list.html"
 TEXT_INPUT_URL = f"file://{__DATA_DIR}/input_type/text_input.html"
 URL_INPUT_URL = f"file://{__DATA_DIR}/input_type/url_input.html"
 CHECKBOX_URL = f"file://{__DATA_DIR}/input_type/checkbox_input.html"
@@ -1136,6 +1137,63 @@ def get_top_bottom_elems(obs):
     env.close()
 
 
+def test_scroll_in_sub_element():
+    action_set = HighLevelActionSet(subsets=["bid", "coord"])
+
+    env = gym.make(
+        "browsergym/openended",
+        task_kwargs={"start_url": SCROLL_LIST_URL},
+        headless=__HEADLESS,
+        slow_mo=__SLOW_MO,
+        timeout=__TIMEOUT,
+        action_mapping=action_set.to_python_code,
+    )
+
+    def get_bid(obs, tag, elem_id):
+        soup = bs4.BeautifulSoup(
+            flatten_dom_to_str(obs["dom_object"], obs["extra_element_properties"]),
+            "lxml",
+        )
+        elem = soup.find(tag, attrs={"id": elem_id})
+        assert elem is not None, f"Element <{tag} id={elem_id}> not found"
+        return elem.get(BID_ATTR)
+
+    def page_scroll_y():
+        return env.unwrapped.page.evaluate("window.scrollY")
+
+    def elem_scroll_top(selector):
+        return env.unwrapped.page.evaluate(f"document.querySelector({selector!r}).scrollTop")
+
+    obs, info = env.reset()
+    listbox_bid = get_bid(obs, "select", "listbox")
+    scrollable_bid = get_bid(obs, "div", "scrollable")
+
+    # baseline: nothing has been scrolled
+    assert page_scroll_y() == 0
+    assert elem_scroll_top("#listbox") == 0
+    assert elem_scroll_top("#scrollable") == 0
+
+    # scrolling the div by bid leaves the page alone
+    obs, _, _, _, _ = env.step(f"scroll(0, 200, bid={scrollable_bid!r})")
+    assert not obs["last_action_error"]
+    assert elem_scroll_top("#scrollable") > 0
+    assert page_scroll_y() == 0
+
+    # same for the listbox
+    obs, _, _, _, _ = env.step(f"scroll(0, 500, bid={listbox_bid!r})")
+    assert not obs["last_action_error"]
+    assert elem_scroll_top("#listbox") > 0
+    assert page_scroll_y() == 0
+
+    # plain scroll (no bid) still scrolls the page
+    prev_page_y = page_scroll_y()
+    obs, _, _, _, _ = env.step("scroll(0, 300)")
+    assert not obs["last_action_error"]
+    assert page_scroll_y() > prev_page_y
+
+    env.close()
+
+
 def test_tab_actions():
     action_set = HighLevelActionSet(subsets=["tab", "nav"])
 

tests/miniwob/test_click-scroll-list.py

@@ -3,8 +3,13 @@
 import re
 import pytest
 
+import bs4
+
 # register gym environments
 import browsergym.miniwob
+from browsergym.core.action.highlevel import HighLevelActionSet
+from browsergym.core.constants import BROWSERGYM_ID_ATTRIBUTE as BID_ATTR
+from browsergym.utils.obs import flatten_dom_to_str
 
 __SLOW_MO = 1000 if "DISPLAY_BROWSER" in os.environ else None
 __HEADLESS = False if "DISPLAY_BROWSER" in os.environ else True
@@ -40,3 +45,42 @@ def test_cheat(seed):
     assert term == True
 
     env.close()
+
+
+def test_scroll_with_bid_scrolls_listbox():
+    # #344: the listbox in this task is taller than its viewport, and the
+    # plain scroll() can't reach it
+    action_set = HighLevelActionSet(subsets=["bid", "coord"])
+    env = gym.make(
+        "browsergym/miniwob.click-scroll-list",
+        headless=__HEADLESS,
+        slow_mo=__SLOW_MO,
+        action_mapping=action_set.to_python_code,
+    )
+    obs, info = env.reset(seed=0)
+    assert obs["last_action_error"] == ""
+
+    soup = bs4.BeautifulSoup(
+        flatten_dom_to_str(obs["dom_object"], obs["extra_element_properties"]),
+        "lxml",
+    )
+    listbox = soup.find("select", attrs={"id": "options"})
+    assert listbox is not None, "miniwob.click-scroll-list should have a #options listbox"
+    listbox_bid = listbox.get(BID_ATTR)
+
+    page = env.unwrapped.page
+
+    # baseline
+    assert page.evaluate("document.querySelector('#options').scrollTop") == 0
+
+    # plain scroll doesn't move the listbox
+    obs, _, _, _, _ = env.step("scroll(0, 200)")
+    assert not obs["last_action_error"]
+    assert page.evaluate("document.querySelector('#options').scrollTop") == 0
+
+    # scroll with bid: the listbox itself scrolls
+    obs, _, _, _, _ = env.step(f"scroll(0, 300, bid={listbox_bid!r})")
+    assert not obs["last_action_error"]
+    assert page.evaluate("document.querySelector('#options').scrollTop") > 0
+
+    env.close()