diff --git a/browsergym/core/src/browsergym/core/action/functions.py b/browsergym/core/src/browsergym/core/action/functions.py index 5942de1cf..af4e41c44 100644 --- a/browsergym/core/src/browsergym/core/action/functions.py +++ b/browsergym/core/src/browsergym/core/action/functions.py @@ -278,18 +278,32 @@ def drag_and_drop(from_bid: str, to_bid: str): # https://playwright.dev/python/docs/api/class-mouse#mouse-wheel -def scroll(delta_x: float, delta_y: float): +def scroll(delta_x: float, delta_y: float, bid: str | None = None): """ - Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling. Dispatches a wheel event. + Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling. + If bid is given, scrolls inside that element (e.g. a listbox or a scrollable div) instead of the page. Examples: scroll(0, 200) scroll(-50.2, -100.5) + scroll(0, 200, bid='a17') """ delta_x, delta_y = map_coordinates( page, delta_x, delta_y ) # map coordinates to page coordinates - page.mouse.wheel(delta_x, delta_y) + if bid is None: + page.mouse.wheel(delta_x, delta_y) + else: + # don't scroll the element into view, that would move the page + elem = get_elem_by_bid(page, bid, scroll_into_view=False) + add_demo_mode_effects(page, elem, bid, demo_mode=demo_mode, move_cursor=False) + # scrollBy instead of a synthetic wheel event: untrusted wheel events + # don't scroll natively. Sites with custom wheel handlers (virtualized + # lists, carousels) won't react to this. + elem.evaluate( + "(el, deltas) => { el.scrollBy(deltas[0], deltas[1]); }", + [delta_x, delta_y], + ) def scroll_at(x: int, y: int, dx: int, dy: int): diff --git a/docs/src/core/action_space.rst b/docs/src/core/action_space.rst index 37b4c9bfa..ee015f59a 100644 --- a/docs/src/core/action_space.rst +++ b/docs/src/core/action_space.rst @@ -59,7 +59,7 @@ Below a list of the primitives supported by BrowserGym: + +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+ | | goto(url) | Navigate to a url. | +----------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------+ -| misc | scroll(dx, dy) | | +| misc | scroll(dx, dy, bid=None) | Scroll the page; if ``bid`` is given, scroll inside that element instead. | + +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+ | | send_msg_to_user(text) | | + +-----------------------------------------------------------+----------------------------------------------------------------------------------------------+ diff --git a/tests/core/data/scroll_list.html b/tests/core/data/scroll_list.html new file mode 100644 index 000000000..893325436 --- /dev/null +++ b/tests/core/data/scroll_list.html @@ -0,0 +1,49 @@ + + + + + + +

This is the top

+ + + +
+
item 0
+
item 1
+
item 2
+
item 3
+
item 4
+
item 5
+
item 6
+
item 7
+
item 8
+
item 9
+
+ +
+

This is the bottom

+ + diff --git a/tests/core/test_actions_highlevel.py b/tests/core/test_actions_highlevel.py index 4f0c08523..3ce363f28 100644 --- a/tests/core/test_actions_highlevel.py +++ b/tests/core/test_actions_highlevel.py @@ -30,6 +30,7 @@ DBLCLICK_URL = f"file://{__DATA_DIR}/dblclick.html" INEXISTANT_FILE_URL = f"file://{__DATA_DIR}/no_file_here.html" LONG_PAGE_URL = f"file://{__DATA_DIR}/long_page.html" +SCROLL_LIST_URL = f"file://{__DATA_DIR}/scroll_list.html" TEXT_INPUT_URL = f"file://{__DATA_DIR}/input_type/text_input.html" URL_INPUT_URL = f"file://{__DATA_DIR}/input_type/url_input.html" CHECKBOX_URL = f"file://{__DATA_DIR}/input_type/checkbox_input.html" @@ -1136,6 +1137,63 @@ def get_top_bottom_elems(obs): env.close() +def test_scroll_in_sub_element(): + action_set = HighLevelActionSet(subsets=["bid", "coord"]) + + env = gym.make( + "browsergym/openended", + task_kwargs={"start_url": SCROLL_LIST_URL}, + headless=__HEADLESS, + slow_mo=__SLOW_MO, + timeout=__TIMEOUT, + action_mapping=action_set.to_python_code, + ) + + def get_bid(obs, tag, elem_id): + soup = bs4.BeautifulSoup( + flatten_dom_to_str(obs["dom_object"], obs["extra_element_properties"]), + "lxml", + ) + elem = soup.find(tag, attrs={"id": elem_id}) + assert elem is not None, f"Element <{tag} id={elem_id}> not found" + return elem.get(BID_ATTR) + + def page_scroll_y(): + return env.unwrapped.page.evaluate("window.scrollY") + + def elem_scroll_top(selector): + return env.unwrapped.page.evaluate(f"document.querySelector({selector!r}).scrollTop") + + obs, info = env.reset() + listbox_bid = get_bid(obs, "select", "listbox") + scrollable_bid = get_bid(obs, "div", "scrollable") + + # baseline: nothing has been scrolled + assert page_scroll_y() == 0 + assert elem_scroll_top("#listbox") == 0 + assert elem_scroll_top("#scrollable") == 0 + + # scrolling the div by bid leaves the page alone + obs, _, _, _, _ = env.step(f"scroll(0, 200, bid={scrollable_bid!r})") + assert not obs["last_action_error"] + assert elem_scroll_top("#scrollable") > 0 + assert page_scroll_y() == 0 + + # same for the listbox + obs, _, _, _, _ = env.step(f"scroll(0, 500, bid={listbox_bid!r})") + assert not obs["last_action_error"] + assert elem_scroll_top("#listbox") > 0 + assert page_scroll_y() == 0 + + # plain scroll (no bid) still scrolls the page + prev_page_y = page_scroll_y() + obs, _, _, _, _ = env.step("scroll(0, 300)") + assert not obs["last_action_error"] + assert page_scroll_y() > prev_page_y + + env.close() + + def test_tab_actions(): action_set = HighLevelActionSet(subsets=["tab", "nav"]) diff --git a/tests/miniwob/test_click-scroll-list.py b/tests/miniwob/test_click-scroll-list.py index 8f16cd7c2..1d546ac72 100644 --- a/tests/miniwob/test_click-scroll-list.py +++ b/tests/miniwob/test_click-scroll-list.py @@ -3,8 +3,13 @@ import re import pytest +import bs4 + # register gym environments import browsergym.miniwob +from browsergym.core.action.highlevel import HighLevelActionSet +from browsergym.core.constants import BROWSERGYM_ID_ATTRIBUTE as BID_ATTR +from browsergym.utils.obs import flatten_dom_to_str __SLOW_MO = 1000 if "DISPLAY_BROWSER" in os.environ else None __HEADLESS = False if "DISPLAY_BROWSER" in os.environ else True @@ -40,3 +45,42 @@ def test_cheat(seed): assert term == True env.close() + + +def test_scroll_with_bid_scrolls_listbox(): + # #344: the listbox in this task is taller than its viewport, and the + # plain scroll() can't reach it + action_set = HighLevelActionSet(subsets=["bid", "coord"]) + env = gym.make( + "browsergym/miniwob.click-scroll-list", + headless=__HEADLESS, + slow_mo=__SLOW_MO, + action_mapping=action_set.to_python_code, + ) + obs, info = env.reset(seed=0) + assert obs["last_action_error"] == "" + + soup = bs4.BeautifulSoup( + flatten_dom_to_str(obs["dom_object"], obs["extra_element_properties"]), + "lxml", + ) + listbox = soup.find("select", attrs={"id": "options"}) + assert listbox is not None, "miniwob.click-scroll-list should have a #options listbox" + listbox_bid = listbox.get(BID_ATTR) + + page = env.unwrapped.page + + # baseline + assert page.evaluate("document.querySelector('#options').scrollTop") == 0 + + # plain scroll doesn't move the listbox + obs, _, _, _, _ = env.step("scroll(0, 200)") + assert not obs["last_action_error"] + assert page.evaluate("document.querySelector('#options').scrollTop") == 0 + + # scroll with bid: the listbox itself scrolls + obs, _, _, _, _ = env.step(f"scroll(0, 300, bid={listbox_bid!r})") + assert not obs["last_action_error"] + assert page.evaluate("document.querySelector('#options').scrollTop") > 0 + + env.close()