refactor(test): use dynamic ports and fixture injection for test parallelization

test/conftest.py

@@ -13,6 +13,7 @@
 
 from . import utilities
 from .openwpmtest import NUM_BROWSERS
+from .utilities import ServerUrls
 
 EXTENSION_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
@@ -52,10 +53,10 @@ def xpi_fixture():
 
 @pytest.fixture(scope="session")
 def server():
-    """Run an HTTP server during the tests."""
+    """Run an HTTP server during the tests and yield its URLs."""
     print("Starting local_http_server")
-    server, server_thread = utilities.start_server()
-    yield
+    server, server_thread, urls = utilities.start_server()
+    yield urls
     print("\nClosing server thread...")
     server.shutdown()
     server_thread.join()
@@ -85,7 +86,7 @@ def default_params(tmp_path: Path, num_browsers: int = NUM_BROWSERS) -> FullConf
 
 
 @pytest.fixture()
-def task_manager_creator(server: None, xpi: None) -> TaskManagerCreator:
+def task_manager_creator(server: ServerUrls, xpi: None) -> TaskManagerCreator:
     """We create a callable that returns a TaskManager that has
     been configured with the Manager and BrowserParams"""
 

test/manual_test.py

@@ -17,7 +17,7 @@
 from openwpm.utilities.platform_utils import get_firefox_binary_path
 
 from .conftest import xpi
-from .utilities import BASE_TEST_URL, start_server
+from .utilities import start_server
 
 # import commonly used modules and utilities so they can be easily accessed
 # in the interactive session
@@ -103,10 +103,10 @@ def start_webdriver(
     firefox_binary_path = get_firefox_binary_path()
 
     fb = FirefoxBinary(firefox_path=firefox_binary_path)
-    server, thread = start_server()
+    server, thread, urls = start_server()
 
     def register_cleanup(driver):
-        driver.get(BASE_TEST_URL)
+        driver.get(urls.base)
 
         def cleanup_server():
             print("Cleanup before shutdown...")
@@ -160,12 +160,12 @@ def cleanup_server():
 
 def start_webext():
     firefox_binary_path = get_firefox_binary_path()
+    server, thread, urls = start_server()
     cmd_webext_run = f"""
     npm start -- \
-            --start-url '{BASE_TEST_URL}' \
+            --start-url '{urls.base}' \
             --firefox '{firefox_binary_path}'
     """
-    server, thread = start_server()
     try:
         # http://stackoverflow.com/a/4417735/3104416
         for line in get_command_output(cmd_webext_run, cwd=EXT_PATH):

test/openwpmtest.py

@@ -8,7 +8,7 @@
 from openwpm.config import BrowserParams, ManagerParams
 from openwpm.storage.sql_provider import SQLiteStorageProvider
 
-from . import utilities
+from .utilities import ServerUrls
 
 NUM_BROWSERS = 2
 
@@ -24,6 +24,11 @@ def set_tmpdir(self, tmpdir):
         """
         self.tmpdir = Path(tmpdir)
 
+    @pytest.fixture(autouse=True)
+    def set_server(self, server: ServerUrls) -> None:
+        """Inject server URLs for use in test methods."""
+        self.server = server
+
     def get_config(
         self, data_dir: Optional[Path]
     ) -> Tuple[ManagerParams, List[BrowserParams]]:
@@ -46,7 +51,7 @@ def visit(
             None,
         )
         if not page_url.startswith("http"):
-            page_url = utilities.BASE_TEST_URL + page_url
+            page_url = self.server.base + page_url
         manager.get(url=page_url, sleep=sleep_after)
         manager.close()
         return db_path

test/test_callback.py

@@ -4,16 +4,18 @@
 from openwpm.command_sequence import CommandSequence
 
 from .conftest import FullConfig, TaskManagerCreator
-from .utilities import BASE_TEST_URL
+from .utilities import ServerUrls
 
 
 def test_local_callbacks(
-    default_params: FullConfig, task_manager_creator: TaskManagerCreator
+    default_params: FullConfig,
+    task_manager_creator: TaskManagerCreator,
+    server: ServerUrls,
 ) -> None:
     """Test the storage controller as well as the entire callback machinery
     to see if all callbacks get correctly called"""
     manager, _ = task_manager_creator(default_params)
-    TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
+    TEST_SITE = server.base + "/simple_a.html"
 
     def callback(argument: List[int], success: bool) -> None:
         argument.extend([1, 2, 3])

test/test_callstack_instrument.py

@@ -2,39 +2,38 @@
 
 from openwpm.utilities import db_utils
 
-from . import utilities
-
-# HTTP request call stack instrumentation
-# Expected stack frames
-HTTP_STACKTRACE_TEST_URL = utilities.BASE_TEST_URL + "/http_stacktrace.html"
-STACK_TRACE_INJECT_IMAGE = (
-    "inject_image@" + HTTP_STACKTRACE_TEST_URL + ":18:7;null\n"
-    "inject_all@" + HTTP_STACKTRACE_TEST_URL + ":22:7;null\n"
-    "onload@" + HTTP_STACKTRACE_TEST_URL + ":1:1;null"
-)
+from .conftest import FullConfig, TaskManagerCreator
+from .utilities import ServerUrls
 
 RAWGIT_HTTP_STACKTRACE_TEST_URL = "https://gist.githack.com/gunesacar/b927d3fe69f3e7bf456da5192f74beea/raw/8d3e490b5988c633101ec45ef1443e61b1fd495e/inject_pixel.js"  # noqa
-# https://gist.github.com/gunesacar/b927d3fe69f3e7bf456da5192f74beea
-STACK_TRACE_INJECT_PIXEL = (
-    "inject_pixel@" + RAWGIT_HTTP_STACKTRACE_TEST_URL + ":4:3;null\n"
-    "null@" + RAWGIT_HTTP_STACKTRACE_TEST_URL + ":6:1;null"
-)
 
-STACK_TRACE_INJECT_JS = (
-    "inject_js@" + HTTP_STACKTRACE_TEST_URL + ":13:28;null\n"
-    "inject_all@" + HTTP_STACKTRACE_TEST_URL + ":21:7;null\n"
-    "onload@" + HTTP_STACKTRACE_TEST_URL + ":1:1;null"
-)
 
-HTTP_STACKTRACES = {
-    STACK_TRACE_INJECT_IMAGE,
-    STACK_TRACE_INJECT_PIXEL,
-    STACK_TRACE_INJECT_JS,
-}
+def _http_stacktraces(base_url: str) -> set[str]:
+    """Build expected stack traces (needs dynamic base_url)."""
+    url = base_url + "/http_stacktrace.html"
+    inject_image = (
+        f"inject_image@{url}:18:7;null\n"
+        f"inject_all@{url}:22:7;null\n"
+        f"onload@{url}:1:1;null"
+    )
+    inject_pixel = (
+        f"inject_pixel@{RAWGIT_HTTP_STACKTRACE_TEST_URL}:4:3;null\n"
+        f"null@{RAWGIT_HTTP_STACKTRACE_TEST_URL}:6:1;null"
+    )
+    inject_js = (
+        f"inject_js@{url}:13:28;null\n"
+        f"inject_all@{url}:21:7;null\n"
+        f"onload@{url}:1:1;null"
+    )
+    return {inject_image, inject_pixel, inject_js}
 
 
 @pytest.mark.skip("We don't have the resources to fix this")
-def test_http_stacktrace(default_params, task_manager_creator):
+def test_http_stacktrace(
+    default_params: FullConfig,
+    task_manager_creator: TaskManagerCreator,
+    server: ServerUrls,
+) -> None:
     manager_params, browser_params = default_params
     for browser_param in browser_params:
         # Record HTTP Requests and Responses
@@ -43,7 +42,7 @@ def test_http_stacktrace(default_params, task_manager_creator):
         browser_param.js_instrument = True
         # Record the callstack of all WebRequests made
         browser_param.callstack_instrument = True
-    test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html"
+    test_url = server.base + "/http_stacktrace.html"
     manager, db = task_manager_creator((manager_params, browser_params))
     manager.get(test_url, sleep=10)
     manager.close()
@@ -59,15 +58,16 @@ def test_http_stacktrace(default_params, task_manager_creator):
         ),
     )
     print("Printing callstacks contents")
-    observed_records = set()
+    observed_records: set[str] = set()
     for row in rows:
+        assert not isinstance(row, tuple)
         print(row["call_stack"])
-        url, call_stack = row
+        url, call_stack = row["url"], row["call_stack"]
         test_urls = (
             "inject_pixel.js",
             "test_image.png",
             "Blank.gif",
         )
         if url.endswith(test_urls):
             observed_records.add(call_stack)
-    assert HTTP_STACKTRACES == observed_records
+    assert _http_stacktraces(server.base) == observed_records

test/test_custom_function_command.py

@@ -12,24 +12,25 @@
 from openwpm.task_manager import TaskManager
 from openwpm.utilities import db_utils
 
-from . import utilities
-
-url_a = utilities.BASE_TEST_URL + "/simple_a.html"
-
-PAGE_LINKS = {
-    (
-        f"{utilities.BASE_TEST_URL}/simple_a.html",
-        f"{utilities.BASE_TEST_URL}/simple_c.html",
-    ),
-    (
-        f"{utilities.BASE_TEST_URL}/simple_a.html",
-        f"{utilities.BASE_TEST_URL}/simple_d.html",
-    ),
-    (
-        f"{utilities.BASE_TEST_URL}/simple_a.html",
-        "http://example.com/test.html?localhost",
-    ),
-}
+from .conftest import FullConfig
+from .utilities import ServerUrls
+
+
+def _page_links(base: str) -> set:
+    return {
+        (
+            f"{base}/simple_a.html",
+            f"{base}/simple_c.html",
+        ),
+        (
+            f"{base}/simple_a.html",
+            f"{base}/simple_d.html",
+        ),
+        (
+            f"{base}/simple_a.html",
+            "http://example.com/test.html?localhost",
+        ),
+    }
 
 
 class CollectLinksCommand(BaseCommand):
@@ -77,9 +78,12 @@ def execute(
         sock.close()
 
 
-def test_custom_function(default_params, xpi, server):
+def test_custom_function(
+    default_params: FullConfig, xpi: None, server: ServerUrls
+) -> None:
     """Test `custom_function` with an inline func that collects links"""
     table_name = TableName("page_links")
+    url_a = server.base + "/simple_a.html"
 
     manager_params, browser_params = default_params
     path = manager_params.data_directory / "crawl-data.sqlite"
@@ -107,4 +111,4 @@ def test_custom_function(default_params, xpi, server):
         "SELECT top_url, link FROM page_links;",
         as_tuple=True,
     )
-    assert PAGE_LINKS == set(query_result)
+    assert _page_links(server.base) == set(query_result)

test/test_dns_instrument.py

@@ -2,14 +2,21 @@
 
 from openwpm.utilities import db_utils
 
+from .conftest import FullConfig, TaskManagerCreator
+from .utilities import ServerUrls
 
-def test_name_resolution(default_params, task_manager_creator):
+
+def test_name_resolution(
+    default_params: FullConfig,
+    task_manager_creator: TaskManagerCreator,
+    server: ServerUrls,
+) -> None:
     manager_params, browser_params = default_params
     for browser_param in browser_params:
         browser_param.dns_instrument = True
 
     manager, db = task_manager_creator((manager_params, browser_params))
-    manager.get("http://test.localhost:8000")
+    manager.get(f"http://test.localhost:{server.port}")
     manager.close()
 
     results = db_utils.query_db(db, "SELECT * FROM dns_responses")