diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 8028a8750..516e19e87 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -195,20 +195,14 @@ def test_011_failed_start_timeout(self): self.assertEqual(self.startup_counter, 1) -class TC_20_DispVMMixin: - def setUp(self): # pylint: disable=invalid-name - logger.info("start") - super().setUp() - if "whonix-g" in self.template: - self.skipTest( - "whonix gateway is not supported as DisposableVM Template" - ) +class DispVMHelpersMixin: + def setup_dispvm_nodes(self): + """Initialize disp_base and related attributes. Called by child setUp""" self.app.add_handler("domain-add", self._on_domain_add) self.addCleanup( self.app.remove_handler, "domain-add", self._on_domain_add ) self.adminvm = self.app.domains["dom0"] - self.init_default_template(self.template) self.disp_base = self.app.add_new_vm( qubes.vm.appvm.AppVM, name=self.make_vm_name("dvm"), @@ -521,6 +515,19 @@ async def run_preload(self): self.assertNotIn(dispvm_name, next_preload_list) logger.info("end") + +class TC_20_DispVMMixin(DispVMHelpersMixin): + def setUp(self): # pylint: disable=invalid-name + logger.info("start") + super().setUp() + if "whonix-g" in self.template: + self.skipTest( + "whonix gateway is not supported as DisposableVM Template" + ) + self.init_default_template(self.template) + self.setup_dispvm_nodes() + logger.info("end") + def test_010_dvm_run_simple(self): dispvm = self.loop.run_until_complete( qubes.vm.dispvm.DispVM.from_appvm(self.disp_base) @@ -536,55 +543,6 @@ def test_010_dvm_run_simple(self): finally: self.loop.run_until_complete(dispvm.cleanup()) - def test_011_preload_reject_max(self): - """Test preloading when max has been reached""" - self.loop.run_until_complete( - qubes.vm.dispvm.DispVM.from_appvm(self.disp_base, preload=True) - ) - self.assertEqual(0, len(self.disp_base.get_feat_preload())) - - def test_012_preload_low_mem(self): - """Test preloading with low memory""" - self.loop.run_until_complete(self._test_012_preload_low_mem()) - - async def _test_012_preload_low_mem(self): - # pylint: disable=unspecified-encoding - logger.info("start") - unpatched_open = open - memory = int(getattr(self.disp_base, "memory", 0) * 1024**2) - - def mock_open_mem(file, *args, **kwargs): - if file == qubes.config.qmemman_avail_mem_file: - return mock_open(read_data=str(memory))() - return unpatched_open(file, *args, **kwargs) - - def mock_open_mem_threshold(file, *args, **kwargs): - if file == qubes.config.qmemman_avail_mem_file: - return mock_open(read_data=str(memory * 2))() - return unpatched_open(file, *args, **kwargs) - - preload_max = 2 - with patch("builtins.open", side_effect=mock_open_mem): - logger.info("low mem standard") - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload( - preload_max, fail_on_timeout=False, timeout=15 - ) - self.assertEqual(1, len(self.disp_base.get_feat_preload())) - # Nothing will be done here, just to prepare to the next test. - self.disp_base.features["preload-dispvm-max"] = str(preload_max - 1) - - with patch("builtins.open", side_effect=mock_open_mem_threshold): - logger.info("low mem threshold") - self.adminvm.features["preload-dispvm-threshold"] = memory - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload( - preload_max, fail_on_timeout=False, timeout=15 - ) - self.assertEqual(1, len(self.disp_base.get_feat_preload())) - - logger.info("end") - def test_013_preload_gui(self): """Test preloading with GUI feature enabled and use after completion.""" @@ -614,382 +572,112 @@ async def _test_014_preload_nogui(self): await self.run_preload() logger.info("end") - def test_015_preload_race_more(self): - """Test race requesting multiple preloaded qubes""" - self.loop.run_until_complete(self._test_015_preload_race_more()) - - async def _test_015_preload_race_more(self): - logger.info("start") - preload_max = 3 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - old_preload = self.disp_base.get_feat_preload() - tasks = [self.run_preload_proc() for _ in range(preload_max)] - targets = await asyncio.gather(*tasks) - await self.wait_preload(preload_max) - preload_dispvm = self.disp_base.get_feat_preload() - self.assertTrue(set(old_preload).isdisjoint(preload_dispvm)) - self.assertEqual(len(targets), preload_max) - self.assertEqual(len(targets), len(set(targets))) - logger.info("end") - - def test_016_preload_race_less(self): - """Test race requesting preloaded qube while the maximum is zeroed.""" - self.loop.run_until_complete(self._test_016_preload_race_less()) + @unittest.skipUnless(which("xdotool"), "xdotool not installed") + def test_080_gui_app(self): + dispvm = self.loop.run_until_complete( + qubes.vm.dispvm.DispVM.from_appvm(self.disp_base) + ) + try: + self.loop.run_until_complete(dispvm.start()) + self.loop.run_until_complete(self.wait_for_session(dispvm)) + p = self.loop.run_until_complete( + dispvm.run_service( + "qubes.VMShell", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + ) + # wait for DispVM startup: + p.stdin.write(b"echo test\n") + self.loop.run_until_complete(p.stdin.drain()) + line = self.loop.run_until_complete(p.stdout.readline()) + self.assertEqual(line, b"test\n") - async def _test_016_preload_race_less(self): - logger.info("start") - preload_max = 1 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max, wait_completion=False) - tasks = [self.run_preload_proc(), self.no_preload()] - target = await asyncio.gather(*tasks) - target_dispvm = target[0] - self.assertTrue(target_dispvm.startswith("disp")) - logger.info("end") + self.assertTrue(dispvm.is_running()) + try: + window_title = "user@%s" % (dispvm.name,) + # close xterm on Return, but after short delay, to allow + # xdotool to send also keyup event + p.stdin.write( + "xterm -e " + '"sh -c \'echo \\"\033]0;{}\007\\";read x;' + "sleep 0.1;'\"\n".format(window_title).encode() + ) + self.loop.run_until_complete(p.stdin.drain()) + self.wait_for_window(window_title) - def test_017_preload_autostart(self): - """The script triggers the API call 'admin.vm.CreateDisposable+preload' - which is responsible for bootstrapping.""" - logger.info("start") - self.app.default_dispvm = self.disp_base + time.sleep(0.5) + self.enter_keys_in_window(window_title, ["Return"]) + # Wait for window to close + self.wait_for_window(window_title, show=False) + p.stdin.close() + self.loop.run_until_complete(asyncio.wait_for(p.wait(), 30)) + except: + with suppress(ProcessLookupError): + p.terminate() + self.loop.run_until_complete(p.wait()) + raise + finally: + del p + finally: + self.loop.run_until_complete(dispvm.cleanup()) + dispvm_name = dispvm.name + del dispvm - logger.info("must not change as max is 0") - proc = self.loop.run_until_complete( - asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") - ) - self.loop.run_until_complete( - asyncio.wait_for(proc.communicate(), timeout=10) - ) - self.assertEqual(self.disp_base.get_feat_preload(), []) + # give it a time for shutdown + cleanup + self.loop.run_until_complete(asyncio.sleep(5)) - preload_max = 1 - logger.info("must not change existing preloaded disposables") - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - self.loop.run_until_complete(self.wait_preload(preload_max)) - old_preload = self.disp_base.get_feat_preload() - proc = self.loop.run_until_complete( - asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") - ) - self.loop.run_until_complete(asyncio.wait_for(proc.wait(), timeout=10)) - preload_dispvm = self.disp_base.get_feat_preload() - self.assertEqual( - old_preload, - preload_dispvm, - msg=f"old_preload={old_preload} preload_dispvm={preload_dispvm}", + self.assertNotIn( + dispvm_name, self.app.domains, "DispVM not removed from qubes.xml" ) - preload_max += 1 - logger.info("global refill must work") - self.adminvm.features["preload-dispvm-max"] = str(preload_max) - self.loop.run_until_complete(self.wait_preload(preload_max)) - old_old_preload = self.disp_base.get_feat_preload() - self.loop.run_until_complete( - self.cleanup_preload_run(self.disp_base, down_to=preload_max - 1) - ) - old_preload = self.disp_base.get_feat_preload() - self.assertEqual(len(old_preload), preload_max - 1) - self.assertIn(old_preload[0], old_old_preload) - proc = self.loop.run_until_complete( - asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") + def _handle_editor(self, winid, copy=False): + with subprocess.Popen( + ["xdotool", "getwindowname", winid], stdout=subprocess.PIPE + ) as proc: + (window_title, _) = proc.communicate() + window_title = ( + window_title.decode() + .strip() + .replace("(", r"\(") + .replace(")", r"\)") ) - try: - self.loop.run_until_complete( - asyncio.wait_for(proc.wait(), timeout=40) + time.sleep(1) + if "LibreOffice" in window_title: + # wait for actual editor (we've got splash screen) + with subprocess.Popen( + [ + "xdotool", + "search", + "--sync", + "--onlyvisible", + "--all", + "--name", + "--class", + "disp*|Writer", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) as search: + retcode = search.wait() + if retcode == 0: + winid = search.stdout.read().strip() + time.sleep(0.5) + subprocess.check_call( + ["xdotool", "windowactivate", "--sync", winid] ) - except asyncio.TimeoutError: - debug_preload = self.disp_base.get_feat_preload() - for qube in debug_preload: - qube = self.app.domains[qube] - if qube.is_paused(): - self.loop.run_until_complete( - asyncio.wait_for(qube.unpause(), timeout=5) - ) - self._run_cmd_and_log_output( - qube, "systemtl --user is-system-running", user="user" - ) - self._run_cmd_and_log_output( - qube, "systemtl is-system-running", user="root" - ) - self._run_cmd_and_log_output( - qube, "systemd-analyze --no-pager --user blame", user="user" - ) - self._run_cmd_and_log_output( - qube, "systemd-analyze --no-pager blame", user="root" - ) - self._run_cmd_and_log_output( - qube, "journalctl --no-pager --user", user="user" - ) - self._run_cmd_and_log_output( - qube, "journalctl --no-pager", user="root" - ) - raise - preload_dispvm = self.disp_base.get_feat_preload() - self.assertIn(old_preload[0], preload_dispvm) - self.assertEqual(len(preload_dispvm), preload_max) - - self.app.default_dispvm = None - logger.info("end") - - def test_018_preload_global(self): - """Tweak global preload setting and global dispvm.""" - self.loop.run_until_complete(self._test_018_preload_global()) - - async def _test_018_preload_global(self): - logger.info("start") - self.log_preload() - preload_max = 1 - - logger.info("set global dispvm") - self.app.default_dispvm = self.disp_base - logger.info("set global feat, state must change") - self.adminvm.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - - self.log_preload() - logger.info("set local feat, state must not change") - preload_max += 1 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max, fail_on_timeout=False, timeout=15) - self.assertEqual(len(self.disp_base.get_feat_preload()), 1) - - self.log_preload() - logger.info("del local feat, state must not change") - del self.disp_base.features["preload-dispvm-max"] - await asyncio.sleep(5) - self.assertEqual(len(self.disp_base.get_feat_preload()), 1) - - self.log_preload() - logger.info("set local feat and del global feat, state must change") - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - del self.adminvm.features["preload-dispvm-max"] - await self.wait_preload(preload_max) - - self.log_preload() - logger.info("del local feat and set global feat, state must change") - preload_max -= 1 - preload_remove = self.app.default_dispvm.get_feat_preload() - self.disp_base.features["preload-dispvm-max"] = "" - self.wait_for_dispvm_destroy(preload_remove) - self.adminvm.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - self.assertEqual(len(self.disp_base.get_feat_preload()), preload_max) - - self.log_preload() - logger.info("switch global dispvm, state must change") - self.app.default_dispvm = self.disp_base_alt - await self.wait_preload(preload_max, appvm=self.disp_base_alt) - - self.log_preload() - logger.info("set local feat, state must not change") - preload_max += 1 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - - self.log_preload() - logger.info("switch back global dispvm, state must change") - preload_remove = self.app.default_dispvm.get_feat_preload() - self.app.default_dispvm = self.disp_base - self.wait_for_dispvm_destroy(preload_remove) - await self.wait_preload(preload_max) - - self.log_preload() - logger.info("unset global dispvm, state must change") - preload_remove = self.app.default_dispvm.get_feat_preload() - self.app.default_dispvm = None - self.wait_for_dispvm_destroy(preload_remove) - - self.log_preload() - logger.info("end") - - def test_019_preload_discard_outdated_volumes(self): - """Discard preload if volumes are outdated compared to its templates.""" - self.loop.run_until_complete( - self._test_019_preload_discard_outdated_volumes() - ) - - async def _test_019_preload_discard_outdated_volumes(self): - logger.info("start") - self.log_preload() - preload_max = 1 - - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - for qube in [self.disp_base, self.disp_base.template]: - logger.info( - "discard because of outdated volume originating from %s", - qube.name, - ) - await self.wait_preload(preload_max) - old_preload = self.disp_base.get_feat_preload() - await qube.start() - # If services are still starting, it may delay shutdown longer than - # the default timeout. Because we can't just kill default - # templates, wait gracefully for system services to have started. - await qube.run_service_for_stdio("qubes.WaitForRunningSystem") - logger.info("shutdown '%s'", qube.name) - await qube.shutdown(wait=True) - await self.wait_preload(preload_max) - preload_dispvm = self.disp_base.get_feat_preload() - self.assertTrue( - set(old_preload).isdisjoint(preload_dispvm), - f"old_preload={old_preload} preload_dispvm={preload_dispvm}", - ) - - self.log_preload() - logger.info("end") - - def test_020_preload_discard_outdated_volume_size(self): - """Discard preload if private size differs with disposable template.""" - self.loop.run_until_complete( - self._test_020_preload_discard_outdated_volume_size() - ) - - async def _test_020_preload_discard_outdated_volume_size(self): - logger.info("start") - self.log_preload() - preload_max = 1 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - preload_dispvm = self.disp_base.get_feat_preload() - old_size = self.disp_base.volume_config["private"]["size"] - size = int(old_size) + 512 - await self.disp_base.storage.resize("private", size) - self.app.save() - dispvm = await asyncio.wait_for( - qubes.vm.dispvm.DispVM.from_appvm(self.disp_base), 30 - ) - self.assertNotIn(dispvm.name, preload_dispvm) - await dispvm.cleanup() - logger.info("end") - - def test_021_preload_discard_outdated_setting(self): - """Discard preload if properties differ with the disposable template.""" - self.loop.run_until_complete( - self._test_021_preload_discard_outdated_setting() - ) - - async def _test_021_preload_discard_outdated_setting(self): - logger.info("start") - self.log_preload() - preload_max = 1 - self.disp_base.features["preload-dispvm-max"] = str(preload_max) - await self.wait_preload(preload_max) - preload_dispvm = self.disp_base.get_feat_preload() - self.disp_base.netvm = None - dispvm = await asyncio.wait_for( - qubes.vm.dispvm.DispVM.from_appvm(self.disp_base), 30 - ) - self.assertNotIn(dispvm.name, preload_dispvm) - await dispvm.cleanup() - await self.wait_preload(preload_max) - logger.info("end") - - @unittest.skipUnless(which("xdotool"), "xdotool not installed") - def test_080_gui_app(self): - dispvm = self.loop.run_until_complete( - qubes.vm.dispvm.DispVM.from_appvm(self.disp_base) - ) - try: - self.loop.run_until_complete(dispvm.start()) - self.loop.run_until_complete(self.wait_for_session(dispvm)) - p = self.loop.run_until_complete( - dispvm.run_service( - "qubes.VMShell", - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - ) - # wait for DispVM startup: - p.stdin.write(b"echo test\n") - self.loop.run_until_complete(p.stdin.drain()) - line = self.loop.run_until_complete(p.stdout.readline()) - self.assertEqual(line, b"test\n") - - self.assertTrue(dispvm.is_running()) - try: - window_title = "user@%s" % (dispvm.name,) - # close xterm on Return, but after short delay, to allow - # xdotool to send also keyup event - p.stdin.write( - "xterm -e " - '"sh -c \'echo \\"\033]0;{}\007\\";read x;' - "sleep 0.1;'\"\n".format(window_title).encode() - ) - self.loop.run_until_complete(p.stdin.drain()) - self.wait_for_window(window_title) - - time.sleep(0.5) - self.enter_keys_in_window(window_title, ["Return"]) - # Wait for window to close - self.wait_for_window(window_title, show=False) - p.stdin.close() - self.loop.run_until_complete(asyncio.wait_for(p.wait(), 30)) - except: - with suppress(ProcessLookupError): - p.terminate() - self.loop.run_until_complete(p.wait()) - raise - finally: - del p - finally: - self.loop.run_until_complete(dispvm.cleanup()) - dispvm_name = dispvm.name - del dispvm - - # give it a time for shutdown + cleanup - self.loop.run_until_complete(asyncio.sleep(5)) - - self.assertNotIn( - dispvm_name, self.app.domains, "DispVM not removed from qubes.xml" - ) - - def _handle_editor(self, winid, copy=False): - with subprocess.Popen( - ["xdotool", "getwindowname", winid], stdout=subprocess.PIPE - ) as proc: - (window_title, _) = proc.communicate() - window_title = ( - window_title.decode() - .strip() - .replace("(", r"\(") - .replace(")", r"\)") - ) - time.sleep(1) - if "LibreOffice" in window_title: - # wait for actual editor (we've got splash screen) - with subprocess.Popen( - [ - "xdotool", - "search", - "--sync", - "--onlyvisible", - "--all", - "--name", - "--class", - "disp*|Writer", - ], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) as search: - retcode = search.wait() - if retcode == 0: - winid = search.stdout.read().strip() - time.sleep(0.5) - subprocess.check_call( - ["xdotool", "windowactivate", "--sync", winid] - ) - if copy: - subprocess.check_call( - [ - "xdotool", - "key", - "--window", - winid, - "key", - "ctrl+a", - "ctrl+c", - "ctrl+shift+c", - ] + if copy: + subprocess.check_call( + [ + "xdotool", + "key", + "--window", + winid, + "key", + "ctrl+a", + "ctrl+c", + "ctrl+shift+c", + ] ) else: subprocess.check_call(["xdotool", "type", "Test test 2"]) @@ -1130,217 +818,554 @@ def test_090_edit_file(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - ) + ) + + if "whonix-workstation" in self.template: + dvm_confirm_rslt = self._whonix_ws_dispvm_confirm("edit file") + if not dvm_confirm_rslt[0]: + self.fail(dvm_confirm_rslt[1]) + + # if first 5 windows isn't expected editor, there is no hope + winid = None + for _ in range(5): + try: + winid = self.wait_for_window( + "disp[0-9]*", + search_class=True, + include_tray=False, + timeout=60, + ) + except Exception as e: # pylint: disable=broad-exception-caught + try: + self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1)) + except asyncio.TimeoutError: + raise e + stdout = self.loop.run_until_complete(p.stdout.read()) + self.fail( + "qvm-open-in-dvm exited prematurely with {}: {}".format( + p.returncode, stdout + ) + ) + # let the application initialize + self.loop.run_until_complete(asyncio.sleep(1)) + try: + self._handle_editor(winid) + break + except KeyError: + winid = None + if winid is None: + self.fail("Timeout waiting for editor window") + + self.loop.run_until_complete(p.communicate()) + (test_txt_content, _) = self.loop.run_until_complete( + self.testvm1.run_for_stdio("cat /home/user/test.txt") + ) + # Drop BOM if added by editor + if test_txt_content.startswith(b"\xef\xbb\xbf"): + test_txt_content = test_txt_content[3:] + self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") + + def _get_open_script(self, application): + """Generate a script to instruct *application* to open *filename*""" + if application == "org.gnome.Nautilus": + return ( + "#!/usr/bin/python3\n" + "import sys, os" + "from dogtail import tree, config\n" + "config.config.actionDelay = 1.0\n" + "config.config.defaultDelay = 1.0\n" + "config.config.searchCutoffCount = 10\n" + "app = tree.root.application('org.gnome.Nautilus')\n" + "app.child(os.path.basename(sys.argv[1])).doubleClick()\n" + ).encode() + if application in ( + "mozilla-thunderbird", + "thunderbird", + "org.mozilla.thunderbird", + "net.thunderbird.Thunderbird", + ): + with open( + "/usr/share/qubes/tests-data/" + "dispvm-open-thunderbird-attachment", + "rb", + ) as file: + return file.read() + assert False + + def _get_apps_list(self, template): + try: + # get first user in the qubes group + qubes_grp = grp.getgrnam("qubes") + qubes_user = pwd.getpwnam(qubes_grp.gr_mem[0]) + except KeyError: + self.skipTest("Cannot find a user in the qubes group") + + desktop_list = os.listdir( + os.path.join( + qubes_user.pw_dir, + f".local/share/qubes-appmenus/{template}/apps.templates", + ) + ) + return [ + l[: -len(".desktop")] + for l in desktop_list + if l.endswith(".desktop") + ] + + @unittest.skipUnless(which("xdotool"), "xdotool not installed") + def test_100_open_in_dispvm(self): + if "whonix-w" in self.template: + self.skipTest( + "whonix workstation does not have default mail client" + ) + self.testvm1 = self.app.add_new_vm( + qubes.vm.appvm.AppVM, + name=self.make_vm_name("vm1"), + label="red", + template=self.app.domains[self.template], + ) + self.loop.run_until_complete(self.testvm1.create_on_disk()) + self.app.save() + + app_id = "mozilla-thunderbird" + if "debian" in self.template or "whonix" in self.template: + app_id = "thunderbird" + # F40+ has org.mozilla.thunderbird + if "org.mozilla.thunderbird" in self._get_apps_list(self.template): + app_id = "org.mozilla.thunderbird" + # F41+ has net.thunderbird.Thunderbird + if "net.thunderbird.Thunderbird" in self._get_apps_list(self.template): + app_id = "net.thunderbird.Thunderbird" + + self.testvm1.features["service.app-dispvm." + app_id] = "1" + self.loop.run_until_complete(self.testvm1.start()) + self.loop.run_until_complete(self.wait_for_session(self.testvm1)) + self.loop.run_until_complete( + self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt") + ) + + self.loop.run_until_complete( + self.testvm1.run_for_stdio( + "cat > /home/user/open-file", + input=self._get_open_script(app_id), + ) + ) + self.loop.run_until_complete( + self.testvm1.run_for_stdio("chmod +x /home/user/open-file") + ) + + # disable donation message as it messes with editor detection + self.loop.run_until_complete( + self.testvm1.run_for_stdio( + "cat > /etc/thunderbird/pref/test.js", + input=b'pref("app.donation.eoy.version.viewed", 100);\n', + user="root", + ) + ) + + self.loop.run_until_complete( + self.testvm1.run_for_stdio( + "gsettings set org.gnome.desktop.interface " + "toolkit-accessibility true" + ) + ) + + app = self.loop.run_until_complete( + self.testvm1.run_service("qubes.StartApp+" + app_id) + ) + # give application a bit of time to start + self.loop.run_until_complete(asyncio.sleep(3)) + + try: + self.loop.run_until_complete( + self.testvm1.run_for_stdio( + "./open-file test.txt", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + ) + except subprocess.CalledProcessError as err: + with contextlib.suppress(asyncio.TimeoutError): + self.loop.run_until_complete(asyncio.wait_for(app.wait(), 30)) + if app.returncode == 127: + self.skipTest("{} not installed".format(app_id)) + self.fail( + "'./open-file test.txt' failed with {}: {}{}".format( + err.returncode, err.stdout, err.stderr + ) + ) if "whonix-workstation" in self.template: - dvm_confirm_rslt = self._whonix_ws_dispvm_confirm("edit file") + dvm_confirm_rslt = self._whonix_ws_dispvm_confirm("DispVM open") if not dvm_confirm_rslt[0]: self.fail(dvm_confirm_rslt[1]) # if first 5 windows isn't expected editor, there is no hope winid = None for _ in range(5): - try: - winid = self.wait_for_window( - "disp[0-9]*", - search_class=True, - include_tray=False, - timeout=60, - ) - except Exception as e: # pylint: disable=broad-exception-caught - try: - self.loop.run_until_complete(asyncio.wait_for(p.wait(), 1)) - except asyncio.TimeoutError: - raise e - stdout = self.loop.run_until_complete(p.stdout.read()) - self.fail( - "qvm-open-in-dvm exited prematurely with {}: {}".format( - p.returncode, stdout - ) - ) + winid = self.wait_for_window( + "disp[0-9]*", search_class=True, include_tray=False, timeout=60 + ) # let the application initialize self.loop.run_until_complete(asyncio.sleep(1)) try: - self._handle_editor(winid) + # copy, not modify - attachment is set as read-only + self._handle_editor(winid, copy=True) break except KeyError: winid = None if winid is None: self.fail("Timeout waiting for editor window") - self.loop.run_until_complete(p.communicate()) - (test_txt_content, _) = self.loop.run_until_complete( - self.testvm1.run_for_stdio("cat /home/user/test.txt") + self.loop.run_until_complete( + self.wait_for_window_hide_coro("editor", winid) ) - # Drop BOM if added by editor - if test_txt_content.startswith(b"\xef\xbb\xbf"): - test_txt_content = test_txt_content[3:] - self.assertEqual(test_txt_content, b"Test test 2\ntest1\n") - def _get_open_script(self, application): - """Generate a script to instruct *application* to open *filename*""" - if application == "org.gnome.Nautilus": - return ( - "#!/usr/bin/python3\n" - "import sys, os" - "from dogtail import tree, config\n" - "config.config.actionDelay = 1.0\n" - "config.config.defaultDelay = 1.0\n" - "config.config.searchCutoffCount = 10\n" - "app = tree.root.application('org.gnome.Nautilus')\n" - "app.child(os.path.basename(sys.argv[1])).doubleClick()\n" - ).encode() - if application in ( - "mozilla-thunderbird", - "thunderbird", - "org.mozilla.thunderbird", - "net.thunderbird.Thunderbird", - ): - with open( - "/usr/share/qubes/tests-data/" - "dispvm-open-thunderbird-attachment", - "rb", - ) as file: - return file.read() - assert False + with open("/var/run/qubes/qubes-clipboard.bin", "rb") as file: + test_txt_content = file.read() + self.assertEqual(test_txt_content.strip(), b"test1") + + # this doesn't really close the application, only the qrexec-client + # process that started it; but clean it up anyway to not leak processes + app.terminate() + self.loop.run_until_complete(app.wait()) + + +class TC_21_DispVM_Preload(DispVMHelpersMixin, qubes.tests.SystemTestCase): + """ + Template-independent DisposableVM preload tests. + + These tests do not depend on the template OS and previously ran once + per template (Debian/Fedora/Whonix), unnecessarily slowing down the + integration test suite. They execute only once on the default template. + """ + + def setUp(self): # pylint: disable=invalid-name + logger.info("start") + super().setUp() + self.init_default_template() + self.template = self.app.default_template + self.setup_dispvm_nodes() + logger.info("end") + + def test_011_preload_reject_max(self): + """Test preloading when max has been reached""" + self.loop.run_until_complete( + qubes.vm.dispvm.DispVM.from_appvm(self.disp_base, preload=True) + ) + self.assertEqual(0, len(self.disp_base.get_feat_preload())) + + def test_012_preload_low_mem(self): + """Test preloading with low memory""" + self.loop.run_until_complete(self._test_012_preload_low_mem()) + + async def _test_012_preload_low_mem(self): + # pylint: disable=unspecified-encoding + logger.info("start") + unpatched_open = open + memory = int(getattr(self.disp_base, "memory", 0) * 1024**2) + + def mock_open_mem(file, *args, **kwargs): + if file == qubes.config.qmemman_avail_mem_file: + return mock_open(read_data=str(memory))() + return unpatched_open(file, *args, **kwargs) + + def mock_open_mem_threshold(file, *args, **kwargs): + if file == qubes.config.qmemman_avail_mem_file: + return mock_open(read_data=str(memory * 2))() + return unpatched_open(file, *args, **kwargs) + + preload_max = 2 + with patch("builtins.open", side_effect=mock_open_mem): + logger.info("low mem standard") + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload( + preload_max, fail_on_timeout=False, timeout=15 + ) + self.assertEqual(1, len(self.disp_base.get_feat_preload())) + # Nothing will be done here, just to prepare to the next test. + self.disp_base.features["preload-dispvm-max"] = str(preload_max - 1) + + with patch("builtins.open", side_effect=mock_open_mem_threshold): + logger.info("low mem threshold") + self.adminvm.features["preload-dispvm-threshold"] = memory + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload( + preload_max, fail_on_timeout=False, timeout=15 + ) + self.assertEqual(1, len(self.disp_base.get_feat_preload())) + + logger.info("end") + + def test_015_preload_race_more(self): + """Test race requesting multiple preloaded qubes""" + self.loop.run_until_complete(self._test_015_preload_race_more()) + + async def _test_015_preload_race_more(self): + logger.info("start") + preload_max = 3 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) + old_preload = self.disp_base.get_feat_preload() + tasks = [self.run_preload_proc() for _ in range(preload_max)] + targets = await asyncio.gather(*tasks) + await self.wait_preload(preload_max) + preload_dispvm = self.disp_base.get_feat_preload() + self.assertTrue(set(old_preload).isdisjoint(preload_dispvm)) + self.assertEqual(len(targets), preload_max) + self.assertEqual(len(targets), len(set(targets))) + logger.info("end") + + def test_016_preload_race_less(self): + """Test race requesting preloaded qube while the maximum is zeroed.""" + self.loop.run_until_complete(self._test_016_preload_race_less()) + + async def _test_016_preload_race_less(self): + logger.info("start") + preload_max = 1 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max, wait_completion=False) + tasks = [self.run_preload_proc(), self.no_preload()] + target = await asyncio.gather(*tasks) + target_dispvm = target[0] + self.assertTrue(target_dispvm.startswith("disp")) + logger.info("end") + + def test_017_preload_autostart(self): + """The script triggers the API call 'admin.vm.CreateDisposable+preload' + which is responsible for bootstrapping.""" + logger.info("start") + self.app.default_dispvm = self.disp_base + + logger.info("must not change as max is 0") + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") + ) + self.loop.run_until_complete( + asyncio.wait_for(proc.communicate(), timeout=10) + ) + self.assertEqual(self.disp_base.get_feat_preload(), []) + + preload_max = 1 + logger.info("must not change existing preloaded disposables") + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + self.loop.run_until_complete(self.wait_preload(preload_max)) + old_preload = self.disp_base.get_feat_preload() + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") + ) + self.loop.run_until_complete(asyncio.wait_for(proc.wait(), timeout=10)) + preload_dispvm = self.disp_base.get_feat_preload() + self.assertEqual( + old_preload, + preload_dispvm, + msg=f"old_preload={old_preload} preload_dispvm={preload_dispvm}", + ) + + preload_max += 1 + logger.info("global refill must work") + self.adminvm.features["preload-dispvm-max"] = str(preload_max) + self.loop.run_until_complete(self.wait_preload(preload_max)) + old_old_preload = self.disp_base.get_feat_preload() + self.loop.run_until_complete( + self.cleanup_preload_run(self.disp_base, down_to=preload_max - 1) + ) + old_preload = self.disp_base.get_feat_preload() + self.assertEqual(len(old_preload), preload_max - 1) + self.assertIn(old_preload[0], old_old_preload) + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec("/usr/lib/qubes/preload-dispvm") + ) + try: + self.loop.run_until_complete( + asyncio.wait_for(proc.wait(), timeout=40) + ) + except asyncio.TimeoutError: + debug_preload = self.disp_base.get_feat_preload() + for qube in debug_preload: + qube = self.app.domains[qube] + if qube.is_paused(): + self.loop.run_until_complete( + asyncio.wait_for(qube.unpause(), timeout=5) + ) + self._run_cmd_and_log_output( + qube, "systemtl --user is-system-running", user="user" + ) + self._run_cmd_and_log_output( + qube, "systemtl is-system-running", user="root" + ) + self._run_cmd_and_log_output( + qube, "systemd-analyze --no-pager --user blame", user="user" + ) + self._run_cmd_and_log_output( + qube, "systemd-analyze --no-pager blame", user="root" + ) + self._run_cmd_and_log_output( + qube, "journalctl --no-pager --user", user="user" + ) + self._run_cmd_and_log_output( + qube, "journalctl --no-pager", user="root" + ) + raise + preload_dispvm = self.disp_base.get_feat_preload() + self.assertIn(old_preload[0], preload_dispvm) + self.assertEqual(len(preload_dispvm), preload_max) + + self.app.default_dispvm = None + logger.info("end") + + def test_018_preload_global(self): + """Tweak global preload setting and global dispvm.""" + self.loop.run_until_complete(self._test_018_preload_global()) + + async def _test_018_preload_global(self): + logger.info("start") + self.log_preload() + preload_max = 1 + + logger.info("set global dispvm") + self.app.default_dispvm = self.disp_base + logger.info("set global feat, state must change") + self.adminvm.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) + + self.log_preload() + logger.info("set local feat, state must not change") + preload_max += 1 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max, fail_on_timeout=False, timeout=15) + self.assertEqual(len(self.disp_base.get_feat_preload()), 1) + + self.log_preload() + logger.info("del local feat, state must not change") + del self.disp_base.features["preload-dispvm-max"] + await asyncio.sleep(5) + self.assertEqual(len(self.disp_base.get_feat_preload()), 1) - def _get_apps_list(self, template): - try: - # get first user in the qubes group - qubes_grp = grp.getgrnam("qubes") - qubes_user = pwd.getpwnam(qubes_grp.gr_mem[0]) - except KeyError: - self.skipTest("Cannot find a user in the qubes group") + self.log_preload() + logger.info("set local feat and del global feat, state must change") + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + del self.adminvm.features["preload-dispvm-max"] + await self.wait_preload(preload_max) - desktop_list = os.listdir( - os.path.join( - qubes_user.pw_dir, - f".local/share/qubes-appmenus/{template}/apps.templates", - ) - ) - return [ - l[: -len(".desktop")] - for l in desktop_list - if l.endswith(".desktop") - ] + self.log_preload() + logger.info("del local feat and set global feat, state must change") + preload_max -= 1 + preload_remove = self.app.default_dispvm.get_feat_preload() + self.disp_base.features["preload-dispvm-max"] = "" + self.wait_for_dispvm_destroy(preload_remove) + self.adminvm.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) + self.assertEqual(len(self.disp_base.get_feat_preload()), preload_max) - @unittest.skipUnless(which("xdotool"), "xdotool not installed") - def test_100_open_in_dispvm(self): - if "whonix-w" in self.template: - self.skipTest( - "whonix workstation does not have default mail client" - ) - self.testvm1 = self.app.add_new_vm( - qubes.vm.appvm.AppVM, - name=self.make_vm_name("vm1"), - label="red", - template=self.app.domains[self.template], - ) - self.loop.run_until_complete(self.testvm1.create_on_disk()) - self.app.save() + self.log_preload() + logger.info("switch global dispvm, state must change") + self.app.default_dispvm = self.disp_base_alt + await self.wait_preload(preload_max, appvm=self.disp_base_alt) - app_id = "mozilla-thunderbird" - if "debian" in self.template or "whonix" in self.template: - app_id = "thunderbird" - # F40+ has org.mozilla.thunderbird - if "org.mozilla.thunderbird" in self._get_apps_list(self.template): - app_id = "org.mozilla.thunderbird" - # F41+ has net.thunderbird.Thunderbird - if "net.thunderbird.Thunderbird" in self._get_apps_list(self.template): - app_id = "net.thunderbird.Thunderbird" + self.log_preload() + logger.info("set local feat, state must not change") + preload_max += 1 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) - self.testvm1.features["service.app-dispvm." + app_id] = "1" - self.loop.run_until_complete(self.testvm1.start()) - self.loop.run_until_complete(self.wait_for_session(self.testvm1)) - self.loop.run_until_complete( - self.testvm1.run_for_stdio("echo test1 > /home/user/test.txt") - ) + self.log_preload() + logger.info("switch back global dispvm, state must change") + preload_remove = self.app.default_dispvm.get_feat_preload() + self.app.default_dispvm = self.disp_base + self.wait_for_dispvm_destroy(preload_remove) + await self.wait_preload(preload_max) - self.loop.run_until_complete( - self.testvm1.run_for_stdio( - "cat > /home/user/open-file", - input=self._get_open_script(app_id), - ) - ) - self.loop.run_until_complete( - self.testvm1.run_for_stdio("chmod +x /home/user/open-file") - ) + self.log_preload() + logger.info("unset global dispvm, state must change") + preload_remove = self.app.default_dispvm.get_feat_preload() + self.app.default_dispvm = None + self.wait_for_dispvm_destroy(preload_remove) - # disable donation message as it messes with editor detection - self.loop.run_until_complete( - self.testvm1.run_for_stdio( - "cat > /etc/thunderbird/pref/test.js", - input=b'pref("app.donation.eoy.version.viewed", 100);\n', - user="root", - ) - ) + self.log_preload() + logger.info("end") + def test_019_preload_discard_outdated_volumes(self): + """Discard preload if volumes are outdated compared to its templates.""" self.loop.run_until_complete( - self.testvm1.run_for_stdio( - "gsettings set org.gnome.desktop.interface " - "toolkit-accessibility true" - ) + self._test_019_preload_discard_outdated_volumes() ) - app = self.loop.run_until_complete( - self.testvm1.run_service("qubes.StartApp+" + app_id) - ) - # give application a bit of time to start - self.loop.run_until_complete(asyncio.sleep(3)) + async def _test_019_preload_discard_outdated_volumes(self): + logger.info("start") + self.log_preload() + preload_max = 1 - try: - self.loop.run_until_complete( - self.testvm1.run_for_stdio( - "./open-file test.txt", - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + for qube in [self.disp_base, self.disp_base.template]: + logger.info( + "discard because of outdated volume originating from %s", + qube.name, ) - except subprocess.CalledProcessError as err: - with contextlib.suppress(asyncio.TimeoutError): - self.loop.run_until_complete(asyncio.wait_for(app.wait(), 30)) - if app.returncode == 127: - self.skipTest("{} not installed".format(app_id)) - self.fail( - "'./open-file test.txt' failed with {}: {}{}".format( - err.returncode, err.stdout, err.stderr - ) + await self.wait_preload(preload_max) + old_preload = self.disp_base.get_feat_preload() + await qube.start() + # If services are still starting, it may delay shutdown longer than + # the default timeout. Because we can't just kill default + # templates, wait gracefully for system services to have started. + await qube.run_service_for_stdio("qubes.WaitForRunningSystem") + logger.info("shutdown '%s'", qube.name) + await qube.shutdown(wait=True) + await self.wait_preload(preload_max) + preload_dispvm = self.disp_base.get_feat_preload() + self.assertTrue( + set(old_preload).isdisjoint(preload_dispvm), + f"old_preload={old_preload} preload_dispvm={preload_dispvm}", ) - if "whonix-workstation" in self.template: - dvm_confirm_rslt = self._whonix_ws_dispvm_confirm("DispVM open") - if not dvm_confirm_rslt[0]: - self.fail(dvm_confirm_rslt[1]) - - # if first 5 windows isn't expected editor, there is no hope - winid = None - for _ in range(5): - winid = self.wait_for_window( - "disp[0-9]*", search_class=True, include_tray=False, timeout=60 - ) - # let the application initialize - self.loop.run_until_complete(asyncio.sleep(1)) - try: - # copy, not modify - attachment is set as read-only - self._handle_editor(winid, copy=True) - break - except KeyError: - winid = None - if winid is None: - self.fail("Timeout waiting for editor window") + self.log_preload() + logger.info("end") + def test_020_preload_discard_outdated_volume_size(self): + """Discard preload if private size differs with disposable template.""" self.loop.run_until_complete( - self.wait_for_window_hide_coro("editor", winid) + self._test_020_preload_discard_outdated_volume_size() ) - with open("/var/run/qubes/qubes-clipboard.bin", "rb") as file: - test_txt_content = file.read() - self.assertEqual(test_txt_content.strip(), b"test1") + async def _test_020_preload_discard_outdated_volume_size(self): + logger.info("start") + self.log_preload() + preload_max = 1 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) + preload_dispvm = self.disp_base.get_feat_preload() + old_size = self.disp_base.volume_config["private"]["size"] + size = int(old_size) + 512 + await self.disp_base.storage.resize("private", size) + self.app.save() + dispvm = await asyncio.wait_for( + qubes.vm.dispvm.DispVM.from_appvm(self.disp_base), 30 + ) + self.assertNotIn(dispvm.name, preload_dispvm) + await dispvm.cleanup() + logger.info("end") - # this doesn't really close the application, only the qrexec-client - # process that started it; but clean it up anyway to not leak processes - app.terminate() - self.loop.run_until_complete(app.wait()) + def test_021_preload_discard_outdated_setting(self): + """Discard preload if properties differ with the disposable template.""" + self.loop.run_until_complete( + self._test_021_preload_discard_outdated_setting() + ) + + async def _test_021_preload_discard_outdated_setting(self): + logger.info("start") + self.log_preload() + preload_max = 1 + self.disp_base.features["preload-dispvm-max"] = str(preload_max) + await self.wait_preload(preload_max) + preload_dispvm = self.disp_base.get_feat_preload() + self.disp_base.netvm = None + dispvm = await asyncio.wait_for( + qubes.vm.dispvm.DispVM.from_appvm(self.disp_base), 30 + ) + self.assertNotIn(dispvm.name, preload_dispvm) + await dispvm.cleanup() + await self.wait_preload(preload_max) + logger.info("end") def create_testcases_for_templates():