From 2954f3ebbea954d8480f273b14e841edb92b846b Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Fri, 2 May 2025 12:52:33 +0200 Subject: [PATCH 01/13] vmupdate: dom0 non-iteractive updater introduce agent_type: for local (dom0), remote (template vms etc.) and proxy (update vm) actions. `qubes-vm-update` uses `qubes-dom0-update --just-print-progress` for non-interactive update, only small subset of action are supported. --- dom0-updates/qubes-dom0-update | 73 ++++++-- vmupdate/agent/entrypoint.py | 160 +++++++++++++----- vmupdate/agent/source/apt/apt_api.py | 7 +- vmupdate/agent/source/apt/apt_cli.py | 10 +- vmupdate/agent/source/args.py | 3 + .../agent/source/common/package_manager.py | 20 ++- vmupdate/agent/source/dnf/dnf5_api.py | 47 +++-- vmupdate/agent/source/dnf/dnf_api.py | 112 +++++++++--- vmupdate/agent/source/dnf/dnf_cli.py | 55 ++++-- vmupdate/agent/source/status.py | 10 ++ vmupdate/agent/source/utils.py | 4 + vmupdate/qube_connection.py | 43 ++--- vmupdate/update_manager.py | 119 +++++++++++-- vmupdate/vmupdate.py | 81 +++++++-- 14 files changed, 575 insertions(+), 169 deletions(-) diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index 5b7d177..284bf91 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -28,6 +28,7 @@ if [ "$1" = "--help" ]; then echo " --force-xen-upgrade force major Xen upgrade even if some qubes are running" echo " --console does nothing; ignored for backward compatibility" echo " --show-output does nothing; ignored for backward compatibility" + echo " --quiet do not print anything to stdout" echo " --preserve-terminal does nothing; ignored for backward compatibility" echo " --skip-boot-check does not check if /boot & /boot/efi should be mounted" echo " --switch-audio-server-to=(pulseaudio|pipewire) switch audio daemon to pipewire or pulseaudio" @@ -46,6 +47,7 @@ YUM_OPTS=() UPDATEVM_OPTS=() QVMTEMPLATE_OPTS=() GUI= +PROGRESS_REPORTING= CHECK_ONLY= CLEAN= TEMPLATE= @@ -79,6 +81,12 @@ while [ $# -gt 0 ]; do --show-output) # ignore ;; + --quiet) + exec > /dev/null + ;; + --just-print-progress) + PROGRESS_REPORTING=1 + ;; --check-only) CHECK_ONLY=1 UPDATEVM_OPTS+=( "$1" ) @@ -310,27 +318,53 @@ qvm-run --nogui -q -- "$UPDATEVM" "rm -rf -- '$dom0_updates_dir/etc' '$dom0_upda exit "$status" } -CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui" +QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io) + +if [ "$PROGRESS_REPORTING" == "1" ]; then + CMD="/usr/lib/qubes/qubes-download-dom0-updates-init.sh" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" -# We avoid using bash’s own facilities for this, as they produce $'\n'-style -# strings in certain cases. These are not portable, whereas the string produced -# by the following is. -for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done + update_agent_log="/var/log/qubes/qubes-update" + qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$update_agent_log'" || exit 1 -QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io) -if [[ -t 1 ]] && [[ -t 2 ]]; then - # Use ‘script’ to emulate a TTY, so that we get status bars and other - # progress output. Since stdout and stderr are both terminals, qvm-run - # will automatically sanitize them, but we explicitly tell it to anyway - # as a precaution. - # - # We MUST NOT use ‘exec script’ here. That causes ‘script’ to - # inherit the child processes of the shell. ‘script’ mishandles - # this and enters an infinite loop. - CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" -fi + # "--no-cleanup" is needed since fakeroot cannot remove entrypoint + qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=DEBUG + RETCODE=$? + if [ "$RETCODE" -eq 100 ]; then + echo "$(hostname) done no_updates" >&2 + exit 100 + fi + if [ "$RETCODE" -ne 0 ]; then + echo "$(hostname) done error" >&2 + exit $RETCODE + fi + + # qubes-vm-update leaves the downloaded packages with root ownership + qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$dom0_updates_dir'" || exit 1 -qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null + CMD="/usr/lib/qubes/qubes-download-dom0-updates-finish.sh" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" +else + CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui" + + # We avoid using bash’s own facilities for this, as they produce $'\n'-style + # strings in certain cases. These are not portable, whereas the string produced + # by the following is. + for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done + + if [[ -t 1 ]] && [[ -t 2 ]]; then + # Use ‘script’ to emulate a TTY, so that we get status bars and other + # progress output. Since stdout and stderr are both terminals, qvm-run + # will automatically sanitize them, but we explicitly tell it to anyway + # as a precaution. + # + # We MUST NOT use ‘exec script’ here. That causes ‘script’ to + # inherit the child processes of the shell. ‘script’ mishandles + # this and enters an infinite loop. + CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" + fi + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null +fi RETCODE=$? if [[ "$REMOTE_ONLY" = '1' ]] || [ "$RETCODE" -ne 0 ]; then @@ -401,6 +435,9 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then # refresh packagekit metadata, GUI utilities use it pkcon refresh force $guiapp + elif [ "$PROGRESS_REPORTING" == 1 ]; then + # report progress to the user + qubes-vm-update --no-refresh --targets dom0 --force-update --log=DEBUG --just-print-progress --show-output else dnf check-update || if [ $? -eq 100 ]; then # Run dnf with options diff --git a/vmupdate/agent/entrypoint.py b/vmupdate/agent/entrypoint.py index 79e7a40..2a3486b 100755 --- a/vmupdate/agent/entrypoint.py +++ b/vmupdate/agent/entrypoint.py @@ -8,6 +8,7 @@ from source.utils import get_os_data from source.log_config import init_logs from source.common.exit_codes import EXIT +from source.common.package_manager import AgentType def main(args=None): @@ -21,18 +22,31 @@ def main(args=None): os_data = get_os_data() log.debug("Selecting package manager.") + agent_type = AgentType.VM + if os_data["id"] == "qubes": + agent_type = AgentType.DOM0 + if args.download_only: + agent_type = AgentType.UPDATE_VM pkg_mng = get_package_manager( - os_data, log, log_handler, log_level, args.no_progress) + os_data, log, log_handler, log_level, agent_type, args.no_progress) log.debug("Running upgrades.") return_code = pkg_mng.upgrade(refresh=not args.no_refresh, hard_fail=not args.force_upgrade, remove_obsolete=not args.leave_obsolete, - print_streams=args.show_output + print_streams=args.show_output, ) - log.debug("Notify dom0 about upgrades.") - os.system("/usr/lib/qubes/upgrades-status-notify") + if not pkg_mng.PROGRESS_REPORTING and not args.no_progress: + # even if progress reporting is unavailable we want info that update finished + if agent_type is AgentType.UPDATE_VM: + print(f"{55:.2f}", flush=True, file=sys.stderr) + else: + print(f"{100:.2f}", flush=True, file=sys.stderr) + + if agent_type is AgentType.VM: + log.debug("Notify dom0 about upgrades.") + os.system("/usr/lib/qubes/upgrades-status-notify") if not args.no_cleanup: return_code = max(pkg_mng.clean(), return_code) @@ -49,7 +63,7 @@ def parse_args(args): return args -def get_package_manager(os_data, log, log_handler, log_level, no_progress): +def get_package_manager(os_data, log, log_handler, log_level, agent_type, no_progress): """ Returns instance of `PackageManager`. @@ -59,57 +73,113 @@ def get_package_manager(os_data, log, log_handler, log_level, no_progress): requirements = {} # plugins MUST be applied before import anything from package managers. # in case of apt configuration is loaded on `import apt`. - for plugin in plugins.entrypoints: - plugin(os_data, log, requirements=requirements) - - if os_data["os_family"] == "Debian": - try: - from source.apt.apt_api import APT as PackageManager - except ImportError: - log.warning("Failed to load apt with progress bar. Using apt cli.") - # no progress reporting - no_progress = True - print(f"Progress reporting not supported.", flush=True) - - if no_progress: - from source.apt.apt_cli import APTCLI as PackageManager - elif os_data["os_family"] == "RedHat": - try: - version = int(os_data["release"].split(".")[0]) - except ValueError: - version = 99 # fedora changed its version - - loaded = False - if version >= 41: - try: - from source.dnf.dnf5_api import DNF as PackageManager - loaded = True - except ImportError: - log.warning("Failed to load dnf5.") - - if not loaded: - try: - from source.dnf.dnf_api import DNF as PackageManager - loaded = True - except ImportError: - log.warning( - "Failed to load dnf with progress bar. Using dnf cli.") - print(f"Progress reporting not supported.", flush=True) - - if no_progress or not loaded: - from source.dnf.dnf_cli import DNFCLI as PackageManager + if agent_type is not AgentType.UPDATE_VM: + for plugin in plugins.entrypoints: + plugin(os_data, log, requirements=requirements) + + if os_data["os_family"] == "RedHat" or agent_type is AgentType.UPDATE_VM: + PackageManager = import_rhel_package_manager(os_data, log, no_progress) + elif os_data["os_family"] == "Debian": + PackageManager = import_debian_package_manager(log, no_progress) elif os_data["os_family"] == "ArchLinux": from source.pacman.pacman_cli import PACMANCLI as PackageManager print(f"Progress reporting not supported.", flush=True) + elif os_data["os_family"] == "Qubes": + PackageManager = import_dom0_package_manager(os_data, log, no_progress) else: raise NotImplementedError( "Only Debian, RedHat and ArchLinux based OS is supported.") - pkg_mng = PackageManager(log_handler, log_level) + pkg_mng = PackageManager(log_handler, log_level, agent_type) pkg_mng.requirements = requirements return pkg_mng +def import_rhel_package_manager(os_data, log, no_progress): + """ + Import dnf package manager. + """ + dnf5_fedora_version = 41 + if os_data["os_family"] == "RedHat": + try: + version = int(os_data["release"].split(".")[0]) + except ValueError: + version = 99 # fedora changed its version + else: + version = dnf5_fedora_version # try to use whatever is available, starting from dnf5 + + loaded = False + if version >= dnf5_fedora_version: + try: + from source.dnf.dnf5_api import DNF5 as PackageManager + loaded = True + except ImportError: + log.warning("Failed to load dnf5.") + + if not loaded: + try: + from source.dnf.dnf_api import DNF as PackageManager + loaded = True + log.debug("Using dnf python API for progress reporting.") + except ImportError: + print(f"Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + log.warning( + "Failed to load dnf with progress bar. Using dnf cli.") + from source.dnf.dnf_cli import DNFCLI as PackageManager + + return PackageManager + + +def import_debian_package_manager(log, no_progress): + """ + Import apt package manager. + """ + loaded = False + try: + from source.apt.apt_api import APT as PackageManager + loaded = True + except ImportError: + log.warning("Failed to load apt with progress bar. Using apt cli.") + print(f"Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + from source.apt.apt_cli import APTCLI as PackageManager + + return PackageManager + + +def import_dom0_package_manager(os_data, log, no_progress): + """ + Import dnf package manager for dom0. + """ + major, minor = os_data["release"].split(".") + major, minor = int(major), int(minor) + loaded = False + if major >= 5 or (major == 4 and minor >= 3): + try: + from source.dnf.dnf5_api import DNF5 as PackageManager + loaded = True + except ImportError: + log.warning("Failed to load dnf5.") + + if not loaded: + try: + from source.dnf.dnf_api import DNF as PackageManager + loaded = True + log.debug("Using dnf python API for progress reporting.") + except ImportError: + print(f"Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + log.warning( + "Failed to load dnf with progress bar. Using dnf cli.") + from source.dnf.dnf_cli import DNFCLI as PackageManager + + return PackageManager + + if __name__ == '__main__': try: sys.exit(main()) diff --git a/vmupdate/agent/source/apt/apt_api.py b/vmupdate/agent/source/apt/apt_api.py index 543ad31..5b52d8f 100644 --- a/vmupdate/agent/source/apt/apt_api.py +++ b/vmupdate/agent/source/apt/apt_api.py @@ -26,6 +26,7 @@ import apt.progress.base import apt_pkg +from source.common.package_manager import AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress @@ -34,8 +35,10 @@ class APT(APTCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) self.apt_cache = apt.Cache() update = FetchProgress( weight=4, log=self.log, refresh=True) # 4% of total time diff --git a/vmupdate/agent/source/apt/apt_cli.py b/vmupdate/agent/source/apt/apt_cli.py index 9779ad7..867f848 100644 --- a/vmupdate/agent/source/apt/apt_cli.py +++ b/vmupdate/agent/source/apt/apt_cli.py @@ -26,14 +26,18 @@ import contextlib from typing import List -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class APTCLI(PackageManager): - def __init__(self, log_handler, log_level,): - super().__init__(log_handler, log_level,) + PROGRESS_REPORTING = False + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + if self.type is AgentType.UPDATE_VM: + raise NotImplementedError("APT do not support update proxy VM.") self.package_manager: str = "apt-get" # to prevent a warning: `debconf: unable to initialize frontend: Dialog` diff --git a/vmupdate/agent/source/args.py b/vmupdate/agent/source/args.py index 5b0593c..611065e 100644 --- a/vmupdate/agent/source/args.py +++ b/vmupdate/agent/source/args.py @@ -41,6 +41,9 @@ class AgentArgs: ("--leave-obsolete",): { "action": 'store_true', "help": 'Do not remove updater and cache files from target qube'}, + ("--download-only",): { + "action": 'store_true', + "help": 'Only download packages'}, } EXCLUSIVE_OPTIONS_1 = { ("--show-output", "--verbose", "-v"): diff --git a/vmupdate/agent/source/common/package_manager.py b/vmupdate/agent/source/common/package_manager.py index 266a4ba..869b648 100644 --- a/vmupdate/agent/source/common/package_manager.py +++ b/vmupdate/agent/source/common/package_manager.py @@ -23,14 +23,21 @@ import logging import subprocess import sys +import enum from typing import Optional, Dict, List from .process_result import ProcessResult from .exit_codes import EXIT +class AgentType(enum.Enum): + VM = "Downloads and install updates in VM" + DOM0 = "Install downloaded updates in dom0" + UPDATE_VM = "Downloads updates for dom0" + + class PackageManager: """ main package manager class """ - def __init__(self, log_handler, log_level): + def __init__(self, log_handler, log_level, agent_type: AgentType): self.package_manager: Optional[str] = None self.log = logging.getLogger( f'vm-update.agent.{self.__class__.__name__}') @@ -38,13 +45,14 @@ def __init__(self, log_handler, log_level): self.log.addHandler(log_handler) self.log.propagate = False self.requirements: Optional[Dict[str, str]] = None + self.type = agent_type def upgrade( self, refresh: bool, hard_fail: bool, remove_obsolete: bool, - print_streams: bool = False + print_streams: bool = False, ): """ Upgrade packages using system package manager. @@ -105,9 +113,15 @@ def _upgrade( return result result_upgrade = self.upgrade_internal(remove_obsolete) - if result_upgrade: + if result_upgrade.code not in (EXIT.OK, EXIT.OK_NO_UPDATES): result_upgrade.code = EXIT.ERR_VM_UPDATE result += result_upgrade + if result: + return result + + if self.type == AgentType.UPDATE_VM: + # No package installation is required in UpdateVM, so changes are not checked. + return result new_pkg = self.get_packages() diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index d04040e..b2cf5de 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -25,11 +25,12 @@ import libdnf5 from libdnf5.repo import DownloadCallbacks from libdnf5.rpm import TransactionCallbacks -from libdnf5.base import Base, Goal +from libdnf5.base import Goal from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress +from source.common.package_manager import AgentType from .dnf_cli import DNFCLI @@ -38,11 +39,27 @@ class TransactionError(RuntimeError): pass -class DNF(DNFCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) - self.base = Base() +class DNF5(DNFCLI): + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + + self.base = libdnf5.base.Base() + conf = self.base.get_config() + + if self.type == AgentType.UPDATE_VM: + conf.config_file_path = self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" + conf.best = True + conf.plugins = False + conf.installroot = self.UPDATE_VM_INSTALLROOT + for opt in ('cachedir', 'logdir', 'persistdir'): + setattr(conf, opt, self.UPDATE_VM_INSTALLROOT + getattr(conf, opt)) + conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] + conf.excludepkgs = ["qubes-template-*"] self.base.load_config() + + # Create base object with the loaded config self.base.setup() self.config = self.base.get_config() update = FetchProgress(weight=0, log=self.log) # % of total time @@ -85,6 +102,8 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: try: self.log.debug("Performing package upgrade...") goal = Goal(self.base) + if self.type == AgentType.UPDATE_VM: + goal.set_allow_erasing(True) goal.add_upgrade("*") transaction = goal.resolve() # fill empty `Command line` column in dnf history @@ -93,7 +112,7 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: if transaction.get_transaction_packages_count() == 0: self.log.info("No packages to upgrade, quitting.") return ProcessResult( - EXIT.OK, out="", + EXIT.OK_NO_UPDATES, out="", err="\n".join(transaction.get_resolve_logs_as_strings())) self.base.set_download_callbacks( @@ -106,8 +125,7 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: raise TransactionError( f"GPG signatures check failed: {problems}") - if result.code == EXIT.OK: - print("Updating packages.", flush=True) + if result.code == EXIT.OK and self.type is not AgentType.UPDATE_VM: self.log.debug("Committing upgrade...") transaction.set_callbacks( libdnf5.rpm.TransactionCallbacksUniquePtr( @@ -117,9 +135,9 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: raise TransactionError( transaction.transaction_result_to_string(tnx_result)) self.log.debug("Package upgrade successful.") - self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) - print("Updated", flush=True) + if self.type is AgentType.VM: + self.log.info("Notifying dom0 about installed applications") + subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) except Exception as exc: self.log.error( "An error occurred while upgrading packages: %s", str(exc)) @@ -194,7 +212,10 @@ def end(self, user_cb_data: int, status: int, msg: str) -> int: :param msg: The error message in case of error. """ if status != 0: - print(msg, flush=True, file=self._stdout) + if isinstance(msg, bytes): + msg = msg.decode('ascii', errors='ignore') + if msg: + print(msg, flush=True, file=self._stdout) return DownloadCallbacks.end(self, user_cb_data, status, msg) def mirror_failure( @@ -208,6 +229,8 @@ def mirror_failure( :param url: Failed mirror URL. :param metadata: the type of metadata that is being downloaded """ + if isinstance(msg, bytes): + msg = msg.decode('ascii', errors='ignore') print(f"Fetching {metadata} failure " f"({self.package_names[user_cb_data]}) {msg}", flush=True, file=self._stdout) diff --git a/vmupdate/agent/source/dnf/dnf_api.py b/vmupdate/agent/source/dnf/dnf_api.py index 506090b..6b38f53 100644 --- a/vmupdate/agent/source/dnf/dnf_api.py +++ b/vmupdate/agent/source/dnf/dnf_api.py @@ -19,8 +19,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. +import os import subprocess import dnf +import dnf.conf +import dnf.rpm from dnf.yum.rpmtrans import TransactionDisplay from dnf.callback import DownloadProgress import dnf.transaction @@ -28,18 +31,56 @@ from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress +from source.common.package_manager import AgentType from .dnf_cli import DNFCLI class DNF(DNFCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) - self.base = dnf.Base() - self.base.conf.read() # load dnf.conf - update = FetchProgress(weight=0, log=self.log) # % of total time - fetch = FetchProgress(weight=50, log=self.log) # % of total time - upgrade = UpgradeProgress(weight=50, log=self.log) # % of total time + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + + if self.type == AgentType.UPDATE_VM: + dnfconf = "/var/lib/qubes/dom0-updates/etc/dnf/dnf.conf" + else: + dnfconf = None + conf = dnf.conf.Conf() + conf.read(filename=dnfconf) + + if self.type == AgentType.UPDATE_VM: + conf.best = True + conf.plugins = False + conf.installroot = self.UPDATE_VM_INSTALLROOT + for opt in ('cachedir', 'logdir', 'persistdir'): + conf.prepend_installroot(opt) + conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] + conf.excludepkgs = ["qubes-template-*"] + + # make sure log file exists + log_dir = self.UPDATE_VM_INSTALLROOT + "/var/log" + log_file = os.path.join(log_dir, "hawkey.log") + os.makedirs(log_dir, exist_ok=True) + if not os.path.exists(log_file): + with open(log_file, 'w'): + pass + + # Passing `conf` to `base` causes `releasever` not to be set + subst = conf.substitutions + if 'releasever' not in subst: + releasever = dnf.rpm.detect_releasever(conf.installroot) + subst['releasever'] = releasever + + self.base = dnf.Base(conf) + if self.type == AgentType.UPDATE_VM: + self.base._allow_erasing = True + # Repositories serve as sources of information about packages. + self.base.read_all_repos() + + update = FetchProgress(weight=10, log=self.log, refresh=True) # % of total time + fetch = FetchProgress(weight=45, log=self.log) # % of total time + upgrade = UpgradeProgress(weight=45, log=self.log) # % of total time self.progress = ProgressReporter(update, fetch, upgrade) def refresh(self, hard_fail: bool) -> ProcessResult: @@ -49,16 +90,18 @@ def refresh(self, hard_fail: bool) -> ProcessResult: :param hard_fail: raise error if some repo is unavailable :return: (exit_code, stdout, stderr) """ - self.base.conf.skip_if_unavailable = int(not hard_fail) - result = ProcessResult() + self.base.conf.skip_if_unavailable = True try: self.log.debug("Refreshing available packages...") - # Repositories serve as sources of information about packages. - self.base.read_all_repos() + repos = [r for r in self.base.repos.iter_enabled()] + # we do not know the size of the repositories + self.progress.update_progress.start(len(repos), len(repos)) + for i, repo in enumerate(repos): + self.progress.update_progress.progress(repo.id, 1) + repo.load() + self.progress.update_progress.end(repo.id, 0, "") updated = self.base.update_cache() - # A sack is needed for querying. - self.base.fill_sack() if updated: self.log.debug("Cache refresh successful.") else: @@ -68,7 +111,6 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.log.error( "An error occurred while refreshing packages: %s", str(exc)) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) - return result def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: @@ -80,6 +122,9 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: result = ProcessResult() try: self.log.debug("Performing package upgrade...") + # A sack is needed for querying. + self.base.fill_sack() + self.base.upgrade_all() # fill empty `Command line` column in dnf history @@ -89,7 +134,7 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: trans = self.base.transaction if not trans: self.log.info("No packages to upgrade, quitting.") - return ProcessResult(EXIT.OK, out="", err="") + return ProcessResult(EXIT.OK_NO_UPDATES, out="", err="") self.base.download_packages( trans.install_set, @@ -97,13 +142,14 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: ) result += sign_check(self.base, trans.install_set, self.log) - if result.code == EXIT.OK: + if result.code == EXIT.OK and self.type is not AgentType.UPDATE_VM: print("Updating packages.", flush=True) self.log.debug("Committing upgrade...") self.base.do_transaction(self.progress.upgrade_progress) self.log.debug("Package upgrade successful.") - self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) + if self.type is AgentType.VM: + self.log.info("Notifying dom0 about installed applications") + subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) print("Updated", flush=True) except Exception as exc: self.log.error( @@ -143,10 +189,11 @@ def sign_check(base, packages, log) -> ProcessResult: class FetchProgress(DownloadProgress, Progress): - def __init__(self, weight: int, log): + def __init__(self, weight: int, log, refresh: bool = False): Progress.__init__(self, weight, log) self.bytes_to_fetch = None self.bytes_fetched = 0 + self.action = "refresh" if refresh else "fetch" self.package_bytes = {} def end(self, payload, status, msg): @@ -155,12 +202,19 @@ def end(self, payload, status, msg): :api, `status` is a constant denoting the type of outcome, `err_msg` is an error message in case the outcome was an error. """ - print(f"{payload}: Fetched", flush=True) + if status != 0: + if isinstance(msg, bytes): + msg = msg.decode('ascii', errors='ignore') + if msg: + print(msg, flush=True, file=self._stdout) + else: + print(f"{payload}: {self.action.capitalize()}ed", flush=True) def message(self, msg): if isinstance(msg, bytes): msg = msg.decode('ascii', errors='ignore') - print(msg, flush=True, file=self._stdout) + if msg: + print(msg, flush=True, file=self._stdout) def progress(self, payload, done): """Update the progress display. :api @@ -181,12 +235,15 @@ def start(self, total_files, total_size, total_drpms=0): `total_size` total size of all files. """ - self.log.info("Fetch started.") + self.log.info(f"{self.action.capitalize()} started.") self.bytes_to_fetch = total_size - print(f"Fetching {total_files} packages " - f"[{self._format_bytes(self.bytes_to_fetch)}]", - flush=True) - self.package_bytes = {} + if self.action == "refresh": + print("Refreshing available packages.", flush=True) + else: + print(f"Fetching {total_files} packages " + f"[{self._format_bytes(self.bytes_to_fetch)}]", + flush=True) + self.package_bytes = {} self.notify_callback(0) @@ -207,11 +264,12 @@ def progress(self, _package, action, ti_done, ti_total, ts_done, :param ts_done: number of actions processed in the whole transaction :param ts_total: total number of actions in the whole transaction """ + self.log.info(_package) fetch = 6 install = 7 if action not in (fetch, install): return - percent = ti_done / ti_total * ts_done / ts_total * 100 + percent = (ti_done / ti_total + ts_done - 1) / ts_total * 100 self.notify_callback(percent) def scriptout(self, msgs): diff --git a/vmupdate/agent/source/dnf/dnf_cli.py b/vmupdate/agent/source/dnf/dnf_cli.py index 20cb0bc..a3db5a5 100644 --- a/vmupdate/agent/source/dnf/dnf_cli.py +++ b/vmupdate/agent/source/dnf/dnf_cli.py @@ -22,14 +22,17 @@ import shutil from typing import List -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class DNFCLI(PackageManager): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) + PROGRESS_REPORTING = False + UPDATE_VM_INSTALLROOT = "/var/lib/qubes/dom0-updates" + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) pck_mng_path = shutil.which('dnf') if pck_mng_path is not None: pck_mngr = 'dnf' @@ -55,12 +58,14 @@ def refresh(self, hard_fail: bool) -> ProcessResult: "check-update", "--assumeyes", f"--setopt=skip_if_unavailable={int(not hard_fail)}"] - result_check = self.run_cmd(cmd) - # ret_code == 100 is not an error - # It means there are packages to be updated - result_check.code = result_check.code if result_check.code != 100 else 0 - result += result_check - result.error_from_messages() + if self.type != AgentType.UPDATE_VM: + # In UpdateVM we use preconfigured repos + result_check = self.run_cmd(cmd) + # ret_code == 100 is not an error + # It means there are packages to be updated + result_check.code = result_check.code if result_check.code != 100 else 0 + result += result_check + result.error_from_messages() return result @@ -72,7 +77,11 @@ def expire_cache(self) -> ProcessResult: "-q", "clean", "expire-cache"] - result = self.run_cmd(cmd) + if self.type != AgentType.UPDATE_VM: + result = self.run_cmd(cmd) + else: + # In UpdateVM we use preconfigured repos + result = ProcessResult() return result def get_packages(self): @@ -102,10 +111,30 @@ def get_action(self, remove_obsolete) -> List[str]: """ Disable or enforce obsolete flag in dnf/yum. """ + result = ["-y"] + if self.type is AgentType.UPDATE_VM: + result.extend(["upgrade", + "--noplugins", + "--best", + "--allowerasing", + "--downloadonly", + "--installroot", self.UPDATE_VM_INSTALLROOT, + f"--setopt=cachedir={self.UPDATE_VM_INSTALLROOT}/var/cache/dnf", + f"--config={self.UPDATE_VM_INSTALLROOT}/etc/dnf/dnf.conf", + f"--setopt=reposdir={self.UPDATE_VM_INSTALLROOT}/etc/yum.repos.d", + "--exclude=qubes-template-*", "-y" + ]) + return result if remove_obsolete: - return ["-y", "--setopt=obsoletes=1", "upgrade"] - return ["-y", "--setopt=obsoletes=0", - "upgrade" if self.package_manager == "dnf" else "update"] + result.extend(["--setopt=obsoletes=1", "upgrade"]) + else: + result.append("--setopt=obsoletes=0") + if self.package_manager == "dnf": + result.append("upgrade") + else: + # yum + result.append("update") + return result def clean(self) -> int: """ diff --git a/vmupdate/agent/source/status.py b/vmupdate/agent/source/status.py index b302615..6c83016 100644 --- a/vmupdate/agent/source/status.py +++ b/vmupdate/agent/source/status.py @@ -59,3 +59,13 @@ def updating(qube, percent: float): @staticmethod def done(qube, status: FinalStatus): return StatusInfo(qube, Status.DONE, info=status) + + +class FormatedLine: + def __init__(self, qube_name, stream: str, message: str): + self.qname = qube_name + self.stream = stream + self.message = message + + def __str__(self): + return f"{self.qname}:{self.stream}: {self.message}" \ No newline at end of file diff --git a/vmupdate/agent/source/utils.py b/vmupdate/agent/source/utils.py index d486954..21678af 100644 --- a/vmupdate/agent/source/utils.py +++ b/vmupdate/agent/source/utils.py @@ -62,6 +62,10 @@ def get_os_data(logger: Optional = None) -> Dict[str, Any]: if 'rhel' in family or 'fedora' in family: data["os_family"] = 'RedHat' + if 'qubes' in family: + # We do not want to use 'RedHat' for dom0 since usually plugins do not apply to dom0 + data["os_family"] = 'Qubes' + if 'arch' in family: data["os_family"] = 'ArchLinux' diff --git a/vmupdate/qube_connection.py b/vmupdate/qube_connection.py index 9720130..91d18b4 100644 --- a/vmupdate/qube_connection.py +++ b/vmupdate/qube_connection.py @@ -21,6 +21,7 @@ import os import shutil import signal +import subprocess import tempfile import concurrent.futures from os.path import join @@ -29,8 +30,8 @@ import qubesadmin from vmupdate.agent.source.args import AgentArgs -from vmupdate.agent.source.log_config import LOGPATH, LOG_FILE -from vmupdate.agent.source.status import StatusInfo, FinalStatus +from vmupdate.agent.source.log_congfig import LOGPATH, LOG_FILE +from vmupdate.agent.source.status import StatusInfo, FinalStatus, FormatedLine from vmupdate.agent.source.common.process_result import ProcessResult @@ -157,14 +158,9 @@ def run_entrypoint( :param agent_args: args for agent entrypoint :return: return code and output of the script """ - # make sure entrypoint is executable - command = ['chmod', 'u+x', entrypoint_path] - result = self._run_shell_command_in_qube(self.qube, command) - - # run entrypoint command = [QubeConnection.PYTHON_PATH, entrypoint_path, *AgentArgs.to_cli_args(agent_args)] - result += self._run_shell_command_in_qube( + result = self._run_shell_command_in_qube( self.qube, command, show=self.show_progress) return result @@ -215,11 +211,21 @@ def _run_command_and_wait_for_output( def _run_command_and_actively_report_progress( self, target, command: List[str] ) -> ProcessResult: - proc = target.run_service( - 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(command), - user='root', - preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) - ) + if self.qube.klass == "AdminVM": + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + elif "--download-only" in command: + # for download-only commands, run with fakeroot + proc = target.run_service( + 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(["fakeroot"] + command), + user='user', + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + ) + else: + proc = target.run_service( + 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(command), + user='root', + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + ) self.logger.debug("Fetching agent process stdout/stderr.") with concurrent.futures.ThreadPoolExecutor() as executor: @@ -249,7 +255,7 @@ def _collect_stderr(self, proc) -> bytes: try: progress = float(line) except ValueError: - self._print('err', line) + self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) continue if progress == 100.: @@ -257,7 +263,7 @@ def _collect_stderr(self, proc) -> bytes: self.status_notifier.put( StatusInfo.updating(self.qube, progress)) else: - self._print('err', line) + self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) proc.stderr.close() self.logger.debug("Agent stderr closed.") @@ -270,12 +276,9 @@ def _collect_stdout(self, proc) -> bytes: line = ProcessResult.sanitize_output( untrusted_line, single=True) if line: - self._print('out', line) + self.status_notifier.put(FormatedLine(self.qube.name, 'out', line)) - proc.stderr.close() + proc.stdout.close() self.logger.debug("Agent stdout closed.") return b'' - - def _print(self, stream: str, line: str): - self.status_notifier.put(f"{self.qube.name}:{stream}: {line}") diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index 0e3aea0..a04a0a0 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -32,7 +32,7 @@ from tqdm import tqdm -from .agent.source.status import StatusInfo, FinalStatus, Status +from .agent.source.status import StatusInfo, FinalStatus, Status, FormatedLine from .qube_connection import QubeConnection from vmupdate.agent.source.log_config import init_logs from vmupdate.agent.source.common.process_result import ProcessResult @@ -51,6 +51,7 @@ def __init__(self, qubes, args, log): self.quiet = args.quiet self.no_progress = args.no_progress self.just_print_progress = args.just_print_progress + self.download_only = args.download_only self.buffered = not args.just_print_progress and not args.no_progress self.buffer = "" self.cleanup = not args.no_cleanup @@ -67,7 +68,7 @@ def run(self, agent_args): return EXIT.OK, {} show_progress = not self.quiet and not self.no_progress - SimpleTerminalBar.reinit_class() + SimpleTerminalBar.reinit_class(self.download_only) progress_output = SimpleTerminalBar \ if self.just_print_progress else tqdm progress_bar = MultipleUpdateMultipleProgressBar( @@ -78,13 +79,19 @@ def run(self, agent_args): ) for qube in self.qubes: - progress_bar.add_bar(qube.name) + disp_name = agent_args.display_name \ + if agent_args.display_name is not None else qube.name + progress_bar.add_bar(disp_name) progress_bar.pool.apply_async( update_qube, (qube, agent_args, show_progress, progress_bar.status_notifier, progress_bar.termination), callback=self.collect_result, error_callback=print ) + if qube.klass == "AdminVM": + # progress of AdminVM is continuation of different process, + # so we want to skip 0 value at beginning + progress_bar.progress_bars[qube.name].progress = None progress_bar.pool.close() progress_bar.feeding() @@ -122,9 +129,9 @@ def collect_result(self, result_tuple: Tuple[str, ProcessResult]): if self.show_output: for line in result.out.split('\n'): - self.print(qube_name + ":out:", line) + self.print(FormatedLine(qube_name, "out", line)) for line in result.err.split('\n'): - self.print(qube_name + ":err:", line) + self.print(FormatedLine(qube_name, "err", line)) elif not self.quiet and self.no_progress: self.print(result.out) @@ -146,6 +153,7 @@ def print(self): class SimpleTerminalBar: PARENT_MULTI_BAR = None + DOWNLOAD_ONLY = False def __init__(self, total, position, desc): assert position == len(SimpleTerminalBar.PARENT_MULTI_BAR.progresses) @@ -162,13 +170,19 @@ def __str__(self): FinalStatus.ERROR.value, FinalStatus.CANCELLED.value, FinalStatus.NO_UPDATES.value): + if SimpleTerminalBar.DOWNLOAD_ONLY: + return "" info = status.replace(" ", "_") status = "done" if status == Status.UPDATING.value: + if self.progress is None: + return "" info = self.progress return f"{name} {status} {info}" def update(self, progress): + if self.progress is None: + self.progress = 0 self.progress += progress SimpleTerminalBar.PARENT_MULTI_BAR.print() @@ -181,8 +195,9 @@ def close(self): pass @staticmethod - def reinit_class(): + def reinit_class(download_only = False): SimpleTerminalBar.PARENT_MULTI_BAR = TerminalMultiBar() + SimpleTerminalBar.DOWNLOAD_ONLY = download_only class MultipleUpdateMultipleProgressBar: @@ -291,18 +306,29 @@ def update_qube( :param termination: signal to gracefully terminate subprocess :return: """ + if agent_args.display_name is not None: + status_notifier = StatusNotifierWrapper(status_notifier, agent_args.display_name) + if termination.value: status_notifier.put(StatusInfo.done(qube, FinalStatus.CANCELLED)) return qube.name, ProcessResult(EXIT.SIGINT, "Canceled") - status_notifier.put(StatusInfo.updating(qube, 0)) try: - runner = UpdateAgentManager( - qube.app, - qube, - agent_args=agent_args, - show_progress=show_progress - ) + if qube.klass == "AdminVM": + # AdminVM update + runner = AdminVMAgentManager( + qube.app, + qube, + agent_args=agent_args, + show_progress=show_progress + ) + else: + runner = UpdateAgentManager( + qube.app, + qube, + agent_args=agent_args, + show_progress=show_progress + ) result = runner.run_agent( agent_args=agent_args, status_notifier=status_notifier, @@ -408,3 +434,70 @@ def _run_agent( self.log_handler.setFormatter(self.log_formatter) return result + +class StatusNotifierWrapper: + """ + Masks proxy VM with display name. + """ + def __init__(self, status_notifier, qube_name): + self.status_notifier = status_notifier + self.qube_name = qube_name + + def put(self, message): + if isinstance(message, (StatusInfo, FormatedLine)): + message.qname = self.qube_name + self.status_notifier.put(message) + + +class AdminVMAgentManager(UpdateAgentManager): + """ + Handle AdminVM updates. + """ + def __init__(self, app, qube, agent_args, show_progress): + super().__init__(app, qube, agent_args, show_progress) + + def run_agent( + self, agent_args, status_notifier, termination + ) -> ProcessResult: + """ + Download updates in UpdateVM and install them in AdminVM. + """ + status_notifier = StatusNotifierWrapper(status_notifier, "dom0") + result = self._run_agent( + agent_args, status_notifier, termination) + output = result.out.split("\n") + result.err.split("\n") + for line in output: + self.log.debug('agent output: %s', line) + self.log.info('agent exit code: %d', result.code) + if not agent_args.show_output or not output: + result.out = "OK" if result.code == EXIT.OK else \ + f"ERROR (exit code {result.code}, details in {self.log_path})" + return result + + def _run_agent( + self, agent_args, status_notifier, termination + ) -> ProcessResult: + self.log.info('Running update agent for %s', self.qube.name) + this_dir = os.path.dirname(os.path.realpath(__file__)) + dest_agent = join(this_dir, UpdateAgentManager.ENTRYPOINT) + + with QubeConnection( + self.qube, + None, + False, + self.log, + self.show_progress, + status_notifier + ) as qconn: + if termination.value: + qconn.status = FinalStatus.CANCELLED + return ProcessResult(EXIT.SIGINT, "", "Cancelled") + + self.log.info( + "The agent is starting the task in qube: %s", self.qube.name) + result = qconn.run_entrypoint(dest_agent, agent_args) + if not result and qconn.status != FinalStatus.NO_UPDATES: + qconn.status = FinalStatus.SUCCESS + + return result + diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index b2e2fe5..f09e25e 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -5,6 +5,7 @@ import argparse import asyncio import logging +import subprocess import sys import os import grp @@ -59,16 +60,36 @@ def main(args=None, app=qubesadmin.Qubes()): print("No qube selected for update") return EXIT.OK_NO_UPDATES if args.signal_no_updates else EXIT.OK + admin = [target for target in targets if target.klass == 'AdminVM'] independent = [target for target in targets if target.klass in ( 'TemplateVM', 'StandaloneVM')] derived = [target for target in targets if target.klass not in ( - 'TemplateVM', 'StandaloneVM')] + 'AdminVM', 'TemplateVM', 'StandaloneVM')] + + no_updates = True + ret_code_admin = EXIT.OK + if admin: + message = f"The admin VM ({admin[0].name}) will be updated." + else: + message = "The admin VM will not be updated." + if args.dry_run: + print(message) + elif admin: + log.debug(message) + if args.just_print_progress and args.no_refresh: + # internal usage just for installing ready updates, use carefully + ret_code_admin, admin_status = run_update(admin, args, log, "admin VM") + else: + # use qubes-dom0-update to update dom0 + ret_code_admin, admin_status = run_admin_update(admin[0], args, log) + no_updates = all(stat == FinalStatus.NO_UPDATES + for stat in admin_status.values()) # independent qubes first (TemplateVMs, StandaloneVMs) ret_code_independent, templ_statuses = run_update( independent, args, log, "templates and standalones") no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in templ_statuses.values()) + for stat in templ_statuses.values()) and no_updates # then derived qubes (AppVMs...) ret_code_appvm, app_statuses = run_update(derived, args, log) no_updates = all(stat == FinalStatus.NO_UPDATES @@ -77,9 +98,11 @@ def main(args=None, app=qubesadmin.Qubes()): ret_code_restart = apply_updates_to_appvm( args, independent, templ_statuses, app_statuses, log) - ret_code = max(ret_code_independent, ret_code_appvm, ret_code_restart) + ret_code = max(ret_code_admin, ret_code_independent, ret_code_appvm, ret_code_restart) if ret_code == EXIT.OK and no_updates and args.signal_no_updates: return EXIT.OK_NO_UPDATES + if ret_code == EXIT.OK_NO_UPDATES and not args.signal_no_updates: + return EXIT.OK return ret_code @@ -153,6 +176,11 @@ def parse_args(args, app): help='DEFAULT. Target all updatable VMs except AdminVM. ' 'Use explicitly with "--targets" to include both.') + # for internal usage, e.g., download updates via proxy vm + parser.add_argument( + '--display-name', action='store', + help=argparse.SUPPRESS) + AgentArgs.add_arguments(parser) args = parser.parse_args(args) @@ -205,10 +233,7 @@ def preselect_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: # remove skipped qubes and dom0 - not a target to_skip = args.skip.split(',') - if 'dom0' in targets and not args.quiet: - print("Skipping dom0. To update AdminVM use `qubes-dom0-update`") - targets = {vm for vm in targets - if vm.name != 'dom0' and vm.name not in to_skip} + targets = {vm for vm in targets if vm.name not in to_skip} # exclude vms with `skip-update` feature, but allow --targets to override it if not args.targets: @@ -265,20 +290,50 @@ def is_stale(vm, expiration_period): return False +def run_admin_update(admin_vm, args, log): + cmd = ["qubes-dom0-update", "-y"] + if args.quiet: + cmd.append('--quiet') + if args.just_print_progress: + cmd.append("--just-print-progress") + elif args.signal_no_updates: + # --just-print-progress checks it by default + proc = subprocess.Popen(["qubes-dom0-update", "--check-only"]) + proc.wait() + if proc.returncode == 0: + return proc.returncode, {admin_vm.name: FinalStatus.NO_UPDATES} + proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + proc.wait() + if proc.returncode == 0: + status = FinalStatus.SUCCESS + elif proc.returncode == 100: + status = FinalStatus.NO_UPDATES + else: + status = FinalStatus.ERROR + return proc.returncode, {admin_vm.name: status} + + def run_update( targets, args, log, qube_klass="qubes" ) -> Tuple[int, Dict[str, FinalStatus]]: - if not targets: - return EXIT.OK, {} - - message = f"Following {qube_klass} will be updated:" + \ - ",".join((target.name for target in targets)) + if targets: + message = f"Following {qube_klass} will be updated: " + \ + ", ".join((target.name for target in targets)) + else: + if qube_klass == "qubes": + message = "" # no need to inform about app VMs etc. + else: + message = f"No {qube_klass} will be updated." if args.dry_run: - print(message) + if message: + print(message) return EXIT.OK, {target.name: FinalStatus.SUCCESS for target in targets} else: log.debug(message) + if not targets: + return EXIT.OK, {} + runner = update_manager.UpdateManager(targets, args, log=log) ret_code, statuses = runner.run(agent_args=args) if ret_code: From 5abf2fb9de6c95748782e17fea22767b5b87ddfa Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Fri, 30 May 2025 11:36:42 +0200 Subject: [PATCH 02/13] vmupdate: update tests --- vmupdate/tests/test_vmupdate.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/vmupdate/tests/test_vmupdate.py b/vmupdate/tests/test_vmupdate.py index c6b43c1..f276dc1 100644 --- a/vmupdate/tests/test_vmupdate.py +++ b/vmupdate/tests/test_vmupdate.py @@ -45,6 +45,14 @@ def test_no_options_do_nothing(_logger, _log_file, _chmod, _chown, test_qapp): assert main(args, test_qapp) == EXIT.OK_NO_UPDATES +class Subproc: + def __init__(self, returncode=0): + self.returncode = returncode + + def wait(self): + pass + + @patch('vmupdate.update_manager.TerminalMultiBar.print') @patch('os.chmod') @patch('os.chown') @@ -53,11 +61,13 @@ def test_no_options_do_nothing(_logger, _log_file, _chmod, _chown, test_qapp): @patch('vmupdate.update_manager.UpdateAgentManager') @patch('multiprocessing.Pool') @patch('multiprocessing.Manager') +@patch('subprocess.Popen') def test_preselection( - mp_manager, mp_pool, agent_mng, + dummy_subprocess, mp_manager, mp_pool, agent_mng, _logger, _log_file, _chmod, _chown, _print, test_qapp, test_manager, test_pool, test_agent, ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -160,12 +170,14 @@ def test_preselection( @patch('vmupdate.update_manager.UpdateAgentManager') @patch('multiprocessing.Pool') @patch('multiprocessing.Manager') +@patch('subprocess.Popen') def test_selection( - mp_manager, mp_pool, agent_mng, + dummy_subprocess, mp_manager, mp_pool, agent_mng, _logger, _log_file, _chmod, _chown, _print, test_qapp, test_manager, test_pool, test_agent, monkeypatch ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -200,7 +212,8 @@ def test_selection( else: feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], 'retcode': EXIT.OK} - for vm in selected} + for vm in selected + if vm.klass != "AdminVM"} # dom0 is not updated via agent monkeypatch.setattr( vmupdate, "preselect_targets", lambda *_: selected) if feed: @@ -233,12 +246,14 @@ def test_selection( @patch('multiprocessing.Pool') @patch('multiprocessing.Manager') @patch('asyncio.run') +@patch('subprocess.Popen') def test_restarting( - arun, mp_manager, mp_pool, agent_mng, + dummy_subprocess, arun, mp_manager, mp_pool, agent_mng, _logger, _log_file, _chmod, _chown, _print, test_qapp, test_manager, test_pool, test_agent, monkeypatch ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -284,7 +299,8 @@ def test_restarting( monkeypatch.setattr(vmupdate, "get_targets", lambda *_: all) feed = {vm.name: {'statuses': [vm.update_result], 'retcode': None} # we don't care - for vm in all} + for vm in all + if vm.klass != "AdminVM"} # dom0 is not updated via agent unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) From 88fd41643baa27b299ca87c1aa0185fa1a163a45 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 19 Jun 2025 16:24:07 +0200 Subject: [PATCH 03/13] vmupdate: @marmarek comments pass allow_erasing as argument instead of setting protected attribute check qubes-agent-version in UpdateVM and inform properly the user about chosen action --- dom0-updates/qubes-dom0-update | 51 ++++++++++++++++++++++++++-- vmupdate/agent/source/dnf/dnf_api.py | 6 ++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index 284bf91..2b718ce 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -320,7 +320,37 @@ qvm-run --nogui -q -- "$UPDATEVM" "rm -rf -- '$dom0_updates_dir/etc' '$dom0_upda QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io) -if [ "$PROGRESS_REPORTING" == "1" ]; then +progress_agent_version="4.3" + +get_base_vm() { + # Resolve base VM (TemplateVM or StandaloneVM) + local vm="$1" + while true; do + # If it's a TemplateVM or StandaloneVM, stop + if qvm-check --template "$vm" &>/dev/null || qvm-check --standalone "$vm" &>/dev/null; then + echo "$vm" + return + fi + # Try to get its template + vm=$(qvm-prefs "$vm" template 2>/dev/null) || return 1 + done +} +version_check() { + # Compare version numbers (e.g., 4.2 < 4.3) + awk 'BEGIN {exit !(ARGV[1] < ARGV[2])}' "$1" "$2" +} +base_vm=$(get_base_vm $UPDATEVM) +OLD_VERSION=0 +if [ -n "$base_vm" ]; then + agent_version=$(qvm-features "$base_vm" qubes-agent-version 2>/dev/null) + + if [ -n "$agent_version" ] && version_check "$agent_version" "$progress_agent_version"; then + OLD_VERSION=1 + fi +fi + + +if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "0" ]; then CMD="/usr/lib/qubes/qubes-download-dom0-updates-init.sh" qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" @@ -331,6 +361,7 @@ if [ "$PROGRESS_REPORTING" == "1" ]; then qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=DEBUG RETCODE=$? if [ "$RETCODE" -eq 100 ]; then + echo "$(hostname):out: Nothing to do." echo "$(hostname) done no_updates" >&2 exit 100 fi @@ -345,6 +376,9 @@ if [ "$PROGRESS_REPORTING" == "1" ]; then CMD="/usr/lib/qubes/qubes-download-dom0-updates-finish.sh" qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" else + if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then + echo "$(hostname):out: Progress reporting requires updateVM based on a template with Qubes 4.3 packages." + fi CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui" # We avoid using bash’s own facilities for this, as they produce $'\n'-style @@ -363,10 +397,23 @@ else # this and enters an infinite loop. CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" fi - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null + if [ "$PROGRESS_REPORTING" == "1" ]; then + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/``^/$(hostname):out: /" + # "consume" the last empty line + echo "" + else + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null + fi fi RETCODE=$? +if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then + echo "$(hostname) updating 50.0" >&2 + if [ "$RETCODE" -ne 0 ]; then + echo "$(hostname) done error" >&2 + exit $RETCODE + fi +fi if [[ "$REMOTE_ONLY" = '1' ]] || [ "$RETCODE" -ne 0 ]; then if [ "$CHECK_ONLY" = '1' ]; then if [ "$RETCODE" -eq 100 ]; then diff --git a/vmupdate/agent/source/dnf/dnf_api.py b/vmupdate/agent/source/dnf/dnf_api.py index 6b38f53..ed2971b 100644 --- a/vmupdate/agent/source/dnf/dnf_api.py +++ b/vmupdate/agent/source/dnf/dnf_api.py @@ -43,7 +43,7 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): super().__init__(log_handler, log_level, agent_type) if self.type == AgentType.UPDATE_VM: - dnfconf = "/var/lib/qubes/dom0-updates/etc/dnf/dnf.conf" + dnfconf = self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" else: dnfconf = None conf = dnf.conf.Conf() @@ -73,8 +73,6 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): subst['releasever'] = releasever self.base = dnf.Base(conf) - if self.type == AgentType.UPDATE_VM: - self.base._allow_erasing = True # Repositories serve as sources of information about packages. self.base.read_all_repos() @@ -130,7 +128,7 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: # fill empty `Command line` column in dnf history self.base.cmds = ["qubes-vm-update"] - self.base.resolve() + self.base.resolve(allow_erasing=self.type == AgentType.UPDATE_VM) trans = self.base.transaction if not trans: self.log.info("No packages to upgrade, quitting.") From 627272f9ee6cf1e9d266b191cd36faddd148cf6e Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 19 Jun 2025 16:26:17 +0200 Subject: [PATCH 04/13] vmupdate: adjust progress weights in DNF5 for fetch and upgrade operations to be compliant with DNF's progress reporting. --- vmupdate/agent/source/dnf/dnf5_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index b2cf5de..7d6e2d2 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -63,8 +63,8 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): self.base.setup() self.config = self.base.get_config() update = FetchProgress(weight=0, log=self.log) # % of total time - fetch = FetchProgress(weight=50, log=self.log) # % of total time - upgrade = UpgradeProgress(weight=50, log=self.log) # % of total time + fetch = FetchProgress(weight=55, log=self.log) # % of total time + upgrade = UpgradeProgress(weight=45, log=self.log) # % of total time self.progress = ProgressReporter(update, fetch, upgrade) def refresh(self, hard_fail: bool) -> ProcessResult: From cf188d77751880aa3fd6759c1a946df6e0b5ae43 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Mon, 23 Jun 2025 14:40:31 +0200 Subject: [PATCH 05/13] vmupdate: fix buffered print --- vmupdate/update_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index a04a0a0..74ab24c 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -137,7 +137,7 @@ def collect_result(self, result_tuple: Tuple[str, ProcessResult]): def print(self, *args): if self.buffered: - self.buffer += ' '.join(args) + '\n' + self.buffer += ' '.join(str(args)) + '\n' else: print(*args, file=sys.stdout, flush=True) From 214f031b6a090684ada50603816d8a755b9ac577 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Fri, 14 Nov 2025 11:52:50 +0100 Subject: [PATCH 06/13] vmupdate: libdnf5 --- rpm_spec/core-dom0-linux.spec.in | 1 + vmupdate/agent/entrypoint.py | 3 ++- vmupdate/agent/source/dnf/dnf5_api.py | 8 +++++--- vmupdate/qube_connection.py | 2 +- vmupdate/vmupdate.py | 3 ++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/rpm_spec/core-dom0-linux.spec.in b/rpm_spec/core-dom0-linux.spec.in index f3d9dce..dcbafeb 100644 --- a/rpm_spec/core-dom0-linux.spec.in +++ b/rpm_spec/core-dom0-linux.spec.in @@ -48,6 +48,7 @@ Requires: qubes-core-admin-client Requires: qubes-utils >= 3.1.3 Requires: qubes-utils-libs >= 4.0.16 Requires: qubes-rpm-oxide +Requires: python-libdnf5 Conflicts: qubes-core-dom0 < 4.0.23 Requires: %{name}-kernel-install Requires: xdotool diff --git a/vmupdate/agent/entrypoint.py b/vmupdate/agent/entrypoint.py index 2a3486b..1c6d76d 100755 --- a/vmupdate/agent/entrypoint.py +++ b/vmupdate/agent/entrypoint.py @@ -31,7 +31,7 @@ def main(args=None): os_data, log, log_handler, log_level, agent_type, args.no_progress) log.debug("Running upgrades.") - return_code = pkg_mng.upgrade(refresh=not args.no_refresh, + return_code = pkg_mng.upgrade(refresh=not args.no_refresh or agent_type is AgentType.DOM0, hard_fail=not args.force_upgrade, remove_obsolete=not args.leave_obsolete, print_streams=args.show_output, @@ -113,6 +113,7 @@ def import_rhel_package_manager(os_data, log, no_progress): try: from source.dnf.dnf5_api import DNF5 as PackageManager loaded = True + log.info("Using dnf5.") except ImportError: log.warning("Failed to load dnf5.") diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index 7d6e2d2..930c6a9 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -115,9 +115,11 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: EXIT.OK_NO_UPDATES, out="", err="\n".join(transaction.get_resolve_logs_as_strings())) - self.base.set_download_callbacks( - libdnf5.repo.DownloadCallbacksUniquePtr( - self.progress.fetch_progress)) + if self.type != AgentType.DOM0: + # + self.base.set_download_callbacks( + libdnf5.repo.DownloadCallbacksUniquePtr( + self.progress.fetch_progress)) transaction.download() if not transaction.check_gpg_signatures(): diff --git a/vmupdate/qube_connection.py b/vmupdate/qube_connection.py index 91d18b4..660b27a 100644 --- a/vmupdate/qube_connection.py +++ b/vmupdate/qube_connection.py @@ -30,7 +30,7 @@ import qubesadmin from vmupdate.agent.source.args import AgentArgs -from vmupdate.agent.source.log_congfig import LOGPATH, LOG_FILE +from vmupdate.agent.source.log_config import LOGPATH, LOG_FILE from vmupdate.agent.source.status import StatusInfo, FinalStatus, FormatedLine from vmupdate.agent.source.common.process_result import ProcessResult diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index f09e25e..15c2d3e 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -2,6 +2,7 @@ """ Update qubes. """ + import argparse import asyncio import logging @@ -291,7 +292,7 @@ def is_stale(vm, expiration_period): def run_admin_update(admin_vm, args, log): - cmd = ["qubes-dom0-update", "-y"] + cmd = ["sudo", "qubes-dom0-update", "-y"] if args.quiet: cmd.append('--quiet') if args.just_print_progress: From 31fb3c481ee6e6d87e4e1d890862af2de00d5276 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Tue, 18 Nov 2025 23:53:30 +0100 Subject: [PATCH 07/13] vmupdate: fix dnf5 --- vmupdate/agent/entrypoint.py | 2 +- vmupdate/agent/source/dnf/dnf5_api.py | 9 +- vmupdate/qube_connection.py | 40 ++++++-- vmupdate/update_manager.py | 140 ++++++++++---------------- vmupdate/vmupdate.py | 29 +----- 5 files changed, 91 insertions(+), 129 deletions(-) diff --git a/vmupdate/agent/entrypoint.py b/vmupdate/agent/entrypoint.py index 1c6d76d..c476a02 100755 --- a/vmupdate/agent/entrypoint.py +++ b/vmupdate/agent/entrypoint.py @@ -31,7 +31,7 @@ def main(args=None): os_data, log, log_handler, log_level, agent_type, args.no_progress) log.debug("Running upgrades.") - return_code = pkg_mng.upgrade(refresh=not args.no_refresh or agent_type is AgentType.DOM0, + return_code = pkg_mng.upgrade(refresh=not args.no_refresh, hard_fail=not args.force_upgrade, remove_obsolete=not args.leave_obsolete, print_streams=args.show_output, diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index 930c6a9..bd4ef85 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -81,11 +81,6 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.log.debug("Refreshing available packages...") result += self.expire_cache() - - repo_sack = self.base.get_repo_sack() - repo_sack.create_repos_from_system_configuration() - repo_sack.load_repos() - self.log.debug("Cache refresh successful.") except Exception as exc: self.log.error( "An error occurred while refreshing packages: %s", str(exc)) @@ -101,6 +96,10 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: result = ProcessResult() try: self.log.debug("Performing package upgrade...") + repo_sack = self.base.get_repo_sack() + repo_sack.create_repos_from_system_configuration() + repo_sack.load_repos() + goal = Goal(self.base) if self.type == AgentType.UPDATE_VM: goal.set_allow_erasing(True) diff --git a/vmupdate/qube_connection.py b/vmupdate/qube_connection.py index 660b27a..5b03192 100644 --- a/vmupdate/qube_connection.py +++ b/vmupdate/qube_connection.py @@ -149,7 +149,7 @@ def _copy_file_from_dom0(self, src, dest) -> ProcessResult: return result def run_entrypoint( - self, entrypoint_path: str, agent_args + self, entrypoint_path: str | List, agent_args ) -> ProcessResult: """ Run a script in the qube. @@ -158,8 +158,12 @@ def run_entrypoint( :param agent_args: args for agent entrypoint :return: return code and output of the script """ - command = [QubeConnection.PYTHON_PATH, entrypoint_path, - *AgentArgs.to_cli_args(agent_args)] + if isinstance(entrypoint_path, str): + command = [QubeConnection.PYTHON_PATH, entrypoint_path, + *AgentArgs.to_cli_args(agent_args)] + else: + command = entrypoint_path + result = self._run_shell_command_in_qube( self.qube, command, show=self.show_progress) @@ -188,11 +192,25 @@ def _run_shell_command_in_qube( def _run_command_and_wait_for_output( self, target, command: List[str] ) -> ProcessResult: + self.logger.debug("Wait for output") + result = ProcessResult() try: - untrusted_stdout_and_stderr = target.run_with_args( - *command, user='root' - ) - result = ProcessResult.from_untrusted_out_err( + if self.qube.klass == "AdminVM": + proc = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + untrusted_stdout_and_stderr = proc.communicate() + if proc.returncode: + raise CalledProcessError( + proc.returncode, + command, + output=untrusted_stdout_and_stderr[0] + + untrusted_stdout_and_stderr[1] + ) + else: + untrusted_stdout_and_stderr = target.run_with_args( + *command, user='root' + ) + result += ProcessResult.from_untrusted_out_err( *untrusted_stdout_and_stderr) except CalledProcessError as err: if err.returncode == 100: @@ -211,6 +229,7 @@ def _run_command_and_wait_for_output( def _run_command_and_actively_report_progress( self, target, command: List[str] ) -> ProcessResult: + self.logger.debug("Progress reporting enabled.") if self.qube.klass == "AdminVM": proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif "--download-only" in command: @@ -255,8 +274,11 @@ def _collect_stderr(self, proc) -> bytes: try: progress = float(line) except ValueError: - self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) - continue + try: + progress = float(line.split()[-1]) + except ValueError: + self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) + continue if progress == 100.: progress_finished = True diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index 74ab24c..9877467 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -2,7 +2,8 @@ # # The Qubes OS Project, http://www.qubes-os.org # -# Copyright (C) 2022 Piotr Bartman +# Copyright (C) 2022-2025 Piotr Bartman-Szwarc +# # Copyright (C) 2022 Marek Marczykowski-Górecki # # @@ -44,7 +45,7 @@ class UpdateManager: Update multiple qubes simultaneously. """ - def __init__(self, qubes, args, log): + def __init__(self, qubes, args, log, dom0=False): self.qubes = qubes self.max_concurrency = args.max_concurrency self.show_output = args.show_output @@ -57,6 +58,7 @@ def __init__(self, qubes, args, log): self.cleanup = not args.no_cleanup self.ret_code = EXIT.OK self.log = log + self.dom0 = dom0 def run(self, agent_args): """ @@ -85,10 +87,10 @@ def run(self, agent_args): progress_bar.pool.apply_async( update_qube, (qube, agent_args, show_progress, - progress_bar.status_notifier, progress_bar.termination), + progress_bar.status_notifier, progress_bar.termination, self.dom0), callback=self.collect_result, error_callback=print ) - if qube.klass == "AdminVM": + if qube.klass == "AdminVM" and show_progress: # progress of AdminVM is continuation of different process, # so we want to skip 0 value at beginning progress_bar.progress_bars[qube.name].progress = None @@ -294,7 +296,7 @@ def close(self): def update_qube( - qube, agent_args, show_progress, status_notifier, termination + qube, agent_args, show_progress, status_notifier, termination, dom0 ) -> Tuple[str, ProcessResult]: """ Create and run `UpdateAgentManager` for qube. @@ -304,6 +306,7 @@ def update_qube( :param show_progress: if progress should be printed in real time :param status_notifier: an object to be fed with the progress data :param termination: signal to gracefully terminate subprocess + :param dom0: whether to use qubes-dom0-update :return: """ if agent_args.display_name is not None: @@ -314,21 +317,13 @@ def update_qube( return qube.name, ProcessResult(EXIT.SIGINT, "Canceled") try: - if qube.klass == "AdminVM": - # AdminVM update - runner = AdminVMAgentManager( - qube.app, - qube, - agent_args=agent_args, - show_progress=show_progress - ) - else: - runner = UpdateAgentManager( - qube.app, - qube, - agent_args=agent_args, - show_progress=show_progress - ) + runner = UpdateAgentManager( + qube.app, + qube, + agent_args=agent_args, + show_progress=show_progress, + dom0=dom0 + ) result = runner.run_agent( agent_args=agent_args, status_notifier=status_notifier, @@ -352,13 +347,14 @@ class UpdateAgentManager: WORKDIR = "/run/qubes-update/" def __init__( - self, app, qube, agent_args, show_progress): + self, app, qube, agent_args, show_progress, dom0): self.qube = qube self.app = app + self.dom0 = dom0 (self.log, self.log_handler, log_level, self.log_path, self.log_formatter) = init_logs( - directory=UpdateAgentManager.LOGPATH, + directory=self.LOGPATH, file=f'update-{qube.name}.log', format_=UpdateAgentManager.FORMAT_LOG, level=agent_args.log, @@ -375,6 +371,9 @@ def run_agent( """ Copy agent file to dest vm, run entrypoint, collect output and logs. """ + if self.qube.klass == "AdminVM" and not self.dom0: + # using UpdateVM to download updates + status_notifier = StatusNotifierWrapper(status_notifier, "dom0") result = self._run_agent( agent_args, status_notifier, termination) output = result.out.split("\n") + result.err.split("\n") @@ -390,25 +389,42 @@ def _run_agent( self, agent_args, status_notifier, termination ) -> ProcessResult: self.log.info('Running update agent for %s', self.qube.name) - dest_dir = UpdateAgentManager.WORKDIR - dest_agent = os.path.join(dest_dir, UpdateAgentManager.ENTRYPOINT) - this_dir = os.path.dirname(os.path.realpath(__file__)) - src_dir = join(this_dir, UpdateAgentManager.AGENT_RELATIVE_DIR) + dest_dir = None + src_dir = None + cleanup = False + if self.qube.klass == "AdminVM": + if self.dom0: + entrypoint = ["sudo", "qubes-dom0-update", "-y"] + if agent_args.just_print_progress or self.show_progress: + entrypoint.append("--just-print-progress") + if agent_args.quiet: + entrypoint.append('--quiet') + else: + this_dir = os.path.dirname(os.path.realpath(__file__)) + entrypoint = join(this_dir, UpdateAgentManager.ENTRYPOINT) + else: + cleanup = self.cleanup + dest_dir = UpdateAgentManager.WORKDIR + entrypoint = os.path.join(dest_dir, UpdateAgentManager.ENTRYPOINT) + this_dir = os.path.dirname(os.path.realpath(__file__)) + src_dir = join(this_dir, UpdateAgentManager.AGENT_RELATIVE_DIR) with QubeConnection( self.qube, dest_dir, - self.cleanup, + cleanup, self.log, self.show_progress, status_notifier ) as qconn: - self.log.info( - "Transferring files to destination qube: %s", self.qube.name) - result = qconn.transfer_agent(src_dir) - if result: - self.log.error('Qube communication error code: %i', result.code) - return result + result = ProcessResult() + if self.qube.klass != "AdminVM": + self.log.info( + "Transferring files to destination qube: %s", self.qube.name) + result += qconn.transfer_agent(src_dir) + if result: + self.log.error('Qube communication error code: %i', result.code) + return result if termination.value: qconn.status = FinalStatus.CANCELLED @@ -416,7 +432,9 @@ def _run_agent( self.log.info( "The agent is starting the task in qube: %s", self.qube.name) - result += qconn.run_entrypoint(dest_agent, agent_args) + self.log.debug( + "%s", entrypoint) + result += qconn.run_entrypoint(entrypoint, agent_args) if not result and qconn.status != FinalStatus.NO_UPDATES: qconn.status = FinalStatus.SUCCESS @@ -447,57 +465,3 @@ def put(self, message): if isinstance(message, (StatusInfo, FormatedLine)): message.qname = self.qube_name self.status_notifier.put(message) - - -class AdminVMAgentManager(UpdateAgentManager): - """ - Handle AdminVM updates. - """ - def __init__(self, app, qube, agent_args, show_progress): - super().__init__(app, qube, agent_args, show_progress) - - def run_agent( - self, agent_args, status_notifier, termination - ) -> ProcessResult: - """ - Download updates in UpdateVM and install them in AdminVM. - """ - status_notifier = StatusNotifierWrapper(status_notifier, "dom0") - result = self._run_agent( - agent_args, status_notifier, termination) - output = result.out.split("\n") + result.err.split("\n") - for line in output: - self.log.debug('agent output: %s', line) - self.log.info('agent exit code: %d', result.code) - if not agent_args.show_output or not output: - result.out = "OK" if result.code == EXIT.OK else \ - f"ERROR (exit code {result.code}, details in {self.log_path})" - return result - - def _run_agent( - self, agent_args, status_notifier, termination - ) -> ProcessResult: - self.log.info('Running update agent for %s', self.qube.name) - this_dir = os.path.dirname(os.path.realpath(__file__)) - dest_agent = join(this_dir, UpdateAgentManager.ENTRYPOINT) - - with QubeConnection( - self.qube, - None, - False, - self.log, - self.show_progress, - status_notifier - ) as qconn: - if termination.value: - qconn.status = FinalStatus.CANCELLED - return ProcessResult(EXIT.SIGINT, "", "Cancelled") - - self.log.info( - "The agent is starting the task in qube: %s", self.qube.name) - result = qconn.run_entrypoint(dest_agent, agent_args) - if not result and qconn.status != FinalStatus.NO_UPDATES: - qconn.status = FinalStatus.SUCCESS - - return result - diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index 15c2d3e..c3a1c8b 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -82,7 +82,7 @@ def main(args=None, app=qubesadmin.Qubes()): ret_code_admin, admin_status = run_update(admin, args, log, "admin VM") else: # use qubes-dom0-update to update dom0 - ret_code_admin, admin_status = run_admin_update(admin[0], args, log) + ret_code_admin, admin_status = run_update(admin, args, log, "admin VM", dom0=True) no_updates = all(stat == FinalStatus.NO_UPDATES for stat in admin_status.values()) @@ -291,31 +291,8 @@ def is_stale(vm, expiration_period): return False -def run_admin_update(admin_vm, args, log): - cmd = ["sudo", "qubes-dom0-update", "-y"] - if args.quiet: - cmd.append('--quiet') - if args.just_print_progress: - cmd.append("--just-print-progress") - elif args.signal_no_updates: - # --just-print-progress checks it by default - proc = subprocess.Popen(["qubes-dom0-update", "--check-only"]) - proc.wait() - if proc.returncode == 0: - return proc.returncode, {admin_vm.name: FinalStatus.NO_UPDATES} - proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) - proc.wait() - if proc.returncode == 0: - status = FinalStatus.SUCCESS - elif proc.returncode == 100: - status = FinalStatus.NO_UPDATES - else: - status = FinalStatus.ERROR - return proc.returncode, {admin_vm.name: status} - - def run_update( - targets, args, log, qube_klass="qubes" + targets, args, log, qube_klass="qubes", dom0=False ) -> Tuple[int, Dict[str, FinalStatus]]: if targets: message = f"Following {qube_klass} will be updated: " + \ @@ -335,7 +312,7 @@ def run_update( if not targets: return EXIT.OK, {} - runner = update_manager.UpdateManager(targets, args, log=log) + runner = update_manager.UpdateManager(targets, args, log=log, dom0=dom0) ret_code, statuses = runner.run(agent_args=args) if ret_code: log.error("Updating fails with code: %d", ret_code) From a4ed2571cd279c10865825b2fd65bb8e8a67379f Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 27 Nov 2025 14:32:23 +0100 Subject: [PATCH 08/13] vmupdate: update tests --- vmupdate/agent/source/dnf/dnf_api.py | 2 +- vmupdate/tests/conftest.py | 2 +- vmupdate/tests/test_vmupdate.py | 11 ++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/vmupdate/agent/source/dnf/dnf_api.py b/vmupdate/agent/source/dnf/dnf_api.py index ed2971b..03c58a6 100644 --- a/vmupdate/agent/source/dnf/dnf_api.py +++ b/vmupdate/agent/source/dnf/dnf_api.py @@ -92,7 +92,7 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.base.conf.skip_if_unavailable = True try: self.log.debug("Refreshing available packages...") - repos = [r for r in self.base.repos.iter_enabled()] + repos = tuple(self.base.repos.iter_enabled()) # we do not know the size of the repositories self.progress.update_progress.start(len(repos), len(repos)) for i, repo in enumerate(repos): diff --git a/vmupdate/tests/conftest.py b/vmupdate/tests/conftest.py index 3bae319..57e25b6 100644 --- a/vmupdate/tests/conftest.py +++ b/vmupdate/tests/conftest.py @@ -124,7 +124,7 @@ def test_qapp(): def test_agent(): def closure(results, unexpected): class UpdateAgentManager: - def __init__(self, app, qube, agent_args, show_progress): + def __init__(self, app, qube, agent_args, show_progress, dom0): self.qube = qube def run_agent(self, agent_args, status_notifier, termination): diff --git a/vmupdate/tests/test_vmupdate.py b/vmupdate/tests/test_vmupdate.py index f276dc1..e8eab58 100644 --- a/vmupdate/tests/test_vmupdate.py +++ b/vmupdate/tests/test_vmupdate.py @@ -82,7 +82,7 @@ def test_preselection( app = domains["klass"]["AppVM"] disp = domains["klass"]["DispVM"] run_app = app & is_running - default = updatable & ((templ | stand) | (is_running & (disp | app))) + default = updatable & ((templ | stand | admin) | (is_running & (disp | app))) AdminVM = next(iter(admin)) TemplVM = next(iter(templ)) @@ -123,8 +123,7 @@ def test_preselection( ("--apps", "--targets", TemplVM.name,): run_app | {TemplVM}, ("--targets", NRunAppVM.name,): {NRunAppVM}, ("--targets", StandVM.name,): {StandVM}, - # dom0 skipped, user warning - ("--targets", AdminVM.name,): EXIT.OK_NO_UPDATES, + ("--targets", AdminVM.name,): {AdminVM}, ("--targets", "unknown",): EXIT.ERR_USAGE, ("--targets", f"{TemplVM.name},{StandVM.name}",): {TemplVM, StandVM}, ("--targets", f"{TemplVM.name},{TemplVM.name}",): EXIT.ERR_USAGE, @@ -212,8 +211,7 @@ def test_selection( else: feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], 'retcode': EXIT.OK} - for vm in selected - if vm.klass != "AdminVM"} # dom0 is not updated via agent + for vm in selected} monkeypatch.setattr( vmupdate, "preselect_targets", lambda *_: selected) if feed: @@ -299,8 +297,7 @@ def test_restarting( monkeypatch.setattr(vmupdate, "get_targets", lambda *_: all) feed = {vm.name: {'statuses': [vm.update_result], 'retcode': None} # we don't care - for vm in all - if vm.klass != "AdminVM"} # dom0 is not updated via agent + for vm in all} unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) From 15222c1493ab87b942e03ab520a0182d3b8db6a2 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Thu, 27 Nov 2025 15:25:35 +0100 Subject: [PATCH 09/13] vmupdate: pylint + black --- vmupdate/__init__.py | 2 +- vmupdate/agent/__init__.py | 2 +- vmupdate/agent/entrypoint.py | 48 +- vmupdate/agent/source/__init__.py | 2 +- vmupdate/agent/source/apt/__init__.py | 2 +- vmupdate/agent/source/apt/apt_api.py | 50 +- vmupdate/agent/source/apt/apt_cli.py | 24 +- vmupdate/agent/source/args.py | 66 +- vmupdate/agent/source/common/__init__.py | 2 +- .../agent/source/common/package_manager.py | 114 +-- .../agent/source/common/process_result.py | 41 +- .../agent/source/common/progress_reporter.py | 43 +- vmupdate/agent/source/dnf/__init__.py | 2 +- vmupdate/agent/source/dnf/dnf5_api.py | 115 ++- vmupdate/agent/source/dnf/dnf_api.py | 52 +- vmupdate/agent/source/dnf/dnf_cli.py | 56 +- vmupdate/agent/source/log_config.py | 22 +- vmupdate/agent/source/pacman/__init__.py | 2 +- vmupdate/agent/source/pacman/pacman_cli.py | 2 + vmupdate/agent/source/plugins/__init__.py | 22 +- .../plugins/allow_release_info_change.py | 5 +- .../source/plugins/bookworm_backports.py | 1 + .../agent/source/plugins/disable_deltarpm.py | 1 + .../plugins/fix_meminfo_writer_label.py | 5 +- .../agent/source/plugins/manage_rpm_macro.py | 5 +- .../source/plugins/pipewire_archlinux.py | 18 +- .../agent/source/plugins/updatesproxy_fix.py | 2 +- vmupdate/agent/source/status.py | 2 +- vmupdate/agent/source/utils.py | 42 +- vmupdate/qube_connection.py | 137 ++-- vmupdate/tests/conftest.py | 160 ++-- vmupdate/tests/test_vmupdate.py | 741 ++++++++++++------ vmupdate/update_manager.py | 228 ++++-- vmupdate/vmupdate.py | 334 +++++--- 34 files changed, 1509 insertions(+), 841 deletions(-) diff --git a/vmupdate/__init__.py b/vmupdate/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/__init__.py +++ b/vmupdate/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/__init__.py b/vmupdate/agent/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/__init__.py +++ b/vmupdate/agent/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/entrypoint.py b/vmupdate/agent/entrypoint.py index c476a02..157e887 100755 --- a/vmupdate/agent/entrypoint.py +++ b/vmupdate/agent/entrypoint.py @@ -17,7 +17,8 @@ def main(args=None): """ args = parse_args(args) log, log_handler, log_level, _log_path, _log_formatter = init_logs( - level=args.log, truncate_file=True) + level=args.log, truncate_file=True + ) log.debug("Run entrypoint with args: %s", str(args)) os_data = get_os_data() @@ -28,14 +29,16 @@ def main(args=None): if args.download_only: agent_type = AgentType.UPDATE_VM pkg_mng = get_package_manager( - os_data, log, log_handler, log_level, agent_type, args.no_progress) + os_data, log, log_handler, log_level, agent_type, args.no_progress + ) log.debug("Running upgrades.") - return_code = pkg_mng.upgrade(refresh=not args.no_refresh, - hard_fail=not args.force_upgrade, - remove_obsolete=not args.leave_obsolete, - print_streams=args.show_output, - ) + return_code = pkg_mng.upgrade( + refresh=not args.no_refresh, + hard_fail=not args.force_upgrade, + remove_obsolete=not args.leave_obsolete, + print_streams=args.show_output, + ) if not pkg_mng.PROGRESS_REPORTING and not args.no_progress: # even if progress reporting is unavailable we want info that update finished @@ -63,13 +66,16 @@ def parse_args(args): return args -def get_package_manager(os_data, log, log_handler, log_level, agent_type, no_progress): +def get_package_manager( + os_data, log, log_handler, log_level, agent_type, no_progress +): """ Returns instance of `PackageManager`. If appropriate python package is not installed or `no_progress` is `True` cli based version is returned. """ + # pylint: disable=import-outside-toplevel requirements = {} # plugins MUST be applied before import anything from package managers. # in case of apt configuration is loaded on `import apt`. @@ -83,12 +89,14 @@ def get_package_manager(os_data, log, log_handler, log_level, agent_type, no_pro PackageManager = import_debian_package_manager(log, no_progress) elif os_data["os_family"] == "ArchLinux": from source.pacman.pacman_cli import PACMANCLI as PackageManager - print(f"Progress reporting not supported.", flush=True) + + print("Progress reporting not supported.", flush=True) elif os_data["os_family"] == "Qubes": PackageManager = import_dom0_package_manager(os_data, log, no_progress) else: raise NotImplementedError( - "Only Debian, RedHat and ArchLinux based OS is supported.") + "Only Debian, RedHat and ArchLinux based OS is supported." + ) pkg_mng = PackageManager(log_handler, log_level, agent_type) pkg_mng.requirements = requirements @@ -99,6 +107,7 @@ def import_rhel_package_manager(os_data, log, no_progress): """ Import dnf package manager. """ + # pylint: disable=import-outside-toplevel dnf5_fedora_version = 41 if os_data["os_family"] == "RedHat": try: @@ -112,6 +121,7 @@ def import_rhel_package_manager(os_data, log, no_progress): if version >= dnf5_fedora_version: try: from source.dnf.dnf5_api import DNF5 as PackageManager + loaded = True log.info("Using dnf5.") except ImportError: @@ -120,14 +130,14 @@ def import_rhel_package_manager(os_data, log, no_progress): if not loaded: try: from source.dnf.dnf_api import DNF as PackageManager + loaded = True log.debug("Using dnf python API for progress reporting.") except ImportError: - print(f"Progress reporting not supported.", flush=True) + print("Progress reporting not supported.", flush=True) if no_progress or not loaded: - log.warning( - "Failed to load dnf with progress bar. Using dnf cli.") + log.warning("Failed to load dnf with progress bar. Using dnf cli.") from source.dnf.dnf_cli import DNFCLI as PackageManager return PackageManager @@ -137,13 +147,15 @@ def import_debian_package_manager(log, no_progress): """ Import apt package manager. """ + # pylint: disable=import-outside-toplevel loaded = False try: from source.apt.apt_api import APT as PackageManager + loaded = True except ImportError: log.warning("Failed to load apt with progress bar. Using apt cli.") - print(f"Progress reporting not supported.", flush=True) + print("Progress reporting not supported.", flush=True) if no_progress or not loaded: from source.apt.apt_cli import APTCLI as PackageManager @@ -155,12 +167,14 @@ def import_dom0_package_manager(os_data, log, no_progress): """ Import dnf package manager for dom0. """ + # pylint: disable=import-outside-toplevel major, minor = os_data["release"].split(".") major, minor = int(major), int(minor) loaded = False if major >= 5 or (major == 4 and minor >= 3): try: from source.dnf.dnf5_api import DNF5 as PackageManager + loaded = True except ImportError: log.warning("Failed to load dnf5.") @@ -168,20 +182,20 @@ def import_dom0_package_manager(os_data, log, no_progress): if not loaded: try: from source.dnf.dnf_api import DNF as PackageManager + loaded = True log.debug("Using dnf python API for progress reporting.") except ImportError: print(f"Progress reporting not supported.", flush=True) if no_progress or not loaded: - log.warning( - "Failed to load dnf with progress bar. Using dnf cli.") + log.warning("Failed to load dnf with progress bar. Using dnf cli.") from source.dnf.dnf_cli import DNFCLI as PackageManager return PackageManager -if __name__ == '__main__': +if __name__ == "__main__": try: sys.exit(main()) except RuntimeError as ex: diff --git a/vmupdate/agent/source/__init__.py b/vmupdate/agent/source/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/__init__.py +++ b/vmupdate/agent/source/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/apt/__init__.py b/vmupdate/agent/source/apt/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/apt/__init__.py +++ b/vmupdate/agent/source/apt/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/apt/apt_api.py b/vmupdate/agent/source/apt/apt_api.py index 5b52d8f..f551b2a 100644 --- a/vmupdate/agent/source/apt/apt_api.py +++ b/vmupdate/agent/source/apt/apt_api.py @@ -41,13 +41,14 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): super().__init__(log_handler, log_level, agent_type) self.apt_cache = apt.Cache() update = FetchProgress( - weight=4, log=self.log, refresh=True) # 4% of total time + weight=4, log=self.log, refresh=True + ) # 4% of total time fetch = FetchProgress(weight=48, log=self.log) # 48% of total time upgrade = UpgradeProgress(weight=48, log=self.log) # 48% of total time self.progress = ProgressReporter(update, fetch, upgrade) # to prevent a warning: `debconf: unable to initialize frontend: Dialog` - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ["DEBIAN_FRONTEND"] = "noninteractive" def refresh(self, hard_fail: bool) -> ProcessResult: """ @@ -62,7 +63,7 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.log.debug("Refreshing available packages...") success = self.apt_cache.update( self.progress.update_progress, - pulse_interval=1000 # microseconds + pulse_interval=1000, # microseconds ) self.apt_cache.open() if success: @@ -72,7 +73,8 @@ def refresh(self, hard_fail: bool) -> ProcessResult: result += ProcessResult(EXIT.ERR_VM_REFRESH) except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) return result @@ -85,20 +87,22 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: try: self.log.debug("Performing package upgrade...") self.apt_cache.upgrade(dist_upgrade=remove_obsolete) - Path(os.path.join( - apt_pkg.config.find_dir("Dir::Cache::Archives"), "partial") + Path( + os.path.join( + apt_pkg.config.find_dir("Dir::Cache::Archives"), "partial" + ) ).mkdir(parents=True, exist_ok=True) - apt_pkg.config.set('Dpkg::Options::', "--force-confdef") - apt_pkg.config.set('Dpkg::Options::', "--force-confold") + apt_pkg.config.set("Dpkg::Options::", "--force-confdef") + apt_pkg.config.set("Dpkg::Options::", "--force-confold") self.log.debug("Committing upgrade...") self.apt_cache.commit( - self.progress.fetch_progress, - self.progress.upgrade_progress + self.progress.fetch_progress, self.progress.upgrade_progress ) self.log.debug("Package upgrade successful.") except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) if remove_obsolete: @@ -118,9 +122,12 @@ def fail(self, item): Write an error message to the fake stderr. """ self.log.info(f"{self.action.capitalize()} failed.") - print(f"Fail to {self.action} {item.shortdesc}: " - f"{item.description} from {item.uri}", - flush=True, file=self._stderr) + print( + f"Fail to {self.action} {item.shortdesc}: " + f"{item.description} from {item.uri}", + flush=True, + file=self._stderr, + ) def pulse(self, _owner): """ @@ -131,9 +138,11 @@ def pulse(self, _owner): acquisition should be continued (True) or cancelled (False). """ if self.action == "fetch" and not self.fetching_notified: - print(f"Fetching {self.total_items} packages " - f"[{self._format_bytes(self.total_bytes)}]", - flush=True) + print( + f"Fetching {self.total_items} packages " + f"[{self._format_bytes(self.total_bytes)}]", + flush=True, + ) self.fetching_notified = True self.notify_callback(self.current_bytes / self.total_bytes * 100) return True @@ -169,8 +178,11 @@ def error(self, pkg, errormsg): """ Write an error message to the fake stderr. """ - print("Error during installation " + str(pkg) + ":" + str(errormsg), - flush=True, file=self._stderr) + print( + "Error during installation " + str(pkg) + ":" + str(errormsg), + flush=True, + file=self._stderr, + ) def start_update(self): print("Updating packages.", flush=True) diff --git a/vmupdate/agent/source/apt/apt_cli.py b/vmupdate/agent/source/apt/apt_cli.py index 867f848..4ff1f27 100644 --- a/vmupdate/agent/source/apt/apt_cli.py +++ b/vmupdate/agent/source/apt/apt_cli.py @@ -41,7 +41,7 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): self.package_manager: str = "apt-get" # to prevent a warning: `debconf: unable to initialize frontend: Dialog` - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ["DEBIAN_FRONTEND"] = "noninteractive" @contextlib.contextmanager def apt_lock(self): @@ -71,8 +71,13 @@ def refresh(self, hard_fail: bool) -> ProcessResult: # apply lock externally to wait for it, until # https://bugs.debian.org/1069167 gets implemented with self.apt_lock(): - cmd = [self.package_manager, - "-o", "Debug::NoLocking=true", "-q", "update"] + cmd = [ + self.package_manager, + "-o", + "Debug::NoLocking=true", + "-q", + "update", + ] result = self.run_cmd(cmd) # 'apt-get update' reports error with exit code 100, but updater as a # whole reserves it for "no updates" @@ -119,10 +124,13 @@ def get_action(self, remove_obsolete: bool) -> List[str]: """ Return command `upgrade` or `dist-upgrade` if `remove_obsolete`. """ - result = ["-y", - "-o", 'Dpkg::Options::=--force-confdef', - "-o", 'Dpkg::Options::=--force-confold' - ] + result = [ + "-y", + "-o", + "Dpkg::Options::=--force-confdef", + "-o", + "Dpkg::Options::=--force-confold", + ] result += ["dist-upgrade"] if remove_obsolete else ["upgrade"] return result @@ -153,7 +161,7 @@ def remove_obsolete_kernels(self) -> ProcessResult: obsoletes = set() for line in result.out.splitlines(): if line.startswith("Remv"): - package_name = line[len("Remv "):] + package_name = line[len("Remv ") :] # consider using wider pattern if package_name.startswith("linux-image"): obsoletes.add(package_name.split(" ")[0]) diff --git a/vmupdate/agent/source/args.py b/vmupdate/agent/source/args.py index 611065e..1090bb3 100644 --- a/vmupdate/agent/source/args.py +++ b/vmupdate/agent/source/args.py @@ -24,39 +24,53 @@ class AgentArgs: # To avoid code repeating when we want to retrieve arguments OPTIONS = { - ("--log",): {"action": 'store', - "default": "INFO", - "help": 'Provide logging level. Values: DEBUG, ' - 'INFO (default), WARNING, ERROR, CRITICAL'}, - ("--no-refresh",): {"action": 'store_true', - "help": 'Do not refresh available packages before ' - 'upgrading'}, - ("--force-upgrade", "-f"): - {"action": 'store_true', - "help": 'Try upgrade even if errors are ' - 'encountered (like a refresh error)'}, + ("--log",): { + "action": "store", + "default": "INFO", + "help": "Provide logging level. Values: DEBUG, " + "INFO (default), WARNING, ERROR, CRITICAL", + }, + ("--no-refresh",): { + "action": "store_true", + "help": "Do not refresh available packages before " "upgrading", + }, + ("--force-upgrade", "-f"): { + "action": "store_true", + "help": "Try upgrade even if errors are " + "encountered (like a refresh error)", + }, ("--no-cleanup",): { - "action": 'store_true', - "help": 'Do not remove cache files after upgrading'}, + "action": "store_true", + "help": "Do not remove cache files after upgrading", + }, ("--leave-obsolete",): { - "action": 'store_true', - "help": 'Do not remove updater and cache files from target qube'}, + "action": "store_true", + "help": "Do not remove updater and cache files from target qube", + }, ("--download-only",): { - "action": 'store_true', - "help": 'Only download packages'}, + "action": "store_true", + "help": "Only download packages", + }, } EXCLUSIVE_OPTIONS_1 = { - ("--show-output", "--verbose", "-v"): - {"action": 'store_true', - "help": 'Show output of management commands'}, - ("--quiet", "-q"): {"action": 'store_true', - "help": 'Do not print anything to stdout'} + ("--show-output", "--verbose", "-v"): { + "action": "store_true", + "help": "Show output of management commands", + }, + ("--quiet", "-q"): { + "action": "store_true", + "help": "Do not print anything to stdout", + }, } EXCLUSIVE_OPTIONS_2 = { - ("--no-progress",): {"action": "store_true", - "help": "Do not show upgrading progress."}, - ("--just-print-progress",): {"action": "store_true", - "help": argparse.SUPPRESS} + ("--no-progress",): { + "action": "store_true", + "help": "Do not show upgrading progress.", + }, + ("--just-print-progress",): { + "action": "store_true", + "help": argparse.SUPPRESS, + }, } ALL_OPTIONS = {**OPTIONS, **EXCLUSIVE_OPTIONS_1, **EXCLUSIVE_OPTIONS_2} diff --git a/vmupdate/agent/source/common/__init__.py b/vmupdate/agent/source/common/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/common/__init__.py +++ b/vmupdate/agent/source/common/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/common/package_manager.py b/vmupdate/agent/source/common/package_manager.py index 869b648..24971e8 100644 --- a/vmupdate/agent/source/common/package_manager.py +++ b/vmupdate/agent/source/common/package_manager.py @@ -18,7 +18,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -""" package manager for VMs """ +"""package manager for VMs""" import io import logging import subprocess @@ -36,11 +36,13 @@ class AgentType(enum.Enum): class PackageManager: - """ main package manager class """ + """main package manager class""" + def __init__(self, log_handler, log_level, agent_type: AgentType): self.package_manager: Optional[str] = None self.log = logging.getLogger( - f'vm-update.agent.{self.__class__.__name__}') + f"vm-update.agent.{self.__class__.__name__}" + ) self.log.setLevel(log_level) self.log.addHandler(log_handler) self.log.propagate = False @@ -48,11 +50,11 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): self.type = agent_type def upgrade( - self, - refresh: bool, - hard_fail: bool, - remove_obsolete: bool, - print_streams: bool = False, + self, + refresh: bool, + hard_fail: bool, + remove_obsolete: bool, + print_streams: bool = False, ): """ Upgrade packages using system package manager. @@ -65,7 +67,8 @@ def upgrade( :return: return code """ result = self._upgrade( - refresh, hard_fail, remove_obsolete, self.requirements) + refresh, hard_fail, remove_obsolete, self.requirements + ) self._log_output("agent", result) if print_streams and not result.posted: if result.out: @@ -75,11 +78,11 @@ def upgrade( return result.code def _upgrade( - self, - refresh: bool, - hard_fail: bool, - remove_obsolete: bool, - requirements: Optional[Dict[str, str]] = None + self, + refresh: bool, + hard_fail: bool, + remove_obsolete: bool, + requirements: Optional[Dict[str, str]] = None, ) -> ProcessResult: result = ProcessResult(realtime=True) @@ -91,25 +94,31 @@ def _upgrade( if result_install: self.log.warning( "Installing requirements failed with exit code: %d", - result_install.code) + result_install.code, + ) result_install.code = EXIT.ERR_VM_PRE result += result_install if result and hard_fail: - self.log.error("Exiting due to a packages install error. " - "Use --force-upgrade to upgrade anyway.") + self.log.error( + "Exiting due to a packages install error. " + "Use --force-upgrade to upgrade anyway." + ) return result if refresh: print("Refreshing package info", flush=True) result_refresh = self.refresh(hard_fail) if result_refresh: - self.log.warning("Refreshing failed with code: %d", - result_refresh.code) + self.log.warning( + "Refreshing failed with code: %d", result_refresh.code + ) result_refresh.code = EXIT.ERR_VM_REFRESH result += result_refresh if result and hard_fail: - self.log.error("Exiting due to a refresh error. " - "Use --force-upgrade to upgrade anyway.") + self.log.error( + "Exiting due to a refresh error. " + "Use --force-upgrade to upgrade anyway." + ) return result result_upgrade = self.upgrade_internal(remove_obsolete) @@ -150,9 +159,9 @@ def _log_output(self, title, result): log("%s err: %s", title, err_line) def install_requirements( - self, - requirements: Optional[Dict[str, str]], - curr_pkg: Dict[str, List[str]] + self, + requirements: Optional[Dict[str, str]], + curr_pkg: Dict[str, List[str]], ) -> ProcessResult: """ Make sure if required packages is installed before upgrading. @@ -173,25 +182,24 @@ def install_requirements( else: to_upgrade[pkg] = version if to_install: - cmd = [self.package_manager, - "-q", - "-y", - "install", - *to_install] + cmd = [self.package_manager, "-q", "-y", "install", *to_install] result += self.run_cmd(cmd) if to_upgrade: - cmd = [self.package_manager, - "-q", - "-y", - *self.get_action(remove_obsolete=False), - *to_upgrade] + cmd = [ + self.package_manager, + "-q", + "-y", + *self.get_action(remove_obsolete=False), + *to_upgrade, + ] result += self.run_cmd(cmd) return result def run_cmd( - self, command: List[str], realtime: bool = True) -> ProcessResult: + self, command: List[str], realtime: bool = True + ) -> ProcessResult: """ Run command and wait. @@ -204,10 +212,12 @@ def run_cmd( result = ProcessResult.process_communicate(proc) result.posted = True else: - with subprocess.Popen(command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + with subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as proc: result = ProcessResult.process_communicate(proc) self.log.debug("command exit code: %i", result.code) @@ -221,12 +231,15 @@ def compare_packages(old, new): :param old: Dict[package_name, version] packages before update :param new: Dict[package_name, version] packages after update """ - return {"installed": {pkg: new[pkg] for pkg in new if pkg not in old}, - "updated": {pkg: {"old": old[pkg], "new": new[pkg]} - for pkg in new - if pkg in old and old[pkg] != new[pkg] - }, - "removed": {pkg: old[pkg] for pkg in old if pkg not in new}} + return { + "installed": {pkg: new[pkg] for pkg in new if pkg not in old}, + "updated": { + pkg: {"old": old[pkg], "new": new[pkg]} + for pkg in new + if pkg in old and old[pkg] != new[pkg] + }, + "removed": {pkg: old[pkg] for pkg in old if pkg not in new}, + } def _print_changes(self, changes): result = ProcessResult() @@ -234,7 +247,8 @@ def _print_changes(self, changes): if changes["installed"]: for pkg in sorted(changes["installed"]): result.out += self._print_to_string( - pkg, changes["installed"][pkg]) + pkg, changes["installed"][pkg] + ) else: result.out += self._print_to_string("None") @@ -244,8 +258,9 @@ def _print_changes(self, changes): result.out += self._print_to_string( pkg, str(changes["updated"][pkg]["old"])[2:-2] - + " -> " + - str(changes["updated"][pkg]["new"])[2:-2]) + + " -> " + + str(changes["updated"][pkg]["new"])[2:-2], + ) else: result.out += self._print_to_string("None") @@ -253,7 +268,8 @@ def _print_changes(self, changes): if changes["removed"]: for pkg in sorted(changes["removed"]): result.out += self._print_to_string( - pkg, changes["removed"][pkg]) + pkg, changes["removed"][pkg] + ) else: result.out += self._print_to_string("None") return result diff --git a/vmupdate/agent/source/common/process_result.py b/vmupdate/agent/source/common/process_result.py index 5ce5702..6747bf5 100644 --- a/vmupdate/agent/source/common/process_result.py +++ b/vmupdate/agent/source/common/process_result.py @@ -31,10 +31,13 @@ class ProcessResult: Controls where the results of subprocesses are directed (e.g., to stdout or buffered). """ + def __init__( - self, - code: int = EXIT.OK, out: str = "", err: str = "", - realtime: bool = False + self, + code: int = EXIT.OK, + out: str = "", + err: str = "", + realtime: bool = False, ): self.code: int = code self.out: str = out @@ -56,12 +59,12 @@ def process_communicate(cls, proc): @classmethod def from_untrusted_out_err( - cls, - untrusted_out: Optional[Union[str, bytes]], - untrusted_err: Optional[Union[str, bytes]] = "" + cls, + untrusted_out: Optional[Union[str, bytes]], + untrusted_err: Optional[Union[str, bytes]] = "", ): if untrusted_out is None: - untrusted_out_bytes = b'' + untrusted_out_bytes = b"" elif isinstance(untrusted_out, str): untrusted_out_bytes: bytes = untrusted_out.encode() else: @@ -69,7 +72,7 @@ def from_untrusted_out_err( out = ProcessResult.sanitize_output(untrusted_out_bytes) if untrusted_err is None: - untrusted_err_bytes = b'' + untrusted_err_bytes = b"" elif isinstance(untrusted_err, str): untrusted_err_bytes: bytes = untrusted_err.encode() else: @@ -80,10 +83,14 @@ def from_untrusted_out_err( @staticmethod def sanitize_output(untrusted_bytes: bytes, single: bool = False) -> str: - untrusted_str = untrusted_bytes.decode('ascii', errors='ignore') - return ''.join([c for c in untrusted_str - if 0x20 <= ord(c) <= 0x7e - or (c == '\n' and not single)]) + untrusted_str = untrusted_bytes.decode("ascii", errors="ignore") + return "".join( + [ + c + for c in untrusted_str + if 0x20 <= ord(c) <= 0x7E or (c == "\n" and not single) + ] + ) def __add__(self, other): new = deepcopy(self) @@ -92,9 +99,11 @@ def __add__(self, other): def __iadd__(self, other): if not isinstance(other, ProcessResult): - raise TypeError("unsupported operand type(s) for +:" - f"'{self.__class__.__name__}' and " - f"'{other.__class__.__name__}'") + raise TypeError( + "unsupported operand type(s) for +:" + f"'{self.__class__.__name__}' and " + f"'{other.__class__.__name__}'" + ) self.code = max(self.code, other.code) self.out += other.out self.err += other.err @@ -113,6 +122,6 @@ def __repr__(self): return f"{self.code}; {self.out}; {self.err}" def error_from_messages(self): - out_lines = (self.out + '\n' + self.err).splitlines() + out_lines = (self.out + "\n" + self.err).splitlines() if any(line.lower().startswith("err") for line in out_lines): self.code = EXIT.ERR diff --git a/vmupdate/agent/source/common/progress_reporter.py b/vmupdate/agent/source/common/progress_reporter.py index e8cfdd2..6e4863c 100644 --- a/vmupdate/agent/source/common/progress_reporter.py +++ b/vmupdate/agent/source/common/progress_reporter.py @@ -27,9 +27,9 @@ class Progress: def __init__( - self, - weight: int, - log, + self, + weight: int, + log, ): self.weight = weight self._callback = None @@ -41,9 +41,12 @@ def __init__( self.log = log def init( - self, start: float, stop: float, - callback: Callable[[float], None], - stdout: io.TextIOWrapper, stderr: io.TextIOWrapper + self, + start: float, + stop: float, + callback: Callable[[float], None], + stdout: io.TextIOWrapper, + stderr: io.TextIOWrapper, ): self._callback = callback self._start_percent = start @@ -57,8 +60,10 @@ def notify_callback(self, percent): Report ongoing progress. """ assert self._start_percent is not None # call init() first! - _percent = self._start_percent + percent * ( - self._stop_percent - self._start_percent) / 100 + _percent = ( + self._start_percent + + percent * (self._stop_percent - self._start_percent) / 100 + ) _percent = round(_percent, 2) if self._last_percent < _percent: self._callback(_percent) @@ -84,20 +89,21 @@ class ProgressReporter: """ def __init__( - self, - update: Progress, - fetch: Progress, - upgrade: Progress, - callback: Optional[Callable[[float], None]] = None + self, + update: Progress, + fetch: Progress, + upgrade: Progress, + callback: Optional[Callable[[float], None]] = None, ): saved_stdout = os.dup(sys.stdout.fileno()) saved_stderr = os.dup(sys.stderr.fileno()) - self.stdout = io.TextIOWrapper(os.fdopen(saved_stdout, 'wb')) - self.stderr = io.TextIOWrapper(os.fdopen(saved_stderr, 'wb')) + self.stdout = io.TextIOWrapper(os.fdopen(saved_stdout, "wb")) + self.stderr = io.TextIOWrapper(os.fdopen(saved_stderr, "wb")) self.last_percent = 0.0 if callback is None: - self.callback = lambda p: \ - print(f"{p:.2f}", flush=True, file=self.stderr) + self.callback = lambda p: print( + f"{p:.2f}", flush=True, file=self.stderr + ) else: self.callback = callback @@ -107,7 +113,8 @@ def __init__( update.init(0, update_end, self.callback, self.stdout, self.stderr) fetch.init( - update_end, fetch_end, self.callback, self.stdout, self.stderr) + update_end, fetch_end, self.callback, self.stdout, self.stderr + ) upgrade.init(fetch_end, 100, self.callback, self.stdout, self.stderr) self.update_progress = update diff --git a/vmupdate/agent/source/dnf/__init__.py b/vmupdate/agent/source/dnf/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/dnf/__init__.py +++ b/vmupdate/agent/source/dnf/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index bd4ef85..38c169f 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -49,12 +49,16 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): conf = self.base.get_config() if self.type == AgentType.UPDATE_VM: - conf.config_file_path = self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" + conf.config_file_path = ( + self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" + ) conf.best = True conf.plugins = False conf.installroot = self.UPDATE_VM_INSTALLROOT - for opt in ('cachedir', 'logdir', 'persistdir'): - setattr(conf, opt, self.UPDATE_VM_INSTALLROOT + getattr(conf, opt)) + for opt in ("cachedir", "logdir", "persistdir"): + setattr( + conf, opt, self.UPDATE_VM_INSTALLROOT + getattr(conf, opt) + ) conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] conf.excludepkgs = ["qubes-template-*"] self.base.load_config() @@ -83,7 +87,8 @@ def refresh(self, hard_fail: bool) -> ProcessResult: result += self.expire_cache() except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) return result @@ -111,37 +116,46 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: if transaction.get_transaction_packages_count() == 0: self.log.info("No packages to upgrade, quitting.") return ProcessResult( - EXIT.OK_NO_UPDATES, out="", - err="\n".join(transaction.get_resolve_logs_as_strings())) + EXIT.OK_NO_UPDATES, + out="", + err="\n".join(transaction.get_resolve_logs_as_strings()), + ) if self.type != AgentType.DOM0: # self.base.set_download_callbacks( libdnf5.repo.DownloadCallbacksUniquePtr( - self.progress.fetch_progress)) + self.progress.fetch_progress + ) + ) transaction.download() if not transaction.check_gpg_signatures(): problems = transaction.get_gpg_signature_problems() raise TransactionError( - f"GPG signatures check failed: {problems}") + f"GPG signatures check failed: {problems}" + ) if result.code == EXIT.OK and self.type is not AgentType.UPDATE_VM: self.log.debug("Committing upgrade...") transaction.set_callbacks( libdnf5.rpm.TransactionCallbacksUniquePtr( - self.progress.upgrade_progress)) + self.progress.upgrade_progress + ) + ) tnx_result = transaction.run() if tnx_result != transaction.TransactionRunResult_SUCCESS: raise TransactionError( - transaction.transaction_result_to_string(tnx_result)) + transaction.transaction_result_to_string(tnx_result) + ) self.log.debug("Package upgrade successful.") if self.type is AgentType.VM: self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) + subprocess.call(["/etc/qubes-rpc/qubes.PostInstall"]) except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) return result @@ -158,7 +172,7 @@ def __init__(self, weight: int, log): self.fetching_notified = False def add_new_download( - self, _user_data, description: str, total_to_download: float + self, _user_data, description: str, total_to_download: float ) -> int: """ Notify the client that a new download has been created. @@ -177,7 +191,7 @@ def add_new_download( return self.count def progress( - self, user_cb_data: int, total_to_download: float, downloaded: float + self, user_cb_data: int, total_to_download: float, downloaded: float ) -> int: """ Download progress callback. @@ -187,22 +201,28 @@ def progress( :param downloaded: Number of bytes downloaded. """ if not self.fetching_notified: - print(f"Fetching {self.count} packages " - f"[{self._format_bytes(self.bytes_to_fetch)}]", - flush=True) + print( + f"Fetching {self.count} packages " + f"[{self._format_bytes(self.bytes_to_fetch)}]", + flush=True, + ) self.fetching_notified = True self.bytes_fetched += downloaded - self.package_bytes[user_cb_data] if downloaded > self.package_bytes[user_cb_data]: if self.package_bytes[user_cb_data] == 0: - print(f"Fetching {self.package_names[user_cb_data]} [{self._format_bytes(total_to_download)}]", - flush=True) + print( + f"Fetching {self.package_names[user_cb_data]} " + f"[{self._format_bytes(total_to_download)}]", + flush=True, + ) self.package_bytes[user_cb_data] = downloaded percent = self.bytes_fetched / self.bytes_to_fetch * 100 self.notify_callback(percent) # Should return 0 on success, # in case anything in dnf5 changed we return their default value return DownloadCallbacks.progress( - self, user_cb_data, total_to_download, downloaded) + self, user_cb_data, total_to_download, downloaded + ) def end(self, user_cb_data: int, status: int, msg: str) -> int: """ @@ -214,13 +234,13 @@ def end(self, user_cb_data: int, status: int, msg: str) -> int: """ if status != 0: if isinstance(msg, bytes): - msg = msg.decode('ascii', errors='ignore') + msg = msg.decode("ascii", errors="ignore") if msg: print(msg, flush=True, file=self._stdout) return DownloadCallbacks.end(self, user_cb_data, status, msg) def mirror_failure( - self, user_cb_data: int, msg: str, url: str, metadata: str + self, user_cb_data: int, msg: str, url: str, metadata: str ) -> int: """ Mirror failure callback. @@ -231,12 +251,16 @@ def mirror_failure( :param metadata: the type of metadata that is being downloaded """ if isinstance(msg, bytes): - msg = msg.decode('ascii', errors='ignore') - print(f"Fetching {metadata} failure " - f"({self.package_names[user_cb_data]}) {msg}", - flush=True, file=self._stdout) + msg = msg.decode("ascii", errors="ignore") + print( + f"Fetching {metadata} failure " + f"({self.package_names[user_cb_data]}) {msg}", + flush=True, + file=self._stdout, + ) return DownloadCallbacks.mirror_failure( - self, user_cb_data, msg, url, metadata) + self, user_cb_data, msg, url, metadata + ) class UpgradeProgress(TransactionCallbacks, Progress): @@ -248,12 +272,13 @@ def __init__(self, weight: int, log): self.processed_packages = set() def install_progress( - self, item: libdnf5.base.TransactionPackage, amount: int, total: int + self, item: libdnf5.base.TransactionPackage, amount: int, total: int ): r""" Report the package installation progress periodically. - :param item: The TransactionPackage class instance for the package currently being installed + :param item: The TransactionPackage class instance for the package + currently being installed :param amount: The portion of the package already installed :param total: The disk space used by the package after installation """ @@ -275,7 +300,7 @@ def transaction_start(self, total: int): self.pgks = total def uninstall_progress( - self, item: libdnf5.base.TransactionPackage, amount: int, total: int + self, item: libdnf5.base.TransactionPackage, amount: int, total: int ): """ Report the package removal progress periodically. @@ -292,27 +317,39 @@ def uninstall_progress( percent = (self.pgks_done + pkg_progress) / self.pgks * 100 self.notify_callback(percent) + # pylint: disable=unused-argument def elem_progress(self, item, amount: int, total: int): r""" The installation/removal process for the item has started - :param item: The TransactionPackage class instance for the package currently being (un)installed - :param amount: Index of the package currently being processed. Items are indexed starting from 0. + :param item: The TransactionPackage class instance for the package + currently being (un)installed + :param amount: Index of the package currently being processed. + Items are indexed starting from 0. :param total: The total number of packages in the transaction """ self.pgks_done = amount percent = amount / total * 100 self.notify_callback(percent) - def script_start(self, item: libdnf5.base.TransactionPackage, nevra, type: int): + # pylint: disable=unused-argument,redefined-builtin + def script_start( + self, item: libdnf5.base.TransactionPackage, nevra, type: int + ): r""" Execution of the rpm scriptlet has started - :param item: The TransactionPackage class instance for the package that owns the executed or triggered - scriptlet. It can be `nullptr` if the scriptlet owner is not part of the transaction - (e.g., a package installation triggered an update of the man database, owned by man-db package). - :param nevra: Nevra of the package that owns the executed or triggered scriptlet. + :param item: The TransactionPackage class instance for the package that + owns the executed or triggered scriptlet. It can be + `nullptr` if the scriptlet owner is not part of + the transaction (e.g., a package installation triggered + an update of the man database, owned by man-db package). + :param nevra: Nevra of the package that owns the executed or triggered + scriptlet. :param type: Type of the scriptlet """ - print(f"Running rpm scriptlet for {nevra.get_name()}-{nevra.get_epoch()}:{nevra.get_version()}" - f"-{nevra.get_release()}.{nevra.get_arch()}", flush=True) + print( + f"Running rpm scriptlet for {nevra.get_name()}-{nevra.get_epoch()}" + f":{nevra.get_version()}-{nevra.get_release()}.{nevra.get_arch()}", + flush=True, + ) diff --git a/vmupdate/agent/source/dnf/dnf_api.py b/vmupdate/agent/source/dnf/dnf_api.py index 03c58a6..adc9c23 100644 --- a/vmupdate/agent/source/dnf/dnf_api.py +++ b/vmupdate/agent/source/dnf/dnf_api.py @@ -53,7 +53,7 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): conf.best = True conf.plugins = False conf.installroot = self.UPDATE_VM_INSTALLROOT - for opt in ('cachedir', 'logdir', 'persistdir'): + for opt in ("cachedir", "logdir", "persistdir"): conf.prepend_installroot(opt) conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] conf.excludepkgs = ["qubes-template-*"] @@ -63,20 +63,21 @@ def __init__(self, log_handler, log_level, agent_type: AgentType): log_file = os.path.join(log_dir, "hawkey.log") os.makedirs(log_dir, exist_ok=True) if not os.path.exists(log_file): - with open(log_file, 'w'): + with open(log_file, "w"): pass # Passing `conf` to `base` causes `releasever` not to be set subst = conf.substitutions - if 'releasever' not in subst: - releasever = dnf.rpm.detect_releasever(conf.installroot) - subst['releasever'] = releasever + if "releasever" not in subst: + subst["releasever"] = dnf.rpm.detect_releasever(conf.installroot) self.base = dnf.Base(conf) # Repositories serve as sources of information about packages. self.base.read_all_repos() - update = FetchProgress(weight=10, log=self.log, refresh=True) # % of total time + update = FetchProgress( + weight=10, log=self.log, refresh=True + ) # % of total time fetch = FetchProgress(weight=45, log=self.log) # % of total time upgrade = UpgradeProgress(weight=45, log=self.log) # % of total time self.progress = ProgressReporter(update, fetch, upgrade) @@ -95,7 +96,7 @@ def refresh(self, hard_fail: bool) -> ProcessResult: repos = tuple(self.base.repos.iter_enabled()) # we do not know the size of the repositories self.progress.update_progress.start(len(repos), len(repos)) - for i, repo in enumerate(repos): + for repo in repos: self.progress.update_progress.progress(repo.id, 1) repo.load() self.progress.update_progress.end(repo.id, 0, "") @@ -107,7 +108,8 @@ def refresh(self, hard_fail: bool) -> ProcessResult: result += ProcessResult(EXIT.ERR_VM_REFRESH) except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) return result @@ -135,8 +137,7 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: return ProcessResult(EXIT.OK_NO_UPDATES, out="", err="") self.base.download_packages( - trans.install_set, - progress=self.progress.fetch_progress + trans.install_set, progress=self.progress.fetch_progress ) result += sign_check(self.base, trans.install_set, self.log) @@ -147,11 +148,12 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: self.log.debug("Package upgrade successful.") if self.type is AgentType.VM: self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) + subprocess.call(["/etc/qubes-rpc/qubes.PostInstall"]) print("Updated", flush=True) except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) finally: self.base.close() @@ -202,7 +204,7 @@ def end(self, payload, status, msg): """ if status != 0: if isinstance(msg, bytes): - msg = msg.decode('ascii', errors='ignore') + msg = msg.decode("ascii", errors="ignore") if msg: print(msg, flush=True, file=self._stdout) else: @@ -210,7 +212,7 @@ def end(self, payload, status, msg): def message(self, msg): if isinstance(msg, bytes): - msg = msg.decode('ascii', errors='ignore') + msg = msg.decode("ascii", errors="ignore") if msg: print(msg, flush=True, file=self._stdout) @@ -238,9 +240,11 @@ def start(self, total_files, total_size, total_drpms=0): if self.action == "refresh": print("Refreshing available packages.", flush=True) else: - print(f"Fetching {total_files} packages " - f"[{self._format_bytes(self.bytes_to_fetch)}]", - flush=True) + print( + f"Fetching {total_files} packages " + f"[{self._format_bytes(self.bytes_to_fetch)}]", + flush=True, + ) self.package_bytes = {} self.notify_callback(0) @@ -250,8 +254,7 @@ def __init__(self, weight: int, log): TransactionDisplay.__init__(self) Progress.__init__(self, weight, log) - def progress(self, _package, action, ti_done, ti_total, ts_done, - ts_total): + def progress(self, _package, action, ti_done, ti_total, ts_done, ts_total): """ Report ongoing progress on a transaction item. @@ -276,7 +279,7 @@ def scriptout(self, msgs): """ if msgs: if isinstance(msgs, bytes): - msgs = msgs.decode('ascii', errors='ignore') + msgs = msgs.decode("ascii", errors="ignore") print(msgs, flush=True) def filelog(self, package, action): @@ -287,6 +290,9 @@ def error(self, message): Write an error message to the fake stderr. """ if isinstance(message, bytes): - message = message.decode('ascii', errors='ignore') - print("Error during installation :" + message, - flush=True, file=self._stderr) + message = message.decode("ascii", errors="ignore") + print( + "Error during installation :" + message, + flush=True, + file=self._stderr, + ) diff --git a/vmupdate/agent/source/dnf/dnf_cli.py b/vmupdate/agent/source/dnf/dnf_cli.py index a3db5a5..afa8d1d 100644 --- a/vmupdate/agent/source/dnf/dnf_cli.py +++ b/vmupdate/agent/source/dnf/dnf_cli.py @@ -33,13 +33,13 @@ class DNFCLI(PackageManager): def __init__(self, log_handler, log_level, agent_type: AgentType): super().__init__(log_handler, log_level, agent_type) - pck_mng_path = shutil.which('dnf') + pck_mng_path = shutil.which("dnf") if pck_mng_path is not None: - pck_mngr = 'dnf' + pck_mngr = "dnf" else: - pck_mng_path = shutil.which('yum') + pck_mng_path = shutil.which("yum") if pck_mng_path is not None: - pck_mngr = 'yum' + pck_mngr = "yum" else: raise RuntimeError("Package manager not found!") self.package_manager: str = pck_mngr @@ -53,17 +53,21 @@ def refresh(self, hard_fail: bool) -> ProcessResult: """ result = self.expire_cache() - cmd = [self.package_manager, - "-q", - "check-update", - "--assumeyes", - f"--setopt=skip_if_unavailable={int(not hard_fail)}"] + cmd = [ + self.package_manager, + "-q", + "check-update", + "--assumeyes", + f"--setopt=skip_if_unavailable={int(not hard_fail)}", + ] if self.type != AgentType.UPDATE_VM: # In UpdateVM we use preconfigured repos result_check = self.run_cmd(cmd) # ret_code == 100 is not an error # It means there are packages to be updated - result_check.code = result_check.code if result_check.code != 100 else 0 + result_check.code = ( + result_check.code if result_check.code != 100 else 0 + ) result += result_check result.error_from_messages() @@ -73,10 +77,7 @@ def expire_cache(self) -> ProcessResult: """ Use package manager to expire cache. """ - cmd = [self.package_manager, - "-q", - "clean", - "expire-cache"] + cmd = [self.package_manager, "-q", "clean", "expire-cache"] if self.type != AgentType.UPDATE_VM: result = self.run_cmd(cmd) else: @@ -113,17 +114,22 @@ def get_action(self, remove_obsolete) -> List[str]: """ result = ["-y"] if self.type is AgentType.UPDATE_VM: - result.extend(["upgrade", - "--noplugins", - "--best", - "--allowerasing", - "--downloadonly", - "--installroot", self.UPDATE_VM_INSTALLROOT, - f"--setopt=cachedir={self.UPDATE_VM_INSTALLROOT}/var/cache/dnf", - f"--config={self.UPDATE_VM_INSTALLROOT}/etc/dnf/dnf.conf", - f"--setopt=reposdir={self.UPDATE_VM_INSTALLROOT}/etc/yum.repos.d", - "--exclude=qubes-template-*", "-y" - ]) + result.extend( + [ + "upgrade", + "--noplugins", + "--best", + "--allowerasing", + "--downloadonly", + "--installroot", + self.UPDATE_VM_INSTALLROOT, + f"--setopt=cachedir={self.UPDATE_VM_INSTALLROOT}/var/cache/dnf", + f"--config={self.UPDATE_VM_INSTALLROOT}/etc/dnf/dnf.conf", + f"--setopt=reposdir={self.UPDATE_VM_INSTALLROOT}/etc/yum.repos.d", + "--exclude=qubes-template-*", + "-y", + ] + ) return result if remove_obsolete: result.extend(["--setopt=obsoletes=1", "upgrade"]) diff --git a/vmupdate/agent/source/log_config.py b/vmupdate/agent/source/log_config.py index 9a4c2be..dcac508 100644 --- a/vmupdate/agent/source/log_config.py +++ b/vmupdate/agent/source/log_config.py @@ -24,18 +24,18 @@ import grp from pathlib import Path -LOGPATH = '/var/log/qubes/qubes-update' -FORMAT_LOG = '%(asctime)s [Agent] %(message)s' -LOG_FILE = 'update-agent.log' +LOGPATH = "/var/log/qubes/qubes-update" +FORMAT_LOG = "%(asctime)s [Agent] %(message)s" +LOG_FILE = "update-agent.log" def init_logs( - directory=LOGPATH, - file=LOG_FILE, - format_=FORMAT_LOG, - level="INFO", - truncate_file=False, - qname=None, + directory=LOGPATH, + file=LOG_FILE, + format_=FORMAT_LOG, + level="INFO", + truncate_file=False, + qname=None, ): Path(directory).mkdir(parents=True, exist_ok=True) log_path = os.path.join(directory, file) @@ -46,14 +46,14 @@ def init_logs( # Persistent logs are at dom0. pass - log_handler = logging.FileHandler(log_path, encoding='utf-8') + log_handler = logging.FileHandler(log_path, encoding="utf-8") log_formatter = logging.Formatter(format_) log_handler.setFormatter(log_formatter) if qname is not None: log = logging.getLogger(qname) else: - log = logging.getLogger('vm-update.agent.PackageManager') + log = logging.getLogger("vm-update.agent.PackageManager") log.addHandler(log_handler) log.propagate = False try: diff --git a/vmupdate/agent/source/pacman/__init__.py b/vmupdate/agent/source/pacman/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/pacman/__init__.py +++ b/vmupdate/agent/source/pacman/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/pacman/pacman_cli.py b/vmupdate/agent/source/pacman/pacman_cli.py index ad218d5..e0b9764 100644 --- a/vmupdate/agent/source/pacman/pacman_cli.py +++ b/vmupdate/agent/source/pacman/pacman_cli.py @@ -31,6 +31,7 @@ def __init__(self, log_handler, log_level): super().__init__(log_handler, log_level) self.package_manager = "pacman" + # pylint: disable=unused-argument def refresh(self, hard_fail: bool) -> ProcessResult: """ Use package manager to refresh available packages. @@ -61,6 +62,7 @@ def get_packages(self) -> Dict[str, List[str]]: return packages + # pylint: disable=unused-argument def get_action(self, remove_obsolete) -> List[str]: """ Pacman will handle obsoletions itself diff --git a/vmupdate/agent/source/plugins/__init__.py b/vmupdate/agent/source/plugins/__init__.py index e254f34..e5e6399 100644 --- a/vmupdate/agent/source/plugins/__init__.py +++ b/vmupdate/agent/source/plugins/__init__.py @@ -1,12 +1,18 @@ import importlib import os.path import glob + modules = sorted(glob.glob(os.path.join(os.path.dirname(__file__), "*.py"))) -__all__ = [os.path.basename(f)[:-3] - for f in modules if os.path.isfile(f) - and not f.endswith('__init__.py')] -modules = [importlib.import_module("source.plugins." + name) - for name in __all__] -entrypoints = [getattr(module, name) - for name, module in zip(__all__, modules) - if callable(getattr(module, name))] +__all__ = [ + os.path.basename(f)[:-3] + for f in modules + if os.path.isfile(f) and not f.endswith("__init__.py") +] +modules = [ + importlib.import_module("source.plugins." + name) for name in __all__ +] +entrypoints = [ + getattr(module, name) + for name, module in zip(__all__, modules) + if callable(getattr(module, name)) +] diff --git a/vmupdate/agent/source/plugins/allow_release_info_change.py b/vmupdate/agent/source/plugins/allow_release_info_change.py index 9816746..6b5ec4c 100644 --- a/vmupdate/agent/source/plugins/allow_release_info_change.py +++ b/vmupdate/agent/source/plugins/allow_release_info_change.py @@ -22,6 +22,7 @@ APT_CONF = "/etc/apt/apt.conf.d/01qubes-update" +# pylint: disable=unused-argument def allow_release_info_change(os_data, log, **kwargs): """ Add apt conf file to disable `AllowReleaseInfoChange` for `buster`. @@ -32,5 +33,5 @@ def allow_release_info_change(os_data, log, **kwargs): if os_data.get("codename", "") == "buster": option = 'Acquire::AllowReleaseInfoChange "false"' log.info("Set %s as workaround for buster.", option) - with open(APT_CONF, "w") as file: - file.write(f'\n{option};\n') + with open(APT_CONF, "wt") as file: + file.write(f"\n{option};\n") diff --git a/vmupdate/agent/source/plugins/bookworm_backports.py b/vmupdate/agent/source/plugins/bookworm_backports.py index fdbe3bb..1abc353 100644 --- a/vmupdate/agent/source/plugins/bookworm_backports.py +++ b/vmupdate/agent/source/plugins/bookworm_backports.py @@ -87,6 +87,7 @@ def check_package_not_from_backports(package): return False +# pylint: disable=unused-argument def bookworm_backports(os_data, log, **kwargs): """ Update firmware and/or pipewire packages from backports repository. diff --git a/vmupdate/agent/source/plugins/disable_deltarpm.py b/vmupdate/agent/source/plugins/disable_deltarpm.py index 4214a21..6bb19af 100644 --- a/vmupdate/agent/source/plugins/disable_deltarpm.py +++ b/vmupdate/agent/source/plugins/disable_deltarpm.py @@ -20,6 +20,7 @@ # USA. +# pylint: disable=unused-argument def disable_deltarpm(os_data, log, dnf_conf="/etc/dnf/dnf.conf", **kwargs): """ Modify dnf.conf file to disable `deltarpm`. diff --git a/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py b/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py index 91254b4..37855bc 100644 --- a/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py +++ b/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py @@ -3,6 +3,7 @@ import signal +# pylint: disable=unused-argument def fix_meminfo_writer_label(os_data, log, **kwargs): """ Fix meminfo-writer SELinux label to make memory ballooning work again @@ -42,7 +43,9 @@ def fix_meminfo_writer_label(os_data, log, **kwargs): if label_changed: try: - with open("/run/meminfo-writer.pid", "r", encoding="utf-8") as f: + with open( + "/run/meminfo-writer.pid", "r", encoding="utf-8" + ) as f: target_pid = int(f.read().strip()) os.kill(target_pid, signal.SIGUSR1) log.info( diff --git a/vmupdate/agent/source/plugins/manage_rpm_macro.py b/vmupdate/agent/source/plugins/manage_rpm_macro.py index dee42eb..d766acf 100644 --- a/vmupdate/agent/source/plugins/manage_rpm_macro.py +++ b/vmupdate/agent/source/plugins/manage_rpm_macro.py @@ -36,8 +36,9 @@ def manage_rpm_macro(os_data, log, **kwargs): if version < 33: log.info("Old fedora version detected.") with open(rpm_macro, "w") as file: - file.write("# CVE-2021-20271 mitigation\n" - "%_pkgverify_level all") + file.write( + "# CVE-2021-20271 mitigation\n" "%_pkgverify_level all" + ) else: if os.path.exists(rpm_macro): os.remove(rpm_macro) diff --git a/vmupdate/agent/source/plugins/pipewire_archlinux.py b/vmupdate/agent/source/plugins/pipewire_archlinux.py index ebf16d2..5bb29bd 100644 --- a/vmupdate/agent/source/plugins/pipewire_archlinux.py +++ b/vmupdate/agent/source/plugins/pipewire_archlinux.py @@ -19,15 +19,18 @@ import subprocess + +# pylint: disable=unused-argument def pipewire_archlinux(os_data, log, **kwargs): """Help with unattended switch from pulseaudio to pipewire-pulse""" # pacman proposes to remove pulseaudio when installing pipewire-pulse, # but the default answer is "n", so the update with --noconfirm fails # workaround it by removing pulseaudio before the update - if os_data["os_family"] != 'ArchLinux': + if os_data["os_family"] != "ArchLinux": return # check if pulseaudio is installed - p = subprocess.call(["pacman", "-Q", "pulseaudio"], + p = subprocess.call( + ["pacman", "-Q", "pulseaudio"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) @@ -36,12 +39,15 @@ def pipewire_archlinux(os_data, log, **kwargs): # ... and whether pipewire-pulse is going to be installed in the update # this will refresh metadata already, before starting progress reporting, # but well... - update_list = subprocess.check_output(["pacman", "-Syup"], - stderr=subprocess.DEVNULL).decode() + update_list = subprocess.check_output( + ["pacman", "-Syup"], stderr=subprocess.DEVNULL + ).decode() if not any("/pipewire-pulse-" in line for line in update_list.splitlines()): return # ... then remove pulseaudio beforehand (temporarily breaking the # dependencies) - log.info("Removing pulseaudio to allow update cleanly migrate to " - "pipewire-pulse") + log.info( + "Removing pulseaudio to allow update cleanly migrate to " + "pipewire-pulse" + ) subprocess.check_call(["pacman", "-Rdd", "--noconfirm", "pulseaudio"]) diff --git a/vmupdate/agent/source/plugins/updatesproxy_fix.py b/vmupdate/agent/source/plugins/updatesproxy_fix.py index 74f387a..cf49488 100644 --- a/vmupdate/agent/source/plugins/updatesproxy_fix.py +++ b/vmupdate/agent/source/plugins/updatesproxy_fix.py @@ -20,10 +20,10 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -import os import pathlib +# pylint: disable=unused-argument def updatesproxy_fix(os_data, log, **kwargs): """ Deploy #9025 fix diff --git a/vmupdate/agent/source/status.py b/vmupdate/agent/source/status.py index 6c83016..08d0c02 100644 --- a/vmupdate/agent/source/status.py +++ b/vmupdate/agent/source/status.py @@ -68,4 +68,4 @@ def __init__(self, qube_name, stream: str, message: str): self.message = message def __str__(self): - return f"{self.qname}:{self.stream}: {self.message}" \ No newline at end of file + return f"{self.qname}:{self.stream}: {self.message}" diff --git a/vmupdate/agent/source/utils.py b/vmupdate/agent/source/utils.py index 21678af..e64f3e8 100644 --- a/vmupdate/agent/source/utils.py +++ b/vmupdate/agent/source/utils.py @@ -38,9 +38,7 @@ def get_os_data(logger: Optional = None) -> Dict[str, Any]: data = {} os_release = _load_os_release( - "/etc/os-release", - "/usr/lib/os-release", - logger=logger + "/etc/os-release", "/usr/lib/os-release", logger=logger ) data["id"] = os_release.get("ID", "linux").strip() @@ -51,23 +49,25 @@ def get_os_data(logger: Optional = None) -> Dict[str, Any]: if "VERSION_CODENAME" in os_release: data["codename"] = os_release["VERSION_CODENAME"] - family = [os_release.get('ID', 'linux'), - *os_release.get('ID_LIKE', '').split()] + family = [ + os_release.get("ID", "linux"), + *os_release.get("ID_LIKE", "").split(), + ] - data["os_family"] = 'Unknown' + data["os_family"] = "Unknown" - if 'debian' in family: - data["os_family"] = 'Debian' + if "debian" in family: + data["os_family"] = "Debian" - if 'rhel' in family or 'fedora' in family: - data["os_family"] = 'RedHat' + if "rhel" in family or "fedora" in family: + data["os_family"] = "RedHat" - if 'qubes' in family: + if "qubes" in family: # We do not want to use 'RedHat' for dom0 since usually plugins do not apply to dom0 - data["os_family"] = 'Qubes' + data["os_family"] = "Qubes" - if 'arch' in family: - data["os_family"] = 'ArchLinux' + if "arch" in family: + data["os_family"] = "ArchLinux" return data @@ -84,19 +84,23 @@ def _load_os_release(*os_release_files, logger: Optional): with open(filename) as file: for line_number, line in enumerate(file): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - match = re.match(r'([A-Z][A-Z_0-9]+)=(.*)', line) + match = re.match(r"([A-Z][A-Z_0-9]+)=(.*)", line) if match: key, val = match.groups() - if val and val[0] in '"\'': + if val and val[0] in "\"'": val = ast.literal_eval(val) result[key] = val else: if logger: - logger.error("%s:%i: error in parsing: %s", - filename, line_number, line) + logger.error( + "%s:%i: error in parsing: %s", + filename, + line_number, + line, + ) break except OSError as exc: if logger: diff --git a/vmupdate/qube_connection.py b/vmupdate/qube_connection.py index 5b03192..543974b 100644 --- a/vmupdate/qube_connection.py +++ b/vmupdate/qube_connection.py @@ -49,13 +49,7 @@ class QubeConnection: PYTHON_PATH = "/usr/bin/python3" def __init__( - self, - qube, - dest_dir, - cleanup, - logger, - show_progress, - status_notifier + self, qube, dest_dir, cleanup, logger, show_progress, status_notifier ): self.qube = qube self.dest_dir = dest_dir @@ -84,16 +78,20 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.status_notifier.put(StatusInfo.done(self.qube, self.status)) if self.cleanup: - self.logger.info('Remove %s', self.dest_dir) + self.logger.info("Remove %s", self.dest_dir) try: self._run_shell_command_in_qube( - self.qube, ['rm', '-r', self.dest_dir]) + self.qube, ["rm", "-r", self.dest_dir] + ) except Exception as err: - self.logger.error('Cannot remove %s, because of error: %s', - self.dest_dir, str(err)) + self.logger.error( + "Cannot remove %s, because of error: %s", + self.dest_dir, + str(err), + ) if self.qube.is_running() and not self._initially_running: - self.logger.info('Shutdown %s', self.qube.name) + self.logger.info("Shutdown %s", self.qube.name) self.qube.shutdown() self.__connected = False @@ -113,11 +111,14 @@ def transfer_agent(self, src_dir: str) -> ProcessResult: base_dir = os.path.basename(src_dir.strip(os.sep)) src_arch = join(arch_dir, base_dir + arch_format) dest_arch = join(self.dest_dir, base_dir + arch_format) - shutil.make_archive(base_name=join(arch_dir, base_dir), - format='gztar', root_dir=root_dir, - base_dir=base_dir) - - command = ['mkdir', '-p', self.dest_dir] + shutil.make_archive( + base_name=join(arch_dir, base_dir), + format="gztar", + root_dir=root_dir, + base_dir=base_dir, + ) + + command = ["mkdir", "-p", self.dest_dir] result = self._run_shell_command_in_qube(self.qube, command) if result: return result @@ -135,12 +136,13 @@ def _copy_file_from_dom0(self, src, dest) -> ProcessResult: command = " ".join(write_dest) self.logger.debug("run command: %s < %s", command, src) try: - with open(src, 'rb') as file: + with open(src, "rb") as file: untrusted_stdout_and_stderr = self.qube.run( - command, user='root', input=file.read() + command, user="root", input=file.read() ) result = ProcessResult.from_untrusted_out_err( - *untrusted_stdout_and_stderr) + *untrusted_stdout_and_stderr + ) if result.code: raise OSError(f"Command returns code: {result.code}") except OSError as exc: @@ -149,7 +151,7 @@ def _copy_file_from_dom0(self, src, dest) -> ProcessResult: return result def run_entrypoint( - self, entrypoint_path: str | List, agent_args + self, entrypoint_path: str | List, agent_args ) -> ProcessResult: """ Run a script in the qube. @@ -159,13 +161,17 @@ def run_entrypoint( :return: return code and output of the script """ if isinstance(entrypoint_path, str): - command = [QubeConnection.PYTHON_PATH, entrypoint_path, - *AgentArgs.to_cli_args(agent_args)] + command = [ + QubeConnection.PYTHON_PATH, + entrypoint_path, + *AgentArgs.to_cli_args(agent_args), + ] else: command = entrypoint_path result = self._run_shell_command_in_qube( - self.qube, command, show=self.show_progress) + self.qube, command, show=self.show_progress + ) return result @@ -173,45 +179,45 @@ def read_logs(self) -> ProcessResult: """ Read vm logs file. """ - command = ['cat', - str(join(LOGPATH, LOG_FILE))] + command = ["cat", str(join(LOGPATH, LOG_FILE))] result = self._run_shell_command_in_qube(self.qube, command) return result def _run_shell_command_in_qube( - self, target, command: List[str], show: bool = False + self, target, command: List[str], show: bool = False ) -> ProcessResult: - self.logger.debug("run command in %s: %s", - target.name, " ".join(command)) + self.logger.debug( + "run command in %s: %s", target.name, " ".join(command) + ) if not show: return self._run_command_and_wait_for_output(target, command) - else: - return self._run_command_and_actively_report_progress( - target, command) + return self._run_command_and_actively_report_progress(target, command) def _run_command_and_wait_for_output( - self, target, command: List[str] + self, target, command: List[str] ) -> ProcessResult: self.logger.debug("Wait for output") result = ProcessResult() try: if self.qube.klass == "AdminVM": - proc = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) untrusted_stdout_and_stderr = proc.communicate() if proc.returncode: raise CalledProcessError( proc.returncode, command, - output=untrusted_stdout_and_stderr[0] + - untrusted_stdout_and_stderr[1] + output=untrusted_stdout_and_stderr[0] + + untrusted_stdout_and_stderr[1], ) else: untrusted_stdout_and_stderr = target.run_with_args( - *command, user='root' + *command, user="root" ) result += ProcessResult.from_untrusted_out_err( - *untrusted_stdout_and_stderr) + *untrusted_stdout_and_stderr + ) except CalledProcessError as err: if err.returncode == 100: self.status = FinalStatus.NO_UPDATES @@ -220,30 +226,34 @@ def _run_command_and_wait_for_output( self.logger.error(str(err)) ret_code = err.returncode result = ProcessResult.from_untrusted_out_err( - err.output, err.output) + err.output, err.output + ) result.code = ret_code except Exception as err: result = ProcessResult(1, "", str(err)) return result def _run_command_and_actively_report_progress( - self, target, command: List[str] + self, target, command: List[str] ) -> ProcessResult: self.logger.debug("Progress reporting enabled.") if self.qube.klass == "AdminVM": - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) elif "--download-only" in command: # for download-only commands, run with fakeroot proc = target.run_service( - 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(["fakeroot"] + command), - user='user', - preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + "qubes.VMExec+" + + qubesadmin.utils.encode_for_vmexec(["fakeroot"] + command), + user="user", + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN), ) else: proc = target.run_service( - 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(command), - user='root', - preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) + "qubes.VMExec+" + qubesadmin.utils.encode_for_vmexec(command), + user="root", + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN), ) self.logger.debug("Fetching agent process stdout/stderr.") @@ -253,7 +263,8 @@ def _run_command_and_actively_report_progress( future_out = executor.submit(self._collect_stdout, proc=proc) result = ProcessResult.from_untrusted_out_err( - future_out.result(), future_err.result()) + future_out.result(), future_err.result() + ) result.code = proc.wait() self.logger.debug("Agent process finished.") @@ -264,7 +275,7 @@ def _run_command_and_actively_report_progress( def _collect_stderr(self, proc) -> bytes: progress_finished = False - for untrusted_line in iter(proc.stderr.readline, b''): + for untrusted_line in iter(proc.stderr.readline, b""): if not untrusted_line: continue line = ProcessResult.sanitize_output(untrusted_line, single=True) @@ -277,30 +288,38 @@ def _collect_stderr(self, proc) -> bytes: try: progress = float(line.split()[-1]) except ValueError: - self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) + self.status_notifier.put( + FormatedLine(self.qube.name, "err", line) + ) continue - if progress == 100.: + if progress == 100.0: progress_finished = True self.status_notifier.put( - StatusInfo.updating(self.qube, progress)) + StatusInfo.updating(self.qube, progress) + ) else: - self.status_notifier.put(FormatedLine(self.qube.name, 'err', line)) + self.status_notifier.put( + FormatedLine(self.qube.name, "err", line) + ) proc.stderr.close() self.logger.debug("Agent stderr closed.") - return b'' + return b"" def _collect_stdout(self, proc) -> bytes: - for untrusted_line in iter(proc.stdout.readline, b''): + for untrusted_line in iter(proc.stdout.readline, b""): if untrusted_line: line = ProcessResult.sanitize_output( - untrusted_line, single=True) + untrusted_line, single=True + ) if line: - self.status_notifier.put(FormatedLine(self.qube.name, 'out', line)) + self.status_notifier.put( + FormatedLine(self.qube.name, "out", line) + ) proc.stdout.close() self.logger.debug("Agent stdout closed.") - return b'' + return b"" diff --git a/vmupdate/tests/conftest.py b/vmupdate/tests/conftest.py index 57e25b6..f6401fb 100644 --- a/vmupdate/tests/conftest.py +++ b/vmupdate/tests/conftest.py @@ -44,7 +44,7 @@ def __init__(self, name, app, klass, template=None, **kwargs): self.app.domains[name] = self self.klass = klass self.running = True - if self.klass in ('AppVM', 'DispVM'): + if self.klass in ("AppVM", "DispVM"): template.derived_vms.append(self) self.derived_vms = [] self.auto_cleanup = False @@ -130,12 +130,12 @@ def __init__(self, app, qube, agent_args, show_progress, dom0): def run_agent(self, agent_args, status_notifier, termination): if self.qube.name not in results: status_notifier.put( - StatusInfo.done(self.qube, FinalStatus.UNKNOWN)) + StatusInfo.done(self.qube, FinalStatus.UNKNOWN) + ) unexpected.append(self.qube.name) return ProcessResult(code=99) for status in results[self.qube.name]["statuses"]: - status_notifier.put( - StatusInfo.done(self.qube, status)) + status_notifier.put(StatusInfo.done(self.qube, status)) result = ProcessResult(code=results[self.qube.name]["retcode"]) del results[self.qube.name] return result @@ -149,41 +149,75 @@ def generate_vm_variations(app, variations): """ Generate all possible variations of vms for the given list of features. """ - dom0 = TestVM("dom0", app, klass="AdminVM", updateable=True, running=True, - update_result=FinalStatus.UNKNOWN, - features=Features("dom0", app, {'updates-available': True})) + dom0 = TestVM( + "dom0", + app, + klass="AdminVM", + updateable=True, + running=True, + update_result=FinalStatus.UNKNOWN, + features=Features("dom0", app, {"updates-available": True}), + ) domains = { - "klass": {"TemplateVM": set(), "StandaloneVM": set(), "AppVM": set(), - "DispVM": set()}, + "klass": { + "TemplateVM": set(), + "StandaloneVM": set(), + "AppVM": set(), + "DispVM": set(), + }, "is_running": {False: set(), True: set()}, "servicevm": {False: set(), True: set()}, "auto_cleanup": {False: set(), True: set()}, "updatable": {True: set(), False: set()}, "updates_available": {False: set(), True: set()}, - "last_updates_check": {None: set(), '2020-01-01 00:00:00': set(), - '3020-01-01 00:00:00': set()}, + "last_updates_check": { + None: set(), + "2020-01-01 00:00:00": set(), + "3020-01-01 00:00:00": set(), + }, "qrexec": {False: set(), True: set()}, - "os": {'Linux': set(), 'BSD': set()}, - "updated": {FinalStatus.UNKNOWN: set(), FinalStatus.SUCCESS: set(), - FinalStatus.NO_UPDATES: set(), FinalStatus.ERROR: set(), - FinalStatus.CANCELLED: set(), - }, + "os": {"Linux": set(), "BSD": set()}, + "updated": { + FinalStatus.UNKNOWN: set(), + FinalStatus.SUCCESS: set(), + FinalStatus.NO_UPDATES: set(), + FinalStatus.ERROR: set(), + FinalStatus.CANCELLED: set(), + }, "has_template_updated": { - FinalStatus.SUCCESS: set(), FinalStatus.NO_UPDATES: set(), - FinalStatus.ERROR: set(), FinalStatus.CANCELLED: set(), - FinalStatus.UNKNOWN: set()}, + FinalStatus.SUCCESS: set(), + FinalStatus.NO_UPDATES: set(), + FinalStatus.ERROR: set(), + FinalStatus.CANCELLED: set(), + FinalStatus.UNKNOWN: set(), + }, } - klasses = list(reversed(sorted(list(domains['klass'].keys())))) + klasses = list(reversed(sorted(list(domains["klass"].keys())))) if "klass" not in variations: klasses = klasses[:1] - rest = [list(domains[key].keys()) - if key in variations else list(domains[key].keys())[:1] - for key in domains.keys() if key != "klass"] + rest = [ + ( + list(domains[key].keys()) + if key in variations + else list(domains[key].keys())[:1] + ) + for key in domains.keys() + if key != "klass" + ] for k in klasses: - for (running, servicevm, auto_cleanup, updatable, updates_available, - last_check, qrexec, os, updated, template_updated - ) in itertools.product(*rest): + for ( + running, + servicevm, + auto_cleanup, + updatable, + updates_available, + last_check, + qrexec, + os, + updated, + template_updated, + ) in itertools.product(*rest): if not updatable and (updates_available or last_check): # do not consider features about updates for non-updatable vms @@ -199,19 +233,34 @@ def generate_vm_variations(app, variations): # `template_updated` continue - lc_enc = {None: '0', '2020-01-01 00:00:00': '1', - '3020-01-01 00:00:00': '2'} - os_enc = {'Linux': '0', 'BSD': '1'} - f_map = {FinalStatus.SUCCESS: "0", FinalStatus.ERROR: "1", - FinalStatus.CANCELLED: "2", FinalStatus.NO_UPDATES: "3", - FinalStatus.UNKNOWN: "4"} + lc_enc = { + None: "0", + "2020-01-01 00:00:00": "1", + "3020-01-01 00:00:00": "2", + } + os_enc = {"Linux": "0", "BSD": "1"} + f_map = { + FinalStatus.SUCCESS: "0", + FinalStatus.ERROR: "1", + FinalStatus.CANCELLED: "2", + FinalStatus.NO_UPDATES: "3", + FinalStatus.UNKNOWN: "4", + } txt = lambda x: str(int(x)) - suffix = (txt(running) + txt(servicevm) + lc_enc[last_check] + - txt(updates_available) + txt(qrexec) + os_enc[os] + - txt(updatable) + txt(auto_cleanup)) - if k in ('DispVM', 'AppVM'): + suffix = ( + txt(running) + + txt(servicevm) + + lc_enc[last_check] + + txt(updates_available) + + txt(qrexec) + + os_enc[os] + + txt(updatable) + + txt(auto_cleanup) + ) + if k in ("DispVM", "AppVM"): template = app.domains[ - 'T' + f_map[template_updated] + "4" + suffix[:-1] + "0"] + "T" + f_map[template_updated] + "4" + suffix[:-1] + "0" + ] ext_suffix = f_map[updated] + f_map[template_updated] + suffix update_result = updated else: @@ -221,21 +270,27 @@ def generate_vm_variations(app, variations): features = {} if servicevm: - features['servicevm'] = True + features["servicevm"] = True if updates_available: - features['updates-available'] = True + features["updates-available"] = True if last_check: - features['last-updates-check'] = last_check + features["last-updates-check"] = last_check if qrexec: - features['qrexec'] = qrexec + features["qrexec"] = qrexec if os: - features['os'] = os + features["os"] = os vm = TestVM( - k[0] + ext_suffix, app, klass=k, updateable=updatable, - running=running, auto_cleanup=auto_cleanup, template=template, + k[0] + ext_suffix, + app, + klass=k, + updateable=updatable, + running=running, + auto_cleanup=auto_cleanup, + template=template, features=Features(k[0] + ext_suffix, app, features), - update_result=update_result) + update_result=update_result, + ) domains["klass"][k].add(vm) domains["is_running"][running].add(vm) @@ -246,7 +301,7 @@ def generate_vm_variations(app, variations): domains["last_updates_check"][last_check].add(vm) domains["qrexec"][qrexec].add(vm) domains["os"][os].add(vm) - if k in ('DispVM', 'AppVM'): + if k in ("DispVM", "AppVM"): domains["updated"][updated].add(vm) domains["has_template_updated"][template_updated].add(vm) else: @@ -255,10 +310,15 @@ def generate_vm_variations(app, variations): domains["klass"]["AdminVM"] = {dom0} dom_prop = { - "is_running": True, "servicevm": False, "auto_cleanup": False, - "updatable": True, "updates_available": True, - "last_updates_check": None, "updated": FinalStatus.UNKNOWN, - "has_template_updated": FinalStatus.UNKNOWN} + "is_running": True, + "servicevm": False, + "auto_cleanup": False, + "updatable": True, + "updates_available": True, + "last_updates_check": None, + "updated": FinalStatus.UNKNOWN, + "has_template_updated": FinalStatus.UNKNOWN, + } for key, subkey in dom_prop.items(): try: domains[key][subkey].add(dom0) diff --git a/vmupdate/tests/test_vmupdate.py b/vmupdate/tests/test_vmupdate.py index e8eab58..7b0b4d5 100644 --- a/vmupdate/tests/test_vmupdate.py +++ b/vmupdate/tests/test_vmupdate.py @@ -32,16 +32,16 @@ from vmupdate import vmupdate -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") def test_no_options_do_nothing(_logger, _log_file, _chmod, _chown, test_qapp): test_qapp.domains = test_qapp.Domains() TestVM("dom0", test_qapp, klass="AdminVM") args = [] assert main(args, test_qapp) == EXIT.OK - args = ['--signal-no-updates'] + args = ["--signal-no-updates"] assert main(args, test_qapp) == EXIT.OK_NO_UPDATES @@ -53,26 +53,37 @@ def wait(self): pass -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') -@patch('subprocess.Popen') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("subprocess.Popen") def test_preselection( - dummy_subprocess, mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, + dummy_subprocess, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, ): dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool domains = generate_vm_variations( - test_qapp, ["klass", "updatable", "is_running"]) + test_qapp, ["klass", "updatable", "is_running"] + ) updatable = domains["updatable"][True] is_running = domains["is_running"][True] @@ -82,7 +93,9 @@ def test_preselection( app = domains["klass"]["AppVM"] disp = domains["klass"]["DispVM"] run_app = app & is_running - default = updatable & ((templ | stand | admin) | (is_running & (disp | app))) + default = updatable & ( + (templ | stand | admin) | (is_running & (disp | app)) + ) AdminVM = next(iter(admin)) TemplVM = next(iter(templ)) @@ -95,40 +108,146 @@ def test_preselection( (): default, ("--skip", UpStandVM.name): default - {UpStandVM}, ("--all",): default, - ("--all", "--apps",): default, - ("--all", "--templates",): default, - ("--all", "--standalones",): default, - ("--all", "--skip", UpStandVM.name,): default - {UpStandVM}, - ("--all", "--targets", UpStandVM.name,): default, - ("--all", "--targets", NRunAppVM.name,): default | {NRunAppVM}, - ("--all", "--targets", NUpStandVM.name,): default | {NUpStandVM}, + ( + "--all", + "--apps", + ): default, + ( + "--all", + "--templates", + ): default, + ( + "--all", + "--standalones", + ): default, + ( + "--all", + "--skip", + UpStandVM.name, + ): default + - {UpStandVM}, + ( + "--all", + "--targets", + UpStandVM.name, + ): default, + ( + "--all", + "--targets", + NRunAppVM.name, + ): default + | {NRunAppVM}, + ( + "--all", + "--targets", + NUpStandVM.name, + ): default + | {NUpStandVM}, ("--apps",): app & is_running, ("--templates",): updatable & templ, ("--standalones",): updatable & stand, - ("--templates", "--apps",): (updatable & templ) | run_app, - ("--templates", "--standalones",): updatable & (templ | stand), - ("--templates", "--standalones", "--apps",): - (updatable & (templ | stand)) | (app & is_running), - ("--standalones", "--skip", StandVM.name,): - (updatable & stand) - {StandVM}, - ("--standalones", "--skip", TemplVM.name,): (updatable & stand), - ("--standalones", "--targets", UpStandVM.name,): (updatable & stand), - ("--standalones", "--targets", NUpStandVM.name,): - (updatable & stand) | {NUpStandVM}, - ("--standalones", "--targets", TemplVM.name,): - (updatable & stand) | {TemplVM}, - ("--apps", "--skip", NRunAppVM.name,): run_app, - ("--apps", "--skip", StandVM.name,): run_app, - ("--apps", "--targets", NRunAppVM.name,): run_app | {NRunAppVM}, - ("--apps", "--targets", TemplVM.name,): run_app | {TemplVM}, - ("--targets", NRunAppVM.name,): {NRunAppVM}, - ("--targets", StandVM.name,): {StandVM}, - ("--targets", AdminVM.name,): {AdminVM}, - ("--targets", "unknown",): EXIT.ERR_USAGE, - ("--targets", f"{TemplVM.name},{StandVM.name}",): {TemplVM, StandVM}, - ("--targets", f"{TemplVM.name},{TemplVM.name}",): EXIT.ERR_USAGE, - ("--targets", TemplVM.name, "--skip", TemplVM.name,): {}, - ("--targets", f"{TemplVM.name},{StandVM.name}", "--skip", TemplVM.name,): {StandVM}, + ( + "--templates", + "--apps", + ): (updatable & templ) + | run_app, + ( + "--templates", + "--standalones", + ): updatable + & (templ | stand), + ( + "--templates", + "--standalones", + "--apps", + ): (updatable & (templ | stand)) + | (app & is_running), + ( + "--standalones", + "--skip", + StandVM.name, + ): (updatable & stand) + - {StandVM}, + ( + "--standalones", + "--skip", + TemplVM.name, + ): (updatable & stand), + ( + "--standalones", + "--targets", + UpStandVM.name, + ): (updatable & stand), + ( + "--standalones", + "--targets", + NUpStandVM.name, + ): (updatable & stand) + | {NUpStandVM}, + ( + "--standalones", + "--targets", + TemplVM.name, + ): (updatable & stand) + | {TemplVM}, + ( + "--apps", + "--skip", + NRunAppVM.name, + ): run_app, + ( + "--apps", + "--skip", + StandVM.name, + ): run_app, + ( + "--apps", + "--targets", + NRunAppVM.name, + ): run_app + | {NRunAppVM}, + ( + "--apps", + "--targets", + TemplVM.name, + ): run_app + | {TemplVM}, + ( + "--targets", + NRunAppVM.name, + ): {NRunAppVM}, + ( + "--targets", + StandVM.name, + ): {StandVM}, + ( + "--targets", + AdminVM.name, + ): {AdminVM}, + ( + "--targets", + "unknown", + ): EXIT.ERR_USAGE, + ( + "--targets", + f"{TemplVM.name},{StandVM.name}", + ): {TemplVM, StandVM}, + ( + "--targets", + f"{TemplVM.name},{TemplVM.name}", + ): EXIT.ERR_USAGE, + ( + "--targets", + TemplVM.name, + "--skip", + TemplVM.name, + ): {}, + ( + "--targets", + f"{TemplVM.name},{StandVM.name}", + "--skip", + TemplVM.name, + ): {StandVM}, } failed = {} @@ -137,9 +256,10 @@ def test_preselection( feed = {} expected_exit = selected else: - feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], - 'retcode': EXIT.OK} - for vm in selected} + feed = { + vm.name: {"statuses": [FinalStatus.SUCCESS], "retcode": EXIT.OK} + for vm in selected + } if feed: expected_exit = EXIT.OK else: @@ -147,34 +267,47 @@ def test_preselection( unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) - retcode = main(("--force-update", "--just-print-progress", *args), test_qapp) + retcode = main( + ("--force-update", "--just-print-progress", *args), test_qapp + ) failed[args] = {} if retcode != expected_exit: failed[args]["unexpected exit code"] = retcode failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - failed[args] = {key: value - for key, value in failed[args].items() if value} + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') -@patch('subprocess.Popen') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("subprocess.Popen") def test_selection( - dummy_subprocess, mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + dummy_subprocess, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager @@ -182,14 +315,19 @@ def test_selection( domains = generate_vm_variations( test_qapp, - ["klass", "updates_available", "last_updates_check", "qrexec", "os"]) + ["klass", "updates_available", "last_updates_check", "qrexec", "os"], + ) all = domains["updatable"][True] qlinux = domains["qrexec"][True] & domains["os"]["Linux"] to_update = domains["updates_available"][True] - stale = qlinux & (domains["updates_available"][False] & - (domains["last_updates_check"][None] | - domains["last_updates_check"]['2020-01-01 00:00:00'])) + stale = qlinux & ( + domains["updates_available"][False] + & ( + domains["last_updates_check"][None] + | domains["last_updates_check"]["2020-01-01 00:00:00"] + ) + ) expected = { ("--force-update",): all, @@ -206,14 +344,15 @@ def test_selection( if isinstance(selected, int): feed = {} expected_exit = selected - monkeypatch.setattr( - vmupdate, "preselect_targets", lambda *_: all) + monkeypatch.setattr(vmupdate, "preselect_targets", lambda *_: all) else: - feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], - 'retcode': EXIT.OK} - for vm in selected} + feed = { + vm.name: {"statuses": [FinalStatus.SUCCESS], "retcode": EXIT.OK} + for vm in selected + } monkeypatch.setattr( - vmupdate, "preselect_targets", lambda *_: selected) + vmupdate, "preselect_targets", lambda *_: selected + ) if feed: expected_exit = EXIT.OK else: @@ -228,28 +367,40 @@ def test_selection( failed[args]["unexpected exit code"] = retcode failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - failed[args] = {key: value - for key, value in failed[args].items() if value} + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') -@patch('asyncio.run') -@patch('subprocess.Popen') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("asyncio.run") +@patch("subprocess.Popen") def test_restarting( - dummy_subprocess, arun, mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + dummy_subprocess, + arun, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager @@ -257,8 +408,15 @@ def test_restarting( domains = generate_vm_variations( test_qapp, - ["klass", "is_running", "servicevm", "auto_cleanup", - "updated", "has_template_updated"]) + [ + "klass", + "is_running", + "servicevm", + "auto_cleanup", + "updated", + "has_template_updated", + ], + ) all = domains["updatable"][True] service = domains["servicevm"][True] @@ -271,33 +429,41 @@ def test_restarting( not_updated = all - domains["updated"][FinalStatus.SUCCESS] running = domains["is_running"][True] template_updated = domains["has_template_updated"][FinalStatus.SUCCESS] - applicable = (derived & not_updated & running & template_updated - ) - (auto_cleanup & disp) + applicable = (derived & not_updated & running & template_updated) - ( + auto_cleanup & disp + ) expected = { - (): {"halted": set(), - "restarted": set(), - "untouched": all}, + (): {"halted": set(), "restarted": set(), "untouched": all}, ("--no-apply",): { "halted": set(), "restarted": set(), - "untouched": all}, + "untouched": all, + }, ("--apply-to-sys",): { "halted": updated & running & templ, "restarted": applicable & service, - "untouched": all - (updated & running & templ) - (applicable & service)}, + "untouched": all + - (updated & running & templ) + - (applicable & service), + }, ("--apply-to-all",): { "halted": (updated & running & templ) | (applicable - service), "restarted": applicable & service, - "untouched": all - (updated & running & templ) - applicable}, + "untouched": all - (updated & running & templ) - applicable, + }, } failed = {} for args, selected in expected.items(): monkeypatch.setattr(vmupdate, "get_targets", lambda *_: all) - feed = {vm.name: {'statuses': [vm.update_result], - 'retcode': None} # we don't care - for vm in all} + feed = { + vm.name: { + "statuses": [vm.update_result], + "retcode": None, + } # we don't care + for vm in all + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) @@ -308,27 +474,35 @@ def test_restarting( failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - halted = {vm for vm in all - if vm.shutdown.called and not vm.start.called} - restarted = {vm for vm in all - if vm.shutdown.called and vm.start.called} - untouched = {vm for vm in all - if not vm.shutdown.called and not vm.start.called} - failed[args]["unexpected restart"] = set(map( - lambda vm: vm.name, restarted - selected["restarted"])) - failed[args]["not restarted"] = set(map( - lambda vm: vm.name, selected["restarted"] - restarted)) - failed[args]["unexpected shutdown"] = set(map( - lambda vm: vm.name, halted - selected["halted"])) - failed[args]["not halted"] = set(map( - lambda vm: vm.name, selected["halted"] - halted)) - failed[args]["unexpected untouched"] = set(map( - lambda vm: vm.name, untouched - selected["untouched"])) - failed[args]["unexpected touched"] = set(map( - lambda vm: vm.name, selected["untouched"] - untouched)) - - failed[args] = {key: value - for key, value in failed[args].items() if value} + halted = { + vm for vm in all if vm.shutdown.called and not vm.start.called + } + restarted = {vm for vm in all if vm.shutdown.called and vm.start.called} + untouched = { + vm for vm in all if not vm.shutdown.called and not vm.start.called + } + failed[args]["unexpected restart"] = set( + map(lambda vm: vm.name, restarted - selected["restarted"]) + ) + failed[args]["not restarted"] = set( + map(lambda vm: vm.name, selected["restarted"] - restarted) + ) + failed[args]["unexpected shutdown"] = set( + map(lambda vm: vm.name, halted - selected["halted"]) + ) + failed[args]["not halted"] = set( + map(lambda vm: vm.name, selected["halted"] - halted) + ) + failed[args]["unexpected untouched"] = set( + map(lambda vm: vm.name, untouched - selected["untouched"]) + ) + failed[args]["unexpected touched"] = set( + map(lambda vm: vm.name, selected["untouched"] - untouched) + ) + + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails @@ -338,60 +512,134 @@ def test_restarting( stat = FinalStatus -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") @pytest.mark.parametrize( "tmpl_status, tmpl_retcode, app_status, app_retcode, expected_retcode", -( - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: 2x OK"), - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: tmpl OK"), - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: app OK"), - pytest.param( - stat.ERROR, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.ERR, id="error: tmpl"), - pytest.param( - stat.SUCCESS, EXIT.OK, stat.ERROR, EXIT.OK, - EXIT.ERR, id="error: app"), - pytest.param( - stat.SUCCESS, EXIT.OK_NO_UPDATES, stat.SUCCESS, EXIT.OK, - EXIT.ERR_VM_UNHANDLED, id="unhandled retcode"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM, stat.ERROR, EXIT.ERR_VM_PRE, - EXIT.ERR_VM_PRE, id="vm inside error"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM_UPDATE, stat.ERROR, EXIT.ERR_VM_REFRESH, - EXIT.ERR_VM_UPDATE, id="vm inside error 2"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM, stat.SUCCESS, EXIT.OK, - EXIT.ERR_VM, id="vm general inside error"), - pytest.param( - stat.CANCELLED, EXIT.OK, stat.SUCCESS, EXIT.OK, - EXIT.SIGINT, id="cancelled"), - pytest.param( - stat.CANCELLED, EXIT.OK, stat.ERROR, EXIT.ERR_VM_UNHANDLED, - EXIT.SIGINT, id="cancelled with error"), - pytest.param( - stat.UNKNOWN, EXIT.OK, stat.SUCCESS, EXIT.OK, - EXIT.ERR_QREXEX, id="communication error"), -)) + ( + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: 2x OK", + ), + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: tmpl OK", + ), + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: app OK", + ), + pytest.param( + stat.ERROR, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.ERR, + id="error: tmpl", + ), + pytest.param( + stat.SUCCESS, + EXIT.OK, + stat.ERROR, + EXIT.OK, + EXIT.ERR, + id="error: app", + ), + pytest.param( + stat.SUCCESS, + EXIT.OK_NO_UPDATES, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_VM_UNHANDLED, + id="unhandled retcode", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM, + stat.ERROR, + EXIT.ERR_VM_PRE, + EXIT.ERR_VM_PRE, + id="vm inside error", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM_UPDATE, + stat.ERROR, + EXIT.ERR_VM_REFRESH, + EXIT.ERR_VM_UPDATE, + id="vm inside error 2", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_VM, + id="vm general inside error", + ), + pytest.param( + stat.CANCELLED, + EXIT.OK, + stat.SUCCESS, + EXIT.OK, + EXIT.SIGINT, + id="cancelled", + ), + pytest.param( + stat.CANCELLED, + EXIT.OK, + stat.ERROR, + EXIT.ERR_VM_UNHANDLED, + EXIT.SIGINT, + id="cancelled with error", + ), + pytest.param( + stat.UNKNOWN, + EXIT.OK, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_QREXEX, + id="communication error", + ), + ), +) def test_return_codes( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch, - tmpl_status, tmpl_retcode, app_status, app_retcode, expected_retcode + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, + tmpl_status, + tmpl_retcode, + app_status, + app_retcode, + expected_retcode, ): mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -401,32 +649,48 @@ def test_return_codes( appvm = TestVM("appvm", test_qapp, klass="AppVM", template=vm) feed = { - vm.name: {'statuses': [tmpl_status], 'retcode': tmpl_retcode}, - appvm.name: {'statuses': [app_status], 'retcode': app_retcode}} + vm.name: {"statuses": [tmpl_status], "retcode": tmpl_retcode}, + appvm.name: {"statuses": [app_status], "retcode": app_retcode}, + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) retcode = main( - ("--just-print-progress", "--all", "--force-update", - "--signal-no-updates"), test_qapp) + ( + "--just-print-progress", + "--all", + "--force-update", + "--signal-no-updates", + ), + test_qapp, + ) assert retcode == expected_retcode -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") def test_error( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -436,47 +700,66 @@ def test_error( appvm = TestVM("appvm", test_qapp, klass="AppVM", template=vm) feed = { - vm.name: {'statuses': [FinalStatus.ERROR], 'retcode': EXIT.OK}, - appvm.name: {'statuses': [FinalStatus.NO_UPDATES], 'retcode': EXIT.OK}} + vm.name: {"statuses": [FinalStatus.ERROR], "retcode": EXIT.OK}, + appvm.name: {"statuses": [FinalStatus.NO_UPDATES], "retcode": EXIT.OK}, + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) - retcode = main(( - "--just-print-progress", "--all", "--force-update"), test_qapp) + retcode = main( + ("--just-print-progress", "--all", "--force-update"), test_qapp + ) assert retcode == EXIT.ERR -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('asyncio.run') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("asyncio.run") @pytest.mark.parametrize( "action, code", -( - pytest.param("template shutdown", EXIT.ERR_SHUTDOWN_TMPL), - pytest.param("app shutdown", EXIT.ERR_SHUTDOWN_APP), - pytest.param("app start", EXIT.ERR_START_APP), -)) + ( + pytest.param("template shutdown", EXIT.ERR_SHUTDOWN_TMPL), + pytest.param("app shutdown", EXIT.ERR_SHUTDOWN_APP), + pytest.param("app start", EXIT.ERR_START_APP), + ), +) def test_error_apply( - _arun, _logger, _log_file, _chmod, _chown, _print, - test_qapp, monkeypatch, action, code + _arun, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + monkeypatch, + action, + code, ): _dom0 = TestVM("dom0", test_qapp, klass="AdminVM") vm = TestVM("vm", test_qapp, klass="TemplateVM") appvm = TestVM( - "appvm", test_qapp, klass="AppVM", template=vm, - features=Features("appvm", test_qapp, {'servicevm': True})) + "appvm", + test_qapp, + klass="AppVM", + template=vm, + features=Features("appvm", test_qapp, {"servicevm": True}), + ) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) - monkeypatch.setattr(vmupdate, "run_update", - lambda *_: [EXIT.OK, {"vm": FinalStatus.SUCCESS}]) + monkeypatch.setattr( + vmupdate, + "run_update", + lambda *_: [EXIT.OK, {"vm": FinalStatus.SUCCESS}], + ) def raiser(*_args, **_kwargs): raise qubesadmin.exc.QubesVMError("foo") + if action == "template shutdown": vm.shutdown = raiser elif action == "app shutdown": @@ -490,17 +773,23 @@ def raiser(*_args, **_kwargs): assert retcode == code -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") def test_error_usage_wrong_param( - _logger, _log_file, _chmod, _chown, _print, test_qapp, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, ): _dom0 = TestVM("dom0", test_qapp, klass="AdminVM") - retcode = main(( - "--just-print-progress", "--targets", 'vm', "--force-update"), - test_qapp) + retcode = main( + ("--just-print-progress", "--targets", "vm", "--force-update"), + test_qapp, + ) assert retcode == EXIT.ERR_USAGE diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index 9877467..53060e0 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -33,11 +33,11 @@ from tqdm import tqdm -from .agent.source.status import StatusInfo, FinalStatus, Status, FormatedLine -from .qube_connection import QubeConnection from vmupdate.agent.source.log_config import init_logs from vmupdate.agent.source.common.process_result import ProcessResult from vmupdate.agent.source.common.exit_codes import EXIT +from .agent.source.status import StatusInfo, FinalStatus, Status, FormatedLine +from .qube_connection import QubeConnection class UpdateManager: @@ -71,24 +71,35 @@ def run(self, agent_args): show_progress = not self.quiet and not self.no_progress SimpleTerminalBar.reinit_class(self.download_only) - progress_output = SimpleTerminalBar \ - if self.just_print_progress else tqdm + progress_output = ( + SimpleTerminalBar if self.just_print_progress else tqdm + ) progress_bar = MultipleUpdateMultipleProgressBar( dummy=not show_progress, output=progress_output, max_concurrency=self.max_concurrency, - printer=self.print if self.show_output else None + printer=self.print if self.show_output else None, ) for qube in self.qubes: - disp_name = agent_args.display_name \ - if agent_args.display_name is not None else qube.name + disp_name = ( + agent_args.display_name + if agent_args.display_name is not None + else qube.name + ) progress_bar.add_bar(disp_name) progress_bar.pool.apply_async( update_qube, - (qube, agent_args, show_progress, - progress_bar.status_notifier, progress_bar.termination, self.dom0), - callback=self.collect_result, error_callback=print + ( + qube, + agent_args, + show_progress, + progress_bar.status_notifier, + progress_bar.termination, + self.dom0, + ), + callback=self.collect_result, + error_callback=print, ) if qube.klass == "AdminVM" and show_progress: # progress of AdminVM is continuation of different process, @@ -130,21 +141,25 @@ def collect_result(self, result_tuple: Tuple[str, ProcessResult]): self.ret_code = max(self.ret_code, vm_code) if self.show_output: - for line in result.out.split('\n'): + for line in result.out.split("\n"): self.print(FormatedLine(qube_name, "out", line)) - for line in result.err.split('\n'): + for line in result.err.split("\n"): self.print(FormatedLine(qube_name, "err", line)) elif not self.quiet and self.no_progress: self.print(result.out) def print(self, *args): if self.buffered: - self.buffer += ' '.join(str(args)) + '\n' + self.buffer += " ".join(str(args)) + "\n" else: print(*args, file=sys.stdout, flush=True) class TerminalMultiBar: + """ + Handles multiple progress bars in terminal. + """ + def __init__(self): self.progresses = [] @@ -154,6 +169,10 @@ def print(self): class SimpleTerminalBar: + """ + Simple progress bar for terminal output. Could be used by TerminalMultiBar. + """ + PARENT_MULTI_BAR = None DOWNLOAD_ONLY = False @@ -166,12 +185,14 @@ def __init__(self, total, position, desc): def __str__(self): info = None - name, status = self.desc.split(' ', 1) + name, status = self.desc.split(" ", 1) status = status[1:-1] # remove brackets - if status in (FinalStatus.SUCCESS.value, - FinalStatus.ERROR.value, - FinalStatus.CANCELLED.value, - FinalStatus.NO_UPDATES.value): + if status in ( + FinalStatus.SUCCESS.value, + FinalStatus.ERROR.value, + FinalStatus.CANCELLED.value, + FinalStatus.NO_UPDATES.value, + ): if SimpleTerminalBar.DOWNLOAD_ONLY: return "" info = status.replace(" ", "_") @@ -194,10 +215,9 @@ def set_description(self, desc: str): def close(self): """Implementation of tqdm API""" - pass @staticmethod - def reinit_class(download_only = False): + def reinit_class(download_only=False): SimpleTerminalBar.PARENT_MULTI_BAR = TerminalMultiBar() SimpleTerminalBar.DOWNLOAD_ONLY = download_only @@ -211,7 +231,7 @@ def __init__(self, dummy, output, max_concurrency, printer: Optional): self.dummy = dummy self.manager = multiprocessing.Manager() - self.termination = self.manager.Value('b', False) + self.termination = self.manager.Value("b", False) self.status_notifier = self.manager.Queue() # save original signal handler for SIGINT @@ -238,8 +258,9 @@ def add_bar(self, qname: str): self.progresses[qname] = 0 self.progress_bars[qname] = self.output_class( - total=100, position=len(self.progress_bars), - desc=f"{qname} ({Status.PENDING.value})" + total=100, + position=len(self.progress_bars), + desc=f"{qname} ({Status.PENDING.value})", ) def feeding(self): @@ -254,18 +275,20 @@ def feeding(self): left_to_finish = len(self.progresses) while left_to_finish: try: - feed: Optional[StatusInfo, str] = \ - self.status_notifier.get(block=True) + feed: Optional[StatusInfo, str] = self.status_notifier.get( + block=True + ) if feed is None: continue - elif isinstance(feed, StatusInfo): + if isinstance(feed, StatusInfo): status_name = feed.status.value if feed.status == Status.DONE: left_to_finish -= 1 status_name = feed.info.value self.statuses[feed.qname] = FinalStatus(status_name) self.progress_bars[feed.qname].set_description( - f"{feed.qname} ({status_name})") + f"{feed.qname} ({status_name})" + ) if feed.status == Status.UPDATING: self._update(feed.qname, feed.info) elif self.print is not None: @@ -296,7 +319,7 @@ def close(self): def update_qube( - qube, agent_args, show_progress, status_notifier, termination, dom0 + qube, agent_args, show_progress, status_notifier, termination, dom0 ) -> Tuple[str, ProcessResult]: """ Create and run `UpdateAgentManager` for qube. @@ -306,11 +329,14 @@ def update_qube( :param show_progress: if progress should be printed in real time :param status_notifier: an object to be fed with the progress data :param termination: signal to gracefully terminate subprocess - :param dom0: whether to use qubes-dom0-update + :param dom0: whether to use qubes-dom0-update (do download&install) + or just update agent to install prepared updates :return: """ if agent_args.display_name is not None: - status_notifier = StatusNotifierWrapper(status_notifier, agent_args.display_name) + status_notifier = StatusNotifierWrapper( + status_notifier, agent_args.display_name + ) if termination.value: status_notifier.put(StatusInfo.done(qube, FinalStatus.CANCELLED)) @@ -322,17 +348,18 @@ def update_qube( qube, agent_args=agent_args, show_progress=show_progress, - dom0=dom0 + dom0=dom0, ) result = runner.run_agent( agent_args=agent_args, status_notifier=status_notifier, - termination=termination + termination=termination, ) except Exception as exc: # pylint: disable=broad-except status_notifier.put(StatusInfo.done(qube, FinalStatus.ERROR)) return qube.name, ProcessResult( - EXIT.ERR_VM_UNHANDLED, f"ERROR (exception {str(exc)})") + EXIT.ERR_VM_UNHANDLED, f"ERROR (exception {str(exc)})" + ) return qube.name, result @@ -340,22 +367,27 @@ class UpdateAgentManager: """ Send update agent files and run it in the qube. """ + AGENT_RELATIVE_DIR = "agent" ENTRYPOINT = AGENT_RELATIVE_DIR + "/entrypoint.py" - LOGPATH = '/var/log/qubes' - FORMAT_LOG = '%(asctime)s %(message)s' + LOGPATH = "/var/log/qubes" + FORMAT_LOG = "%(asctime)s %(message)s" WORKDIR = "/run/qubes-update/" - def __init__( - self, app, qube, agent_args, show_progress, dom0): + def __init__(self, app, qube, agent_args, show_progress, dom0): self.qube = qube self.app = app self.dom0 = dom0 - (self.log, self.log_handler, log_level, - self.log_path, self.log_formatter) = init_logs( + ( + self.log, + self.log_handler, + _log_level, + self.log_path, + self.log_formatter, + ) = init_logs( directory=self.LOGPATH, - file=f'update-{qube.name}.log', + file=f"update-{qube.name}.log", format_=UpdateAgentManager.FORMAT_LOG, level=agent_args.log, truncate_file=False, @@ -366,7 +398,7 @@ def __init__( self.show_progress = show_progress def run_agent( - self, agent_args, status_notifier, termination + self, agent_args, status_notifier, termination ) -> ProcessResult: """ Copy agent file to dest vm, run entrypoint, collect output and logs. @@ -374,21 +406,16 @@ def run_agent( if self.qube.klass == "AdminVM" and not self.dom0: # using UpdateVM to download updates status_notifier = StatusNotifierWrapper(status_notifier, "dom0") - result = self._run_agent( - agent_args, status_notifier, termination) - output = result.out.split("\n") + result.err.split("\n") - for line in output: - self.log.debug('agent output: %s', line) - self.log.info('agent exit code: %d', result.code) - if not agent_args.show_output or not output: - result.out = "OK" if result.code == EXIT.OK else \ - f"ERROR (exit code {result.code}, details in {self.log_path})" + + result = self._run_agent(agent_args, status_notifier, termination) + + self._log_output(result, agent_args.show_output) return result def _run_agent( - self, agent_args, status_notifier, termination + self, agent_args, status_notifier, termination ) -> ProcessResult: - self.log.info('Running update agent for %s', self.qube.name) + self.log.info("Running update agent for %s", self.qube.name) dest_dir = None src_dir = None cleanup = False @@ -398,7 +425,7 @@ def _run_agent( if agent_args.just_print_progress or self.show_progress: entrypoint.append("--just-print-progress") if agent_args.quiet: - entrypoint.append('--quiet') + entrypoint.append("--quiet") else: this_dir = os.path.dirname(os.path.realpath(__file__)) entrypoint = join(this_dir, UpdateAgentManager.ENTRYPOINT) @@ -410,53 +437,82 @@ def _run_agent( src_dir = join(this_dir, UpdateAgentManager.AGENT_RELATIVE_DIR) with QubeConnection( - self.qube, - dest_dir, - cleanup, - self.log, - self.show_progress, - status_notifier + self.qube, + dest_dir, + cleanup, + self.log, + self.show_progress, + status_notifier, ) as qconn: - result = ProcessResult() - if self.qube.klass != "AdminVM": - self.log.info( - "Transferring files to destination qube: %s", self.qube.name) - result += qconn.transfer_agent(src_dir) - if result: - self.log.error('Qube communication error code: %i', result.code) - return result + result = self._transfer_agent(qconn, src_dir) if termination.value: qconn.status = FinalStatus.CANCELLED return ProcessResult(EXIT.SIGINT, "", "Cancelled") + result += self._run_entrypoint(qconn, entrypoint, agent_args) + + self._read_logs(qconn) + + return result + + def _transfer_agent(self, qconn, src_dir) -> ProcessResult: + result = ProcessResult() + if self.qube.klass != "AdminVM": self.log.info( - "The agent is starting the task in qube: %s", self.qube.name) - self.log.debug( - "%s", entrypoint) - result += qconn.run_entrypoint(entrypoint, agent_args) - if not result and qconn.status != FinalStatus.NO_UPDATES: - qconn.status = FinalStatus.SUCCESS - - result_logs = qconn.read_logs() - if result_logs: - self.log.error( - "Problem with collecting logs from %s, return code: %i", - self.qube.name, result_logs.code) - # agent logs already have timestamp - self.log_handler.setFormatter(logging.Formatter('%(message)s')) - # critical -> always write agent logs - for log_line in result_logs.out.split("\n"): - if log_line: - self.log.critical("%s", log_line) - self.log_handler.setFormatter(self.log_formatter) + "Transferring files to destination qube: %s", self.qube.name + ) + result += qconn.transfer_agent(src_dir) + if result: + self.log.error("Qube communication error code: %i", result.code) + return result + return result + def _run_entrypoint(self, qconn, entrypoint, agent_args) -> ProcessResult: + result = ProcessResult() + self.log.info( + "The agent is starting the task in qube: %s", self.qube.name + ) + self.log.debug("%s", entrypoint) + result += qconn.run_entrypoint(entrypoint, agent_args) + if not result and qconn.status != FinalStatus.NO_UPDATES: + qconn.status = FinalStatus.SUCCESS return result + def _read_logs(self, qconn): + result_logs = qconn.read_logs() + if result_logs: + self.log.error( + "Problem with collecting logs from %s, return code: %i", + self.qube.name, + result_logs.code, + ) + # agent logs already have timestamp + self.log_handler.setFormatter(logging.Formatter("%(message)s")) + # critical -> always write agent logs + for log_line in result_logs.out.split("\n"): + if log_line: + self.log.critical("%s", log_line) + self.log_handler.setFormatter(self.log_formatter) + + def _log_output(self, result: ProcessResult, show_output: bool): + output = result.out.split("\n") + result.err.split("\n") + for line in output: + self.log.debug("agent output: %s", line) + self.log.info("agent exit code: %d", result.code) + if not show_output or not output: + result.out = ( + "OK" + if result.code == EXIT.OK + else f"ERROR (exit code {result.code}, details in {self.log_path})" + ) + + class StatusNotifierWrapper: """ Masks proxy VM with display name. """ + def __init__(self, status_notifier, qube_name): self.status_notifier = status_notifier self.qube_name = qube_name diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index c3a1c8b..eaff377 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -6,7 +6,6 @@ import argparse import asyncio import logging -import subprocess import sys import os import grp @@ -22,23 +21,22 @@ from .agent.source.args import AgentArgs DEFAULT_UPDATE_IF_STALE = 7 -LOGPATH = '/var/log/qubes/qubes-vm-update.log' -LOG_FORMAT = '%(asctime)s %(message)s' +LOGPATH = "/var/log/qubes/qubes-vm-update.log" +LOG_FORMAT = "%(asctime)s %(message)s" class ArgumentError(Exception): - """Nonsense arguments - """ + """Nonsense arguments""" def main(args=None, app=qubesadmin.Qubes()): args = parse_args(args, app) - log_handler = logging.FileHandler(LOGPATH, encoding='utf-8') + log_handler = logging.FileHandler(LOGPATH, encoding="utf-8") log_formatter = logging.Formatter(LOG_FORMAT) log_handler.setFormatter(log_formatter) - log = logging.getLogger('vm-update') + log = logging.getLogger("vm-update") log.setLevel(args.log) log.addHandler(log_handler) try: @@ -61,11 +59,17 @@ def main(args=None, app=qubesadmin.Qubes()): print("No qube selected for update") return EXIT.OK_NO_UPDATES if args.signal_no_updates else EXIT.OK - admin = [target for target in targets if target.klass == 'AdminVM'] - independent = [target for target in targets if target.klass in ( - 'TemplateVM', 'StandaloneVM')] - derived = [target for target in targets if target.klass not in ( - 'AdminVM', 'TemplateVM', 'StandaloneVM')] + admin = [target for target in targets if target.klass == "AdminVM"] + independent = [ + target + for target in targets + if target.klass in ("TemplateVM", "StandaloneVM") + ] + derived = [ + target + for target in targets + if target.klass not in ("AdminVM", "TemplateVM", "StandaloneVM") + ] no_updates = True ret_code_admin = EXIT.OK @@ -79,27 +83,40 @@ def main(args=None, app=qubesadmin.Qubes()): log.debug(message) if args.just_print_progress and args.no_refresh: # internal usage just for installing ready updates, use carefully - ret_code_admin, admin_status = run_update(admin, args, log, "admin VM") + ret_code_admin, admin_status = run_update( + admin, args, log, "admin VM" + ) else: # use qubes-dom0-update to update dom0 - ret_code_admin, admin_status = run_update(admin, args, log, "admin VM", dom0=True) - no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in admin_status.values()) + ret_code_admin, admin_status = run_update( + admin, args, log, "admin VM", dom0=True + ) + no_updates = all( + stat == FinalStatus.NO_UPDATES for stat in admin_status.values() + ) # independent qubes first (TemplateVMs, StandaloneVMs) ret_code_independent, templ_statuses = run_update( - independent, args, log, "templates and standalones") - no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in templ_statuses.values()) and no_updates + independent, args, log, "templates and standalones" + ) + no_updates = ( + all(stat == FinalStatus.NO_UPDATES for stat in templ_statuses.values()) + and no_updates + ) # then derived qubes (AppVMs...) ret_code_appvm, app_statuses = run_update(derived, args, log) - no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in app_statuses.values()) and no_updates + no_updates = ( + all(stat == FinalStatus.NO_UPDATES for stat in app_statuses.values()) + and no_updates + ) ret_code_restart = apply_updates_to_appvm( - args, independent, templ_statuses, app_statuses, log) + args, independent, templ_statuses, app_statuses, log + ) - ret_code = max(ret_code_admin, ret_code_independent, ret_code_appvm, ret_code_restart) + ret_code = max( + ret_code_admin, ret_code_independent, ret_code_appvm, ret_code_restart + ) if ret_code == EXIT.OK and no_updates and args.signal_no_updates: return EXIT.OK_NO_UPDATES if ret_code == EXIT.OK_NO_UPDATES and not args.signal_no_updates: @@ -110,77 +127,117 @@ def main(args=None, app=qubesadmin.Qubes()): def parse_args(args, app): parser = argparse.ArgumentParser() try: - default_update_if_stale = int(app.domains["dom0"].features.get( - "qubes-vm-update-update-if-stale", DEFAULT_UPDATE_IF_STALE)) + default_update_if_stale = int( + app.domains["dom0"].features.get( + "qubes-vm-update-update-if-stale", DEFAULT_UPDATE_IF_STALE + ) + ) except qubesadmin.exc.QubesDaemonAccessError: default_update_if_stale = DEFAULT_UPDATE_IF_STALE - parser.add_argument('--max-concurrency', '-x', - action='store', - help='Maximum number of VMs configured simultaneously ' - '(default: number of cpus)', - type=int) - parser.add_argument('--dry-run', action='store_true', - help='Just print what happens.') parser.add_argument( - '--signal-no-updates', action='store_true', - help='Return exit code 100 instead of 0 ' - 'if there is no updates available.') + "--max-concurrency", + "-x", + action="store", + help="Maximum number of VMs configured simultaneously " + "(default: number of cpus)", + type=int, + ) + parser.add_argument( + "--dry-run", action="store_true", help="Just print what happens." + ) + parser.add_argument( + "--signal-no-updates", + action="store_true", + help="Return exit code 100 instead of 0 " + "if there is no updates available.", + ) restart = parser.add_mutually_exclusive_group() restart.add_argument( - '--apply-to-sys', '--restart', '-r', - action='store_true', - help='Restart not updated ServiceVMs whose template has been updated.') + "--apply-to-sys", + "--restart", + "-r", + action="store_true", + help="Restart not updated ServiceVMs whose template has been updated.", + ) restart.add_argument( - '--apply-to-all', '-R', action='store_true', - help='Restart not updated ServiceVMs and shutdown not updated AppVMs ' - 'whose template has been updated.') + "--apply-to-all", + "-R", + action="store_true", + help="Restart not updated ServiceVMs and shutdown not updated AppVMs " + "whose template has been updated.", + ) restart.add_argument( - '--no-apply', action='store_true', - help='DEFAULT. Do not restart/shutdown any AppVMs.') + "--no-apply", + action="store_true", + help="DEFAULT. Do not restart/shutdown any AppVMs.", + ) update_state = parser.add_mutually_exclusive_group() update_state.add_argument( - '--force-update', action='store_true', - help='Attempt to update all targeted VMs ' - 'even if no updates are available') + "--force-update", + action="store_true", + help="Attempt to update all targeted VMs " + "even if no updates are available", + ) update_state.add_argument( - '--update-if-stale', action='store', - help='DEFAULT. ' - 'Attempt to update targeted VMs with known updates available ' - 'or for which last update check was more than N days ago. ' - '(default: %(default)d)', - type=int, default=default_update_if_stale) + "--update-if-stale", + action="store", + help="DEFAULT. " + "Attempt to update targeted VMs with known updates available " + "or for which last update check was more than N days ago. " + "(default: %(default)d)", + type=int, + default=default_update_if_stale, + ) update_state.add_argument( - '--update-if-available', action='store_true', - help='Update targeted VMs with known updates available.') + "--update-if-available", + action="store_true", + help="Update targeted VMs with known updates available.", + ) parser.add_argument( - '--skip', action='store', - help='Comma separated list of VMs to be skipped, ' - 'works with all other options.', default="") + "--skip", + action="store", + help="Comma separated list of VMs to be skipped, " + "works with all other options.", + default="", + ) parser.add_argument( - '--targets', action='store', - help='Comma separated list of VMs to target. Ignores conditions.') + "--targets", + action="store", + help="Comma separated list of VMs to target. Ignores conditions.", + ) parser.add_argument( - '--templates', '-T', action='store_true', - help='Target all updatable TemplateVMs.') + "--templates", + "-T", + action="store_true", + help="Target all updatable TemplateVMs.", + ) parser.add_argument( - '--standalones', '-S', action='store_true', - help='Target all updatable StandaloneVMs.') + "--standalones", + "-S", + action="store_true", + help="Target all updatable StandaloneVMs.", + ) parser.add_argument( - '--apps', '-A', action='store_true', - help='Target running updatable AppVMs to update in place.') + "--apps", + "-A", + action="store_true", + help="Target running updatable AppVMs to update in place.", + ) parser.add_argument( - '--all', action='store_true', - help='DEFAULT. Target all updatable VMs except AdminVM. ' - 'Use explicitly with "--targets" to include both.') + "--all", + action="store_true", + help="DEFAULT. Target all updatable VMs except AdminVM. " + 'Use explicitly with "--targets" to include both.', + ) # for internal usage, e.g., download updates via proxy vm parser.add_argument( - '--display-name', action='store', - help=argparse.SUPPRESS) + "--display-name", action="store", help=argparse.SUPPRESS + ) AgentArgs.add_arguments(parser) args = parser.parse_args(args) @@ -199,28 +256,40 @@ def get_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: def preselect_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: targets = set() - updatable = {vm for vm in app.domains if getattr(vm, 'updateable', False)} - default_targeting = (not args.templates and not args.standalones and - not args.apps and not args.targets) + updatable = {vm for vm in app.domains if getattr(vm, "updateable", False)} + default_targeting = ( + not args.templates + and not args.standalones + and not args.apps + and not args.targets + ) if args.all or default_targeting: # filter out stopped AppVMs and DispVMs (?) - targets = {vm for vm in updatable - if vm.klass not in ("AppVM", "DispVM") or vm.is_running()} + targets = { + vm + for vm in updatable + if vm.klass not in ("AppVM", "DispVM") or vm.is_running() + } else: # if not all updatable are included, target a specific classes if args.templates: - targets.update([vm for vm in updatable - if vm.klass == 'TemplateVM']) + targets.update([vm for vm in updatable if vm.klass == "TemplateVM"]) if args.standalones: - targets.update([vm for vm in updatable - if vm.klass == 'StandaloneVM']) + targets.update( + [vm for vm in updatable if vm.klass == "StandaloneVM"] + ) if args.apps: - targets.update({vm for vm in app.domains - if vm.klass == 'AppVM' and vm.is_running()}) + targets.update( + { + vm + for vm in app.domains + if vm.klass == "AppVM" and vm.is_running() + } + ) # user can target non-updatable vm if she like if args.targets: - names = args.targets.split(',') + names = args.targets.split(",") explicit_targets = {vm for vm in app.domains if vm.name in names} if len(names) != len(explicit_targets): target_names = {q.name for q in explicit_targets} @@ -233,13 +302,16 @@ def preselect_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: targets.update(explicit_targets) # remove skipped qubes and dom0 - not a target - to_skip = args.skip.split(',') + to_skip = args.skip.split(",") targets = {vm for vm in targets if vm.name not in to_skip} # exclude vms with `skip-update` feature, but allow --targets to override it if not args.targets: - targets = {vm for vm in targets - if not bool(vm.features.get('skip-update', False))} + targets = { + vm + for vm in targets + if not bool(vm.features.get("skip-update", False)) + } return targets @@ -252,7 +324,7 @@ def select_targets(targets, args) -> Set[qubesadmin.vm.QubesVM]: selected = set() for vm in targets: try: - to_update = vm.features.get('updates-available', False) + to_update = vm.features.get("updates-available", False) except qubesadmin.exc.QubesDaemonCommunicationError: to_update = False @@ -275,13 +347,15 @@ def select_targets(targets, args) -> Set[qubesadmin.vm.QubesVM]: def is_stale(vm, expiration_period): today = datetime.today() try: - if not ('qrexec' in vm.features.keys() - and vm.features.get('os', '') == 'Linux'): + if not ( + "qrexec" in vm.features.keys() + and vm.features.get("os", "") == "Linux" + ): return False last_update_str = vm.features.check_with_template( - 'last-updates-check', - datetime.fromtimestamp(0).strftime('%Y-%m-%d %H:%M:%S') + "last-updates-check", + datetime.fromtimestamp(0).strftime("%Y-%m-%d %H:%M:%S"), ) last_update = datetime.fromisoformat(last_update_str) if (today - last_update).days > expiration_period: @@ -292,11 +366,12 @@ def is_stale(vm, expiration_period): def run_update( - targets, args, log, qube_klass="qubes", dom0=False + targets, args, log, qube_klass="qubes", dom0=False ) -> Tuple[int, Dict[str, FinalStatus]]: if targets: - message = f"Following {qube_klass} will be updated: " + \ - ", ".join((target.name for target in targets)) + message = f"Following {qube_klass} will be updated: " + ", ".join( + (target.name for target in targets) + ) else: if qube_klass == "qubes": message = "" # no need to inform about app VMs etc. @@ -306,8 +381,7 @@ def run_update( if message: print(message) return EXIT.OK, {target.name: FinalStatus.SUCCESS for target in targets} - else: - log.debug(message) + log.debug(message) if not targets: return EXIT.OK, {} @@ -316,8 +390,10 @@ def run_update( ret_code, statuses = runner.run(agent_args=args) if ret_code: log.error("Updating fails with code: %d", ret_code) - log.debug("Updating report: %s", - ", ".join((k + ":" + v.value for k, v in statuses.items()))) + log.debug( + "Updating report: %s", + ", ".join((k + ":" + v.value for k, v in statuses.items())), + ) return ret_code, statuses @@ -342,11 +418,11 @@ def get_boolean_feature(vm, feature_name, default=False): def apply_updates_to_appvm( - args, - vm_updated: Iterable, - template_statuses: Dict[str, FinalStatus], - derived_statuses: Dict[str, FinalStatus], - log + args, + vm_updated: Iterable, + template_statuses: Dict[str, FinalStatus], + derived_statuses: Dict[str, FinalStatus], + log, ) -> int: """ Shutdown running templates and then restart/shutdown derived AppVMs. @@ -361,22 +437,31 @@ def apply_updates_to_appvm( return EXIT.OK updated_tmpls = [ - vm for vm in vm_updated - if bool(template_statuses[vm.name]) and vm.klass == 'TemplateVM' + vm + for vm in vm_updated + if bool(template_statuses[vm.name]) and vm.klass == "TemplateVM" ] to_restart, to_shutdown = get_derived_vm_to_apply( - updated_tmpls, derived_statuses) - templates_to_shutdown = [template for template in updated_tmpls - if template.is_running()] + updated_tmpls, derived_statuses + ) + templates_to_shutdown = [ + template for template in updated_tmpls if template.is_running() + ] if args.dry_run: - print("Following templates will be shutdown:", - ",".join((target.name for target in templates_to_shutdown))) + print( + "Following templates will be shutdown:", + ",".join((target.name for target in templates_to_shutdown)), + ) # we do not check if any volume is outdated, we expect it will be. - print("Following qubes CAN be restarted:", - ",".join((target.name for target in to_restart))) - print("Following qubes CAN be shutdown:", - ",".join((target.name for target in to_shutdown))) + print( + "Following qubes CAN be restarted:", + ",".join((target.name for target in to_restart)), + ) + print( + "Following qubes CAN be shutdown:", + ",".join((target.name for target in to_shutdown)), + ) return EXIT.OK # first shutdown templates to apply changes to the root volume @@ -387,14 +472,17 @@ def apply_updates_to_appvm( log.error("Shutdown of some templates fails with code %d", ret_code) log.warning( "Derived VMs of the following templates will be omitted: %s", - ", ".join((t.name for t in updated_tmpls if t.is_running()))) + ", ".join((t.name for t in updated_tmpls if t.is_running())), + ) ret_code = EXIT.ERR_SHUTDOWN_TMPL # Some templates are not down dur to errors, there is no point in # restarting their derived AppVMs - ready_templates = [tmpl for tmpl in updated_tmpls - if not tmpl.is_running()] + ready_templates = [ + tmpl for tmpl in updated_tmpls if not tmpl.is_running() + ] to_restart, to_shutdown = get_derived_vm_to_apply( - ready_templates, derived_statuses) + ready_templates, derived_statuses + ) # both flags `restart` and `apply-to-all` include service vms ret_code_ = restart_vms(to_restart, log) @@ -416,10 +504,12 @@ def get_derived_vm_to_apply(templates, derived_statuses): to_shutdown = set() for vm in possibly_changed_vms: - if (not bool(derived_statuses.get(vm.name, False)) - and vm.is_running() - and (vm.klass != 'DispVM' or not vm.auto_cleanup)): - if get_boolean_feature(vm, 'servicevm', False): + if ( + not bool(derived_statuses.get(vm.name, False)) + and vm.is_running() + and (vm.klass != "DispVM" or not vm.auto_cleanup) + ): + if get_boolean_feature(vm, "servicevm", False): to_restart.add(vm) else: to_shutdown.add(vm) @@ -463,5 +553,5 @@ def restart_vms(to_restart, log): return ret_code -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) From a379ed05dcd1fc7e7f3bc3d98792e82c892f3714 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Wed, 10 Dec 2025 13:56:23 +0100 Subject: [PATCH 10/13] vmupdate: recognize skip-update and prohibit-start features VMs with the above features should be excluded from updating. --- vmupdate/vmupdate.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index eaff377..edc6cb8 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -327,6 +327,17 @@ def select_targets(targets, args) -> Set[qubesadmin.vm.QubesVM]: to_update = vm.features.get("updates-available", False) except qubesadmin.exc.QubesDaemonCommunicationError: to_update = False + try: + prohibit_start = vm.features.get("prohibit-start", False) + except qubesadmin.exc.QubesDaemonCommunicationError: + prohibit_start = False + try: + skip_update = vm.features.get("skip-update", False) + except qubesadmin.exc.QubesDaemonCommunicationError: + skip_update = False + + if skip_update or prohibit_start: + continue # there are updates available => select if to_update: From 69c1a80c86381439681c349057ec38e36fce6aa5 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Sat, 3 Jan 2026 19:28:01 +0100 Subject: [PATCH 11/13] vmupdate: update pacman init and fix qubes-dom0-update issue --- dom0-updates/qubes-dom0-update | 2 +- vmupdate/agent/source/pacman/pacman_cli.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index 2b718ce..f930133 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -398,7 +398,7 @@ else CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" fi if [ "$PROGRESS_REPORTING" == "1" ]; then - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/``^/$(hostname):out: /" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/^/$(hostname):out: /" # "consume" the last empty line echo "" else diff --git a/vmupdate/agent/source/pacman/pacman_cli.py b/vmupdate/agent/source/pacman/pacman_cli.py index e0b9764..590ae0e 100644 --- a/vmupdate/agent/source/pacman/pacman_cli.py +++ b/vmupdate/agent/source/pacman/pacman_cli.py @@ -21,14 +21,18 @@ from typing import List, Dict -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class PACMANCLI(PackageManager): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) + PROGRESS_REPORTING = False + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + if self.type is AgentType.UPDATE_VM: + raise NotImplementedError("Pacman do not support update proxy VM.") self.package_manager = "pacman" # pylint: disable=unused-argument From a7074f9c4e2bcf74e95511b2276a42898c296662 Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Sun, 4 Jan 2026 13:06:18 +0100 Subject: [PATCH 12/13] vmupdate: error handling and fix output buffer --- dom0-updates/qubes-dom0-update | 37 ++++++++++++++++------------------ vmupdate/update_manager.py | 2 +- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index f930133..3da473a 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -339,7 +339,7 @@ version_check() { # Compare version numbers (e.g., 4.2 < 4.3) awk 'BEGIN {exit !(ARGV[1] < ARGV[2])}' "$1" "$2" } -base_vm=$(get_base_vm $UPDATEVM) +base_vm=$(get_base_vm "$UPDATEVM") OLD_VERSION=0 if [ -n "$base_vm" ]; then agent_version=$(qvm-features "$base_vm" qubes-agent-version 2>/dev/null) @@ -352,29 +352,28 @@ fi if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "0" ]; then CMD="/usr/lib/qubes/qubes-download-dom0-updates-init.sh" - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" || exit 1 update_agent_log="/var/log/qubes/qubes-update" qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$update_agent_log'" || exit 1 # "--no-cleanup" is needed since fakeroot cannot remove entrypoint - qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=DEBUG - RETCODE=$? - if [ "$RETCODE" -eq 100 ]; then + qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=INFO ; RETCODE=$? + if [ "${RETCODE-0}" -eq 100 ]; then echo "$(hostname):out: Nothing to do." echo "$(hostname) done no_updates" >&2 exit 100 fi - if [ "$RETCODE" -ne 0 ]; then + if [ "${RETCODE-0}" -ne 0 ]; then echo "$(hostname) done error" >&2 - exit $RETCODE + exit "$RETCODE" fi # qubes-vm-update leaves the downloaded packages with root ownership qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$dom0_updates_dir'" || exit 1 CMD="/usr/lib/qubes/qubes-download-dom0-updates-finish.sh" - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" || exit 1 else if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then echo "$(hostname):out: Progress reporting requires updateVM based on a template with Qubes 4.3 packages." @@ -398,33 +397,32 @@ else CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" fi if [ "$PROGRESS_REPORTING" == "1" ]; then - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/^/$(hostname):out: /" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/^/$(hostname):out: /" ; RETCODE=$? # "consume" the last empty line echo "" else - qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null ; RETCODE=$? fi fi -RETCODE=$? if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then echo "$(hostname) updating 50.0" >&2 - if [ "$RETCODE" -ne 0 ]; then + if [ "${RETCODE-0}" -ne 0 ]; then echo "$(hostname) done error" >&2 - exit $RETCODE + exit "$RETCODE" fi fi -if [[ "$REMOTE_ONLY" = '1' ]] || [ "$RETCODE" -ne 0 ]; then +if [[ "$REMOTE_ONLY" = '1' ]] || [ "${RETCODE-0}" -ne 0 ]; then if [ "$CHECK_ONLY" = '1' ]; then - if [ "$RETCODE" -eq 100 ]; then + if [ "${RETCODE-0}" -eq 100 ]; then echo "There are dom0 updates available" >&2 - elif [ "$RETCODE" -eq 0 ]; then + elif [ "${RETCODE-0}" -eq 0 ]; then echo "No dom0 updates available" >&2 else echo "Failed to check for dom0 updates" >&2 fi fi - exit $RETCODE + exit "${RETCODE-0}" fi # Wait for download completed while pidof -x qubes-receive-updates >/dev/null; do sleep 0.5; done @@ -484,7 +482,7 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then $guiapp elif [ "$PROGRESS_REPORTING" == 1 ]; then # report progress to the user - qubes-vm-update --no-refresh --targets dom0 --force-update --log=DEBUG --just-print-progress --show-output + qubes-vm-update --no-refresh --targets dom0 --force-update --log=DEBUG --just-print-progress --show-output ; RETCODE=$? else dnf check-update || if [ $? -eq 100 ]; then # Run dnf with options @@ -500,9 +498,8 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then fi fi else - if ! qvm-features dom0 updates-available '' 2>/dev/null; then + qvm-features dom0 updates-available '' >/dev/null 2>&1 || echo "*** WARNING: cannot set feature 'updates-available'" >&2 - fi echo "No updates available" >&2 if [ "$GUI" == "1" ]; then if [ "$KDE_FULL_SESSION" ]; then diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index 53060e0..a0017c1 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -150,7 +150,7 @@ def collect_result(self, result_tuple: Tuple[str, ProcessResult]): def print(self, *args): if self.buffered: - self.buffer += " ".join(str(args)) + "\n" + self.buffer += " ".join(map(str, args)) + "\n" else: print(*args, file=sys.stdout, flush=True) From 4824edd343b1bcbb4da1c0ecc421c886ae4d28fb Mon Sep 17 00:00:00 2001 From: Piotr Bartman-Szwarc Date: Tue, 6 Jan 2026 23:03:04 +0100 Subject: [PATCH 13/13] vmupdate: rename --quiet to --silent to avoid name collision https://github.com/QubesOS/qubes-issues/issues/10543#issuecomment-3716492445 --- dom0-updates/qubes-dom0-update | 4 ++-- vmupdate/update_manager.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index 3da473a..1ae565b 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -28,7 +28,7 @@ if [ "$1" = "--help" ]; then echo " --force-xen-upgrade force major Xen upgrade even if some qubes are running" echo " --console does nothing; ignored for backward compatibility" echo " --show-output does nothing; ignored for backward compatibility" - echo " --quiet do not print anything to stdout" + echo " --silent do not print anything to stdout" echo " --preserve-terminal does nothing; ignored for backward compatibility" echo " --skip-boot-check does not check if /boot & /boot/efi should be mounted" echo " --switch-audio-server-to=(pulseaudio|pipewire) switch audio daemon to pipewire or pulseaudio" @@ -81,7 +81,7 @@ while [ $# -gt 0 ]; do --show-output) # ignore ;; - --quiet) + --silent) exec > /dev/null ;; --just-print-progress) diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index a0017c1..fdabe4e 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -425,7 +425,8 @@ def _run_agent( if agent_args.just_print_progress or self.show_progress: entrypoint.append("--just-print-progress") if agent_args.quiet: - entrypoint.append("--quiet") + # silent is equivalent to quiet for dom0-update + entrypoint.append("--silent") else: this_dir = os.path.dirname(os.path.realpath(__file__)) entrypoint = join(this_dir, UpdateAgentManager.ENTRYPOINT)