diff --git a/dom0-updates/qubes-dom0-update b/dom0-updates/qubes-dom0-update index 5b7d177..1ae565b 100755 --- a/dom0-updates/qubes-dom0-update +++ b/dom0-updates/qubes-dom0-update @@ -28,6 +28,7 @@ if [ "$1" = "--help" ]; then echo " --force-xen-upgrade force major Xen upgrade even if some qubes are running" echo " --console does nothing; ignored for backward compatibility" echo " --show-output does nothing; ignored for backward compatibility" + echo " --silent do not print anything to stdout" echo " --preserve-terminal does nothing; ignored for backward compatibility" echo " --skip-boot-check does not check if /boot & /boot/efi should be mounted" echo " --switch-audio-server-to=(pulseaudio|pipewire) switch audio daemon to pipewire or pulseaudio" @@ -46,6 +47,7 @@ YUM_OPTS=() UPDATEVM_OPTS=() QVMTEMPLATE_OPTS=() GUI= +PROGRESS_REPORTING= CHECK_ONLY= CLEAN= TEMPLATE= @@ -79,6 +81,12 @@ while [ $# -gt 0 ]; do --show-output) # ignore ;; + --silent) + exec > /dev/null + ;; + --just-print-progress) + PROGRESS_REPORTING=1 + ;; --check-only) CHECK_ONLY=1 UPDATEVM_OPTS+=( "$1" ) @@ -310,40 +318,111 @@ qvm-run --nogui -q -- "$UPDATEVM" "rm -rf -- '$dom0_updates_dir/etc' '$dom0_upda exit "$status" } -CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui" +QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io) -# We avoid using bash’s own facilities for this, as they produce $'\n'-style -# strings in certain cases. These are not portable, whereas the string produced -# by the following is. -for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done +progress_agent_version="4.3" -QVMRUN_OPTS=(--quiet --filter-escape-chars --nogui --pass-io) -if [[ -t 1 ]] && [[ -t 2 ]]; then - # Use ‘script’ to emulate a TTY, so that we get status bars and other - # progress output. Since stdout and stderr are both terminals, qvm-run - # will automatically sanitize them, but we explicitly tell it to anyway - # as a precaution. - # - # We MUST NOT use ‘exec script’ here. That causes ‘script’ to - # inherit the child processes of the shell. ‘script’ mishandles - # this and enters an infinite loop. - CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" +get_base_vm() { + # Resolve base VM (TemplateVM or StandaloneVM) + local vm="$1" + while true; do + # If it's a TemplateVM or StandaloneVM, stop + if qvm-check --template "$vm" &>/dev/null || qvm-check --standalone "$vm" &>/dev/null; then + echo "$vm" + return + fi + # Try to get its template + vm=$(qvm-prefs "$vm" template 2>/dev/null) || return 1 + done +} +version_check() { + # Compare version numbers (e.g., 4.2 < 4.3) + awk 'BEGIN {exit !(ARGV[1] < ARGV[2])}' "$1" "$2" +} +base_vm=$(get_base_vm "$UPDATEVM") +OLD_VERSION=0 +if [ -n "$base_vm" ]; then + agent_version=$(qvm-features "$base_vm" qubes-agent-version 2>/dev/null) + + if [ -n "$agent_version" ] && version_check "$agent_version" "$progress_agent_version"; then + OLD_VERSION=1 + fi fi -qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null -RETCODE=$? -if [[ "$REMOTE_ONLY" = '1' ]] || [ "$RETCODE" -ne 0 ]; then +if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "0" ]; then + CMD="/usr/lib/qubes/qubes-download-dom0-updates-init.sh" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" || exit 1 + + update_agent_log="/var/log/qubes/qubes-update" + qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$update_agent_log'" || exit 1 + + # "--no-cleanup" is needed since fakeroot cannot remove entrypoint + qubes-vm-update --force-update --targets "$UPDATEVM" --signal-no-updates --just-print-progress --display-name dom0 --download-only --no-cleanup --show-output --log=INFO ; RETCODE=$? + if [ "${RETCODE-0}" -eq 100 ]; then + echo "$(hostname):out: Nothing to do." + echo "$(hostname) done no_updates" >&2 + exit 100 + fi + if [ "${RETCODE-0}" -ne 0 ]; then + echo "$(hostname) done error" >&2 + exit "$RETCODE" + fi + + # qubes-vm-update leaves the downloaded packages with root ownership + qvm-run --nogui -q -u root -- "$UPDATEVM" "user=\$(qubesdb-read /default-user) && chown -R -- \"\$user:qubes\" '$dom0_updates_dir'" || exit 1 + + CMD="/usr/lib/qubes/qubes-download-dom0-updates-finish.sh" + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" || exit 1 +else + if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then + echo "$(hostname):out: Progress reporting requires updateVM based on a template with Qubes 4.3 packages." + fi + CMD="/usr/lib/qubes/qubes-download-dom0-updates.sh --doit --nogui" + + # We avoid using bash’s own facilities for this, as they produce $'\n'-style + # strings in certain cases. These are not portable, whereas the string produced + # by the following is. + for i in "${UPDATEVM_OPTS[@]}"; do CMD+=" '${i//\'/\'\\\'\'}'"; done + + if [[ -t 1 ]] && [[ -t 2 ]]; then + # Use ‘script’ to emulate a TTY, so that we get status bars and other + # progress output. Since stdout and stderr are both terminals, qvm-run + # will automatically sanitize them, but we explicitly tell it to anyway + # as a precaution. + # + # We MUST NOT use ‘exec script’ here. That causes ‘script’ to + # inherit the child processes of the shell. ‘script’ mishandles + # this and enters an infinite loop. + CMD="script --quiet --return --command '${CMD//\'/\'\\\'\'}' /dev/null" + fi + if [ "$PROGRESS_REPORTING" == "1" ]; then + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null | sed "s/^/$(hostname):out: /" ; RETCODE=$? + # "consume" the last empty line + echo "" + else + qvm-run "${QVMRUN_OPTS[@]}" -- "$UPDATEVM" "$CMD" < /dev/null ; RETCODE=$? + fi +fi + +if [ "$PROGRESS_REPORTING" == "1" ] && [ "$OLD_VERSION" == "1" ]; then + echo "$(hostname) updating 50.0" >&2 + if [ "${RETCODE-0}" -ne 0 ]; then + echo "$(hostname) done error" >&2 + exit "$RETCODE" + fi +fi +if [[ "$REMOTE_ONLY" = '1' ]] || [ "${RETCODE-0}" -ne 0 ]; then if [ "$CHECK_ONLY" = '1' ]; then - if [ "$RETCODE" -eq 100 ]; then + if [ "${RETCODE-0}" -eq 100 ]; then echo "There are dom0 updates available" >&2 - elif [ "$RETCODE" -eq 0 ]; then + elif [ "${RETCODE-0}" -eq 0 ]; then echo "No dom0 updates available" >&2 else echo "Failed to check for dom0 updates" >&2 fi fi - exit $RETCODE + exit "${RETCODE-0}" fi # Wait for download completed while pidof -x qubes-receive-updates >/dev/null; do sleep 0.5; done @@ -401,6 +480,9 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then # refresh packagekit metadata, GUI utilities use it pkcon refresh force $guiapp + elif [ "$PROGRESS_REPORTING" == 1 ]; then + # report progress to the user + qubes-vm-update --no-refresh --targets dom0 --force-update --log=DEBUG --just-print-progress --show-output ; RETCODE=$? else dnf check-update || if [ $? -eq 100 ]; then # Run dnf with options @@ -416,9 +498,8 @@ elif [ -f /var/lib/qubes/updates/repodata/repomd.xml ]; then fi fi else - if ! qvm-features dom0 updates-available '' 2>/dev/null; then + qvm-features dom0 updates-available '' >/dev/null 2>&1 || echo "*** WARNING: cannot set feature 'updates-available'" >&2 - fi echo "No updates available" >&2 if [ "$GUI" == "1" ]; then if [ "$KDE_FULL_SESSION" ]; then diff --git a/rpm_spec/core-dom0-linux.spec.in b/rpm_spec/core-dom0-linux.spec.in index f3d9dce..dcbafeb 100644 --- a/rpm_spec/core-dom0-linux.spec.in +++ b/rpm_spec/core-dom0-linux.spec.in @@ -48,6 +48,7 @@ Requires: qubes-core-admin-client Requires: qubes-utils >= 3.1.3 Requires: qubes-utils-libs >= 4.0.16 Requires: qubes-rpm-oxide +Requires: python-libdnf5 Conflicts: qubes-core-dom0 < 4.0.23 Requires: %{name}-kernel-install Requires: xdotool diff --git a/vmupdate/__init__.py b/vmupdate/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/__init__.py +++ b/vmupdate/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/__init__.py b/vmupdate/agent/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/__init__.py +++ b/vmupdate/agent/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/entrypoint.py b/vmupdate/agent/entrypoint.py index 79e7a40..157e887 100755 --- a/vmupdate/agent/entrypoint.py +++ b/vmupdate/agent/entrypoint.py @@ -8,6 +8,7 @@ from source.utils import get_os_data from source.log_config import init_logs from source.common.exit_codes import EXIT +from source.common.package_manager import AgentType def main(args=None): @@ -16,23 +17,39 @@ def main(args=None): """ args = parse_args(args) log, log_handler, log_level, _log_path, _log_formatter = init_logs( - level=args.log, truncate_file=True) + level=args.log, truncate_file=True + ) log.debug("Run entrypoint with args: %s", str(args)) os_data = get_os_data() log.debug("Selecting package manager.") + agent_type = AgentType.VM + if os_data["id"] == "qubes": + agent_type = AgentType.DOM0 + if args.download_only: + agent_type = AgentType.UPDATE_VM pkg_mng = get_package_manager( - os_data, log, log_handler, log_level, args.no_progress) + os_data, log, log_handler, log_level, agent_type, args.no_progress + ) log.debug("Running upgrades.") - return_code = pkg_mng.upgrade(refresh=not args.no_refresh, - hard_fail=not args.force_upgrade, - remove_obsolete=not args.leave_obsolete, - print_streams=args.show_output - ) - - log.debug("Notify dom0 about upgrades.") - os.system("/usr/lib/qubes/upgrades-status-notify") + return_code = pkg_mng.upgrade( + refresh=not args.no_refresh, + hard_fail=not args.force_upgrade, + remove_obsolete=not args.leave_obsolete, + print_streams=args.show_output, + ) + + if not pkg_mng.PROGRESS_REPORTING and not args.no_progress: + # even if progress reporting is unavailable we want info that update finished + if agent_type is AgentType.UPDATE_VM: + print(f"{55:.2f}", flush=True, file=sys.stderr) + else: + print(f"{100:.2f}", flush=True, file=sys.stderr) + + if agent_type is AgentType.VM: + log.debug("Notify dom0 about upgrades.") + os.system("/usr/lib/qubes/upgrades-status-notify") if not args.no_cleanup: return_code = max(pkg_mng.clean(), return_code) @@ -49,68 +66,136 @@ def parse_args(args): return args -def get_package_manager(os_data, log, log_handler, log_level, no_progress): +def get_package_manager( + os_data, log, log_handler, log_level, agent_type, no_progress +): """ Returns instance of `PackageManager`. If appropriate python package is not installed or `no_progress` is `True` cli based version is returned. """ + # pylint: disable=import-outside-toplevel requirements = {} # plugins MUST be applied before import anything from package managers. # in case of apt configuration is loaded on `import apt`. - for plugin in plugins.entrypoints: - plugin(os_data, log, requirements=requirements) + if agent_type is not AgentType.UPDATE_VM: + for plugin in plugins.entrypoints: + plugin(os_data, log, requirements=requirements) + + if os_data["os_family"] == "RedHat" or agent_type is AgentType.UPDATE_VM: + PackageManager = import_rhel_package_manager(os_data, log, no_progress) + elif os_data["os_family"] == "Debian": + PackageManager = import_debian_package_manager(log, no_progress) + elif os_data["os_family"] == "ArchLinux": + from source.pacman.pacman_cli import PACMANCLI as PackageManager + + print("Progress reporting not supported.", flush=True) + elif os_data["os_family"] == "Qubes": + PackageManager = import_dom0_package_manager(os_data, log, no_progress) + else: + raise NotImplementedError( + "Only Debian, RedHat and ArchLinux based OS is supported." + ) + + pkg_mng = PackageManager(log_handler, log_level, agent_type) + pkg_mng.requirements = requirements + return pkg_mng - if os_data["os_family"] == "Debian": - try: - from source.apt.apt_api import APT as PackageManager - except ImportError: - log.warning("Failed to load apt with progress bar. Using apt cli.") - # no progress reporting - no_progress = True - print(f"Progress reporting not supported.", flush=True) - if no_progress: - from source.apt.apt_cli import APTCLI as PackageManager - elif os_data["os_family"] == "RedHat": +def import_rhel_package_manager(os_data, log, no_progress): + """ + Import dnf package manager. + """ + # pylint: disable=import-outside-toplevel + dnf5_fedora_version = 41 + if os_data["os_family"] == "RedHat": try: version = int(os_data["release"].split(".")[0]) except ValueError: version = 99 # fedora changed its version - - loaded = False - if version >= 41: - try: - from source.dnf.dnf5_api import DNF as PackageManager - loaded = True - except ImportError: - log.warning("Failed to load dnf5.") - - if not loaded: - try: - from source.dnf.dnf_api import DNF as PackageManager - loaded = True - except ImportError: - log.warning( - "Failed to load dnf with progress bar. Using dnf cli.") - print(f"Progress reporting not supported.", flush=True) - - if no_progress or not loaded: - from source.dnf.dnf_cli import DNFCLI as PackageManager - elif os_data["os_family"] == "ArchLinux": - from source.pacman.pacman_cli import PACMANCLI as PackageManager - print(f"Progress reporting not supported.", flush=True) else: - raise NotImplementedError( - "Only Debian, RedHat and ArchLinux based OS is supported.") + version = dnf5_fedora_version # try to use whatever is available, starting from dnf5 - pkg_mng = PackageManager(log_handler, log_level) - pkg_mng.requirements = requirements - return pkg_mng + loaded = False + if version >= dnf5_fedora_version: + try: + from source.dnf.dnf5_api import DNF5 as PackageManager + + loaded = True + log.info("Using dnf5.") + except ImportError: + log.warning("Failed to load dnf5.") + + if not loaded: + try: + from source.dnf.dnf_api import DNF as PackageManager + + loaded = True + log.debug("Using dnf python API for progress reporting.") + except ImportError: + print("Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + log.warning("Failed to load dnf with progress bar. Using dnf cli.") + from source.dnf.dnf_cli import DNFCLI as PackageManager + + return PackageManager + + +def import_debian_package_manager(log, no_progress): + """ + Import apt package manager. + """ + # pylint: disable=import-outside-toplevel + loaded = False + try: + from source.apt.apt_api import APT as PackageManager + + loaded = True + except ImportError: + log.warning("Failed to load apt with progress bar. Using apt cli.") + print("Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + from source.apt.apt_cli import APTCLI as PackageManager + + return PackageManager + + +def import_dom0_package_manager(os_data, log, no_progress): + """ + Import dnf package manager for dom0. + """ + # pylint: disable=import-outside-toplevel + major, minor = os_data["release"].split(".") + major, minor = int(major), int(minor) + loaded = False + if major >= 5 or (major == 4 and minor >= 3): + try: + from source.dnf.dnf5_api import DNF5 as PackageManager + + loaded = True + except ImportError: + log.warning("Failed to load dnf5.") + + if not loaded: + try: + from source.dnf.dnf_api import DNF as PackageManager + + loaded = True + log.debug("Using dnf python API for progress reporting.") + except ImportError: + print(f"Progress reporting not supported.", flush=True) + + if no_progress or not loaded: + log.warning("Failed to load dnf with progress bar. Using dnf cli.") + from source.dnf.dnf_cli import DNFCLI as PackageManager + + return PackageManager -if __name__ == '__main__': +if __name__ == "__main__": try: sys.exit(main()) except RuntimeError as ex: diff --git a/vmupdate/agent/source/__init__.py b/vmupdate/agent/source/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/__init__.py +++ b/vmupdate/agent/source/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/apt/__init__.py b/vmupdate/agent/source/apt/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/apt/__init__.py +++ b/vmupdate/agent/source/apt/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/apt/apt_api.py b/vmupdate/agent/source/apt/apt_api.py index 543ad31..f551b2a 100644 --- a/vmupdate/agent/source/apt/apt_api.py +++ b/vmupdate/agent/source/apt/apt_api.py @@ -26,6 +26,7 @@ import apt.progress.base import apt_pkg +from source.common.package_manager import AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress @@ -34,17 +35,20 @@ class APT(APTCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) self.apt_cache = apt.Cache() update = FetchProgress( - weight=4, log=self.log, refresh=True) # 4% of total time + weight=4, log=self.log, refresh=True + ) # 4% of total time fetch = FetchProgress(weight=48, log=self.log) # 48% of total time upgrade = UpgradeProgress(weight=48, log=self.log) # 48% of total time self.progress = ProgressReporter(update, fetch, upgrade) # to prevent a warning: `debconf: unable to initialize frontend: Dialog` - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ["DEBIAN_FRONTEND"] = "noninteractive" def refresh(self, hard_fail: bool) -> ProcessResult: """ @@ -59,7 +63,7 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.log.debug("Refreshing available packages...") success = self.apt_cache.update( self.progress.update_progress, - pulse_interval=1000 # microseconds + pulse_interval=1000, # microseconds ) self.apt_cache.open() if success: @@ -69,7 +73,8 @@ def refresh(self, hard_fail: bool) -> ProcessResult: result += ProcessResult(EXIT.ERR_VM_REFRESH) except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) return result @@ -82,20 +87,22 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: try: self.log.debug("Performing package upgrade...") self.apt_cache.upgrade(dist_upgrade=remove_obsolete) - Path(os.path.join( - apt_pkg.config.find_dir("Dir::Cache::Archives"), "partial") + Path( + os.path.join( + apt_pkg.config.find_dir("Dir::Cache::Archives"), "partial" + ) ).mkdir(parents=True, exist_ok=True) - apt_pkg.config.set('Dpkg::Options::', "--force-confdef") - apt_pkg.config.set('Dpkg::Options::', "--force-confold") + apt_pkg.config.set("Dpkg::Options::", "--force-confdef") + apt_pkg.config.set("Dpkg::Options::", "--force-confold") self.log.debug("Committing upgrade...") self.apt_cache.commit( - self.progress.fetch_progress, - self.progress.upgrade_progress + self.progress.fetch_progress, self.progress.upgrade_progress ) self.log.debug("Package upgrade successful.") except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) if remove_obsolete: @@ -115,9 +122,12 @@ def fail(self, item): Write an error message to the fake stderr. """ self.log.info(f"{self.action.capitalize()} failed.") - print(f"Fail to {self.action} {item.shortdesc}: " - f"{item.description} from {item.uri}", - flush=True, file=self._stderr) + print( + f"Fail to {self.action} {item.shortdesc}: " + f"{item.description} from {item.uri}", + flush=True, + file=self._stderr, + ) def pulse(self, _owner): """ @@ -128,9 +138,11 @@ def pulse(self, _owner): acquisition should be continued (True) or cancelled (False). """ if self.action == "fetch" and not self.fetching_notified: - print(f"Fetching {self.total_items} packages " - f"[{self._format_bytes(self.total_bytes)}]", - flush=True) + print( + f"Fetching {self.total_items} packages " + f"[{self._format_bytes(self.total_bytes)}]", + flush=True, + ) self.fetching_notified = True self.notify_callback(self.current_bytes / self.total_bytes * 100) return True @@ -166,8 +178,11 @@ def error(self, pkg, errormsg): """ Write an error message to the fake stderr. """ - print("Error during installation " + str(pkg) + ":" + str(errormsg), - flush=True, file=self._stderr) + print( + "Error during installation " + str(pkg) + ":" + str(errormsg), + flush=True, + file=self._stderr, + ) def start_update(self): print("Updating packages.", flush=True) diff --git a/vmupdate/agent/source/apt/apt_cli.py b/vmupdate/agent/source/apt/apt_cli.py index 9779ad7..4ff1f27 100644 --- a/vmupdate/agent/source/apt/apt_cli.py +++ b/vmupdate/agent/source/apt/apt_cli.py @@ -26,18 +26,22 @@ import contextlib from typing import List -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class APTCLI(PackageManager): - def __init__(self, log_handler, log_level,): - super().__init__(log_handler, log_level,) + PROGRESS_REPORTING = False + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + if self.type is AgentType.UPDATE_VM: + raise NotImplementedError("APT do not support update proxy VM.") self.package_manager: str = "apt-get" # to prevent a warning: `debconf: unable to initialize frontend: Dialog` - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ["DEBIAN_FRONTEND"] = "noninteractive" @contextlib.contextmanager def apt_lock(self): @@ -67,8 +71,13 @@ def refresh(self, hard_fail: bool) -> ProcessResult: # apply lock externally to wait for it, until # https://bugs.debian.org/1069167 gets implemented with self.apt_lock(): - cmd = [self.package_manager, - "-o", "Debug::NoLocking=true", "-q", "update"] + cmd = [ + self.package_manager, + "-o", + "Debug::NoLocking=true", + "-q", + "update", + ] result = self.run_cmd(cmd) # 'apt-get update' reports error with exit code 100, but updater as a # whole reserves it for "no updates" @@ -115,10 +124,13 @@ def get_action(self, remove_obsolete: bool) -> List[str]: """ Return command `upgrade` or `dist-upgrade` if `remove_obsolete`. """ - result = ["-y", - "-o", 'Dpkg::Options::=--force-confdef', - "-o", 'Dpkg::Options::=--force-confold' - ] + result = [ + "-y", + "-o", + "Dpkg::Options::=--force-confdef", + "-o", + "Dpkg::Options::=--force-confold", + ] result += ["dist-upgrade"] if remove_obsolete else ["upgrade"] return result @@ -149,7 +161,7 @@ def remove_obsolete_kernels(self) -> ProcessResult: obsoletes = set() for line in result.out.splitlines(): if line.startswith("Remv"): - package_name = line[len("Remv "):] + package_name = line[len("Remv ") :] # consider using wider pattern if package_name.startswith("linux-image"): obsoletes.add(package_name.split(" ")[0]) diff --git a/vmupdate/agent/source/args.py b/vmupdate/agent/source/args.py index 5b0593c..1090bb3 100644 --- a/vmupdate/agent/source/args.py +++ b/vmupdate/agent/source/args.py @@ -24,36 +24,53 @@ class AgentArgs: # To avoid code repeating when we want to retrieve arguments OPTIONS = { - ("--log",): {"action": 'store', - "default": "INFO", - "help": 'Provide logging level. Values: DEBUG, ' - 'INFO (default), WARNING, ERROR, CRITICAL'}, - ("--no-refresh",): {"action": 'store_true', - "help": 'Do not refresh available packages before ' - 'upgrading'}, - ("--force-upgrade", "-f"): - {"action": 'store_true', - "help": 'Try upgrade even if errors are ' - 'encountered (like a refresh error)'}, + ("--log",): { + "action": "store", + "default": "INFO", + "help": "Provide logging level. Values: DEBUG, " + "INFO (default), WARNING, ERROR, CRITICAL", + }, + ("--no-refresh",): { + "action": "store_true", + "help": "Do not refresh available packages before " "upgrading", + }, + ("--force-upgrade", "-f"): { + "action": "store_true", + "help": "Try upgrade even if errors are " + "encountered (like a refresh error)", + }, ("--no-cleanup",): { - "action": 'store_true', - "help": 'Do not remove cache files after upgrading'}, + "action": "store_true", + "help": "Do not remove cache files after upgrading", + }, ("--leave-obsolete",): { - "action": 'store_true', - "help": 'Do not remove updater and cache files from target qube'}, + "action": "store_true", + "help": "Do not remove updater and cache files from target qube", + }, + ("--download-only",): { + "action": "store_true", + "help": "Only download packages", + }, } EXCLUSIVE_OPTIONS_1 = { - ("--show-output", "--verbose", "-v"): - {"action": 'store_true', - "help": 'Show output of management commands'}, - ("--quiet", "-q"): {"action": 'store_true', - "help": 'Do not print anything to stdout'} + ("--show-output", "--verbose", "-v"): { + "action": "store_true", + "help": "Show output of management commands", + }, + ("--quiet", "-q"): { + "action": "store_true", + "help": "Do not print anything to stdout", + }, } EXCLUSIVE_OPTIONS_2 = { - ("--no-progress",): {"action": "store_true", - "help": "Do not show upgrading progress."}, - ("--just-print-progress",): {"action": "store_true", - "help": argparse.SUPPRESS} + ("--no-progress",): { + "action": "store_true", + "help": "Do not show upgrading progress.", + }, + ("--just-print-progress",): { + "action": "store_true", + "help": argparse.SUPPRESS, + }, } ALL_OPTIONS = {**OPTIONS, **EXCLUSIVE_OPTIONS_1, **EXCLUSIVE_OPTIONS_2} diff --git a/vmupdate/agent/source/common/__init__.py b/vmupdate/agent/source/common/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/common/__init__.py +++ b/vmupdate/agent/source/common/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/common/package_manager.py b/vmupdate/agent/source/common/package_manager.py index 266a4ba..24971e8 100644 --- a/vmupdate/agent/source/common/package_manager.py +++ b/vmupdate/agent/source/common/package_manager.py @@ -18,33 +18,43 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -""" package manager for VMs """ +"""package manager for VMs""" import io import logging import subprocess import sys +import enum from typing import Optional, Dict, List from .process_result import ProcessResult from .exit_codes import EXIT +class AgentType(enum.Enum): + VM = "Downloads and install updates in VM" + DOM0 = "Install downloaded updates in dom0" + UPDATE_VM = "Downloads updates for dom0" + + class PackageManager: - """ main package manager class """ - def __init__(self, log_handler, log_level): + """main package manager class""" + + def __init__(self, log_handler, log_level, agent_type: AgentType): self.package_manager: Optional[str] = None self.log = logging.getLogger( - f'vm-update.agent.{self.__class__.__name__}') + f"vm-update.agent.{self.__class__.__name__}" + ) self.log.setLevel(log_level) self.log.addHandler(log_handler) self.log.propagate = False self.requirements: Optional[Dict[str, str]] = None + self.type = agent_type def upgrade( - self, - refresh: bool, - hard_fail: bool, - remove_obsolete: bool, - print_streams: bool = False + self, + refresh: bool, + hard_fail: bool, + remove_obsolete: bool, + print_streams: bool = False, ): """ Upgrade packages using system package manager. @@ -57,7 +67,8 @@ def upgrade( :return: return code """ result = self._upgrade( - refresh, hard_fail, remove_obsolete, self.requirements) + refresh, hard_fail, remove_obsolete, self.requirements + ) self._log_output("agent", result) if print_streams and not result.posted: if result.out: @@ -67,11 +78,11 @@ def upgrade( return result.code def _upgrade( - self, - refresh: bool, - hard_fail: bool, - remove_obsolete: bool, - requirements: Optional[Dict[str, str]] = None + self, + refresh: bool, + hard_fail: bool, + remove_obsolete: bool, + requirements: Optional[Dict[str, str]] = None, ) -> ProcessResult: result = ProcessResult(realtime=True) @@ -83,31 +94,43 @@ def _upgrade( if result_install: self.log.warning( "Installing requirements failed with exit code: %d", - result_install.code) + result_install.code, + ) result_install.code = EXIT.ERR_VM_PRE result += result_install if result and hard_fail: - self.log.error("Exiting due to a packages install error. " - "Use --force-upgrade to upgrade anyway.") + self.log.error( + "Exiting due to a packages install error. " + "Use --force-upgrade to upgrade anyway." + ) return result if refresh: print("Refreshing package info", flush=True) result_refresh = self.refresh(hard_fail) if result_refresh: - self.log.warning("Refreshing failed with code: %d", - result_refresh.code) + self.log.warning( + "Refreshing failed with code: %d", result_refresh.code + ) result_refresh.code = EXIT.ERR_VM_REFRESH result += result_refresh if result and hard_fail: - self.log.error("Exiting due to a refresh error. " - "Use --force-upgrade to upgrade anyway.") + self.log.error( + "Exiting due to a refresh error. " + "Use --force-upgrade to upgrade anyway." + ) return result result_upgrade = self.upgrade_internal(remove_obsolete) - if result_upgrade: + if result_upgrade.code not in (EXIT.OK, EXIT.OK_NO_UPDATES): result_upgrade.code = EXIT.ERR_VM_UPDATE result += result_upgrade + if result: + return result + + if self.type == AgentType.UPDATE_VM: + # No package installation is required in UpdateVM, so changes are not checked. + return result new_pkg = self.get_packages() @@ -136,9 +159,9 @@ def _log_output(self, title, result): log("%s err: %s", title, err_line) def install_requirements( - self, - requirements: Optional[Dict[str, str]], - curr_pkg: Dict[str, List[str]] + self, + requirements: Optional[Dict[str, str]], + curr_pkg: Dict[str, List[str]], ) -> ProcessResult: """ Make sure if required packages is installed before upgrading. @@ -159,25 +182,24 @@ def install_requirements( else: to_upgrade[pkg] = version if to_install: - cmd = [self.package_manager, - "-q", - "-y", - "install", - *to_install] + cmd = [self.package_manager, "-q", "-y", "install", *to_install] result += self.run_cmd(cmd) if to_upgrade: - cmd = [self.package_manager, - "-q", - "-y", - *self.get_action(remove_obsolete=False), - *to_upgrade] + cmd = [ + self.package_manager, + "-q", + "-y", + *self.get_action(remove_obsolete=False), + *to_upgrade, + ] result += self.run_cmd(cmd) return result def run_cmd( - self, command: List[str], realtime: bool = True) -> ProcessResult: + self, command: List[str], realtime: bool = True + ) -> ProcessResult: """ Run command and wait. @@ -190,10 +212,12 @@ def run_cmd( result = ProcessResult.process_communicate(proc) result.posted = True else: - with subprocess.Popen(command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + with subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as proc: result = ProcessResult.process_communicate(proc) self.log.debug("command exit code: %i", result.code) @@ -207,12 +231,15 @@ def compare_packages(old, new): :param old: Dict[package_name, version] packages before update :param new: Dict[package_name, version] packages after update """ - return {"installed": {pkg: new[pkg] for pkg in new if pkg not in old}, - "updated": {pkg: {"old": old[pkg], "new": new[pkg]} - for pkg in new - if pkg in old and old[pkg] != new[pkg] - }, - "removed": {pkg: old[pkg] for pkg in old if pkg not in new}} + return { + "installed": {pkg: new[pkg] for pkg in new if pkg not in old}, + "updated": { + pkg: {"old": old[pkg], "new": new[pkg]} + for pkg in new + if pkg in old and old[pkg] != new[pkg] + }, + "removed": {pkg: old[pkg] for pkg in old if pkg not in new}, + } def _print_changes(self, changes): result = ProcessResult() @@ -220,7 +247,8 @@ def _print_changes(self, changes): if changes["installed"]: for pkg in sorted(changes["installed"]): result.out += self._print_to_string( - pkg, changes["installed"][pkg]) + pkg, changes["installed"][pkg] + ) else: result.out += self._print_to_string("None") @@ -230,8 +258,9 @@ def _print_changes(self, changes): result.out += self._print_to_string( pkg, str(changes["updated"][pkg]["old"])[2:-2] - + " -> " + - str(changes["updated"][pkg]["new"])[2:-2]) + + " -> " + + str(changes["updated"][pkg]["new"])[2:-2], + ) else: result.out += self._print_to_string("None") @@ -239,7 +268,8 @@ def _print_changes(self, changes): if changes["removed"]: for pkg in sorted(changes["removed"]): result.out += self._print_to_string( - pkg, changes["removed"][pkg]) + pkg, changes["removed"][pkg] + ) else: result.out += self._print_to_string("None") return result diff --git a/vmupdate/agent/source/common/process_result.py b/vmupdate/agent/source/common/process_result.py index 5ce5702..6747bf5 100644 --- a/vmupdate/agent/source/common/process_result.py +++ b/vmupdate/agent/source/common/process_result.py @@ -31,10 +31,13 @@ class ProcessResult: Controls where the results of subprocesses are directed (e.g., to stdout or buffered). """ + def __init__( - self, - code: int = EXIT.OK, out: str = "", err: str = "", - realtime: bool = False + self, + code: int = EXIT.OK, + out: str = "", + err: str = "", + realtime: bool = False, ): self.code: int = code self.out: str = out @@ -56,12 +59,12 @@ def process_communicate(cls, proc): @classmethod def from_untrusted_out_err( - cls, - untrusted_out: Optional[Union[str, bytes]], - untrusted_err: Optional[Union[str, bytes]] = "" + cls, + untrusted_out: Optional[Union[str, bytes]], + untrusted_err: Optional[Union[str, bytes]] = "", ): if untrusted_out is None: - untrusted_out_bytes = b'' + untrusted_out_bytes = b"" elif isinstance(untrusted_out, str): untrusted_out_bytes: bytes = untrusted_out.encode() else: @@ -69,7 +72,7 @@ def from_untrusted_out_err( out = ProcessResult.sanitize_output(untrusted_out_bytes) if untrusted_err is None: - untrusted_err_bytes = b'' + untrusted_err_bytes = b"" elif isinstance(untrusted_err, str): untrusted_err_bytes: bytes = untrusted_err.encode() else: @@ -80,10 +83,14 @@ def from_untrusted_out_err( @staticmethod def sanitize_output(untrusted_bytes: bytes, single: bool = False) -> str: - untrusted_str = untrusted_bytes.decode('ascii', errors='ignore') - return ''.join([c for c in untrusted_str - if 0x20 <= ord(c) <= 0x7e - or (c == '\n' and not single)]) + untrusted_str = untrusted_bytes.decode("ascii", errors="ignore") + return "".join( + [ + c + for c in untrusted_str + if 0x20 <= ord(c) <= 0x7E or (c == "\n" and not single) + ] + ) def __add__(self, other): new = deepcopy(self) @@ -92,9 +99,11 @@ def __add__(self, other): def __iadd__(self, other): if not isinstance(other, ProcessResult): - raise TypeError("unsupported operand type(s) for +:" - f"'{self.__class__.__name__}' and " - f"'{other.__class__.__name__}'") + raise TypeError( + "unsupported operand type(s) for +:" + f"'{self.__class__.__name__}' and " + f"'{other.__class__.__name__}'" + ) self.code = max(self.code, other.code) self.out += other.out self.err += other.err @@ -113,6 +122,6 @@ def __repr__(self): return f"{self.code}; {self.out}; {self.err}" def error_from_messages(self): - out_lines = (self.out + '\n' + self.err).splitlines() + out_lines = (self.out + "\n" + self.err).splitlines() if any(line.lower().startswith("err") for line in out_lines): self.code = EXIT.ERR diff --git a/vmupdate/agent/source/common/progress_reporter.py b/vmupdate/agent/source/common/progress_reporter.py index e8cfdd2..6e4863c 100644 --- a/vmupdate/agent/source/common/progress_reporter.py +++ b/vmupdate/agent/source/common/progress_reporter.py @@ -27,9 +27,9 @@ class Progress: def __init__( - self, - weight: int, - log, + self, + weight: int, + log, ): self.weight = weight self._callback = None @@ -41,9 +41,12 @@ def __init__( self.log = log def init( - self, start: float, stop: float, - callback: Callable[[float], None], - stdout: io.TextIOWrapper, stderr: io.TextIOWrapper + self, + start: float, + stop: float, + callback: Callable[[float], None], + stdout: io.TextIOWrapper, + stderr: io.TextIOWrapper, ): self._callback = callback self._start_percent = start @@ -57,8 +60,10 @@ def notify_callback(self, percent): Report ongoing progress. """ assert self._start_percent is not None # call init() first! - _percent = self._start_percent + percent * ( - self._stop_percent - self._start_percent) / 100 + _percent = ( + self._start_percent + + percent * (self._stop_percent - self._start_percent) / 100 + ) _percent = round(_percent, 2) if self._last_percent < _percent: self._callback(_percent) @@ -84,20 +89,21 @@ class ProgressReporter: """ def __init__( - self, - update: Progress, - fetch: Progress, - upgrade: Progress, - callback: Optional[Callable[[float], None]] = None + self, + update: Progress, + fetch: Progress, + upgrade: Progress, + callback: Optional[Callable[[float], None]] = None, ): saved_stdout = os.dup(sys.stdout.fileno()) saved_stderr = os.dup(sys.stderr.fileno()) - self.stdout = io.TextIOWrapper(os.fdopen(saved_stdout, 'wb')) - self.stderr = io.TextIOWrapper(os.fdopen(saved_stderr, 'wb')) + self.stdout = io.TextIOWrapper(os.fdopen(saved_stdout, "wb")) + self.stderr = io.TextIOWrapper(os.fdopen(saved_stderr, "wb")) self.last_percent = 0.0 if callback is None: - self.callback = lambda p: \ - print(f"{p:.2f}", flush=True, file=self.stderr) + self.callback = lambda p: print( + f"{p:.2f}", flush=True, file=self.stderr + ) else: self.callback = callback @@ -107,7 +113,8 @@ def __init__( update.init(0, update_end, self.callback, self.stdout, self.stderr) fetch.init( - update_end, fetch_end, self.callback, self.stdout, self.stderr) + update_end, fetch_end, self.callback, self.stdout, self.stderr + ) upgrade.init(fetch_end, 100, self.callback, self.stdout, self.stderr) self.update_progress = update diff --git a/vmupdate/agent/source/dnf/__init__.py b/vmupdate/agent/source/dnf/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/dnf/__init__.py +++ b/vmupdate/agent/source/dnf/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/dnf/dnf5_api.py b/vmupdate/agent/source/dnf/dnf5_api.py index d04040e..38c169f 100644 --- a/vmupdate/agent/source/dnf/dnf5_api.py +++ b/vmupdate/agent/source/dnf/dnf5_api.py @@ -25,11 +25,12 @@ import libdnf5 from libdnf5.repo import DownloadCallbacks from libdnf5.rpm import TransactionCallbacks -from libdnf5.base import Base, Goal +from libdnf5.base import Goal from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress +from source.common.package_manager import AgentType from .dnf_cli import DNFCLI @@ -38,16 +39,36 @@ class TransactionError(RuntimeError): pass -class DNF(DNFCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) - self.base = Base() +class DNF5(DNFCLI): + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + + self.base = libdnf5.base.Base() + conf = self.base.get_config() + + if self.type == AgentType.UPDATE_VM: + conf.config_file_path = ( + self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" + ) + conf.best = True + conf.plugins = False + conf.installroot = self.UPDATE_VM_INSTALLROOT + for opt in ("cachedir", "logdir", "persistdir"): + setattr( + conf, opt, self.UPDATE_VM_INSTALLROOT + getattr(conf, opt) + ) + conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] + conf.excludepkgs = ["qubes-template-*"] self.base.load_config() + + # Create base object with the loaded config self.base.setup() self.config = self.base.get_config() update = FetchProgress(weight=0, log=self.log) # % of total time - fetch = FetchProgress(weight=50, log=self.log) # % of total time - upgrade = UpgradeProgress(weight=50, log=self.log) # % of total time + fetch = FetchProgress(weight=55, log=self.log) # % of total time + upgrade = UpgradeProgress(weight=45, log=self.log) # % of total time self.progress = ProgressReporter(update, fetch, upgrade) def refresh(self, hard_fail: bool) -> ProcessResult: @@ -64,14 +85,10 @@ def refresh(self, hard_fail: bool) -> ProcessResult: self.log.debug("Refreshing available packages...") result += self.expire_cache() - - repo_sack = self.base.get_repo_sack() - repo_sack.create_repos_from_system_configuration() - repo_sack.load_repos() - self.log.debug("Cache refresh successful.") except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) return result @@ -84,7 +101,13 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: result = ProcessResult() try: self.log.debug("Performing package upgrade...") + repo_sack = self.base.get_repo_sack() + repo_sack.create_repos_from_system_configuration() + repo_sack.load_repos() + goal = Goal(self.base) + if self.type == AgentType.UPDATE_VM: + goal.set_allow_erasing(True) goal.add_upgrade("*") transaction = goal.resolve() # fill empty `Command line` column in dnf history @@ -93,36 +116,46 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: if transaction.get_transaction_packages_count() == 0: self.log.info("No packages to upgrade, quitting.") return ProcessResult( - EXIT.OK, out="", - err="\n".join(transaction.get_resolve_logs_as_strings())) - - self.base.set_download_callbacks( - libdnf5.repo.DownloadCallbacksUniquePtr( - self.progress.fetch_progress)) + EXIT.OK_NO_UPDATES, + out="", + err="\n".join(transaction.get_resolve_logs_as_strings()), + ) + + if self.type != AgentType.DOM0: + # + self.base.set_download_callbacks( + libdnf5.repo.DownloadCallbacksUniquePtr( + self.progress.fetch_progress + ) + ) transaction.download() if not transaction.check_gpg_signatures(): problems = transaction.get_gpg_signature_problems() raise TransactionError( - f"GPG signatures check failed: {problems}") + f"GPG signatures check failed: {problems}" + ) - if result.code == EXIT.OK: - print("Updating packages.", flush=True) + if result.code == EXIT.OK and self.type is not AgentType.UPDATE_VM: self.log.debug("Committing upgrade...") transaction.set_callbacks( libdnf5.rpm.TransactionCallbacksUniquePtr( - self.progress.upgrade_progress)) + self.progress.upgrade_progress + ) + ) tnx_result = transaction.run() if tnx_result != transaction.TransactionRunResult_SUCCESS: raise TransactionError( - transaction.transaction_result_to_string(tnx_result)) + transaction.transaction_result_to_string(tnx_result) + ) self.log.debug("Package upgrade successful.") - self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) - print("Updated", flush=True) + if self.type is AgentType.VM: + self.log.info("Notifying dom0 about installed applications") + subprocess.call(["/etc/qubes-rpc/qubes.PostInstall"]) except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) return result @@ -139,7 +172,7 @@ def __init__(self, weight: int, log): self.fetching_notified = False def add_new_download( - self, _user_data, description: str, total_to_download: float + self, _user_data, description: str, total_to_download: float ) -> int: """ Notify the client that a new download has been created. @@ -158,7 +191,7 @@ def add_new_download( return self.count def progress( - self, user_cb_data: int, total_to_download: float, downloaded: float + self, user_cb_data: int, total_to_download: float, downloaded: float ) -> int: """ Download progress callback. @@ -168,22 +201,28 @@ def progress( :param downloaded: Number of bytes downloaded. """ if not self.fetching_notified: - print(f"Fetching {self.count} packages " - f"[{self._format_bytes(self.bytes_to_fetch)}]", - flush=True) + print( + f"Fetching {self.count} packages " + f"[{self._format_bytes(self.bytes_to_fetch)}]", + flush=True, + ) self.fetching_notified = True self.bytes_fetched += downloaded - self.package_bytes[user_cb_data] if downloaded > self.package_bytes[user_cb_data]: if self.package_bytes[user_cb_data] == 0: - print(f"Fetching {self.package_names[user_cb_data]} [{self._format_bytes(total_to_download)}]", - flush=True) + print( + f"Fetching {self.package_names[user_cb_data]} " + f"[{self._format_bytes(total_to_download)}]", + flush=True, + ) self.package_bytes[user_cb_data] = downloaded percent = self.bytes_fetched / self.bytes_to_fetch * 100 self.notify_callback(percent) # Should return 0 on success, # in case anything in dnf5 changed we return their default value return DownloadCallbacks.progress( - self, user_cb_data, total_to_download, downloaded) + self, user_cb_data, total_to_download, downloaded + ) def end(self, user_cb_data: int, status: int, msg: str) -> int: """ @@ -194,11 +233,14 @@ def end(self, user_cb_data: int, status: int, msg: str) -> int: :param msg: The error message in case of error. """ if status != 0: - print(msg, flush=True, file=self._stdout) + if isinstance(msg, bytes): + msg = msg.decode("ascii", errors="ignore") + if msg: + print(msg, flush=True, file=self._stdout) return DownloadCallbacks.end(self, user_cb_data, status, msg) def mirror_failure( - self, user_cb_data: int, msg: str, url: str, metadata: str + self, user_cb_data: int, msg: str, url: str, metadata: str ) -> int: """ Mirror failure callback. @@ -208,11 +250,17 @@ def mirror_failure( :param url: Failed mirror URL. :param metadata: the type of metadata that is being downloaded """ - print(f"Fetching {metadata} failure " - f"({self.package_names[user_cb_data]}) {msg}", - flush=True, file=self._stdout) + if isinstance(msg, bytes): + msg = msg.decode("ascii", errors="ignore") + print( + f"Fetching {metadata} failure " + f"({self.package_names[user_cb_data]}) {msg}", + flush=True, + file=self._stdout, + ) return DownloadCallbacks.mirror_failure( - self, user_cb_data, msg, url, metadata) + self, user_cb_data, msg, url, metadata + ) class UpgradeProgress(TransactionCallbacks, Progress): @@ -224,12 +272,13 @@ def __init__(self, weight: int, log): self.processed_packages = set() def install_progress( - self, item: libdnf5.base.TransactionPackage, amount: int, total: int + self, item: libdnf5.base.TransactionPackage, amount: int, total: int ): r""" Report the package installation progress periodically. - :param item: The TransactionPackage class instance for the package currently being installed + :param item: The TransactionPackage class instance for the package + currently being installed :param amount: The portion of the package already installed :param total: The disk space used by the package after installation """ @@ -251,7 +300,7 @@ def transaction_start(self, total: int): self.pgks = total def uninstall_progress( - self, item: libdnf5.base.TransactionPackage, amount: int, total: int + self, item: libdnf5.base.TransactionPackage, amount: int, total: int ): """ Report the package removal progress periodically. @@ -268,27 +317,39 @@ def uninstall_progress( percent = (self.pgks_done + pkg_progress) / self.pgks * 100 self.notify_callback(percent) + # pylint: disable=unused-argument def elem_progress(self, item, amount: int, total: int): r""" The installation/removal process for the item has started - :param item: The TransactionPackage class instance for the package currently being (un)installed - :param amount: Index of the package currently being processed. Items are indexed starting from 0. + :param item: The TransactionPackage class instance for the package + currently being (un)installed + :param amount: Index of the package currently being processed. + Items are indexed starting from 0. :param total: The total number of packages in the transaction """ self.pgks_done = amount percent = amount / total * 100 self.notify_callback(percent) - def script_start(self, item: libdnf5.base.TransactionPackage, nevra, type: int): + # pylint: disable=unused-argument,redefined-builtin + def script_start( + self, item: libdnf5.base.TransactionPackage, nevra, type: int + ): r""" Execution of the rpm scriptlet has started - :param item: The TransactionPackage class instance for the package that owns the executed or triggered - scriptlet. It can be `nullptr` if the scriptlet owner is not part of the transaction - (e.g., a package installation triggered an update of the man database, owned by man-db package). - :param nevra: Nevra of the package that owns the executed or triggered scriptlet. + :param item: The TransactionPackage class instance for the package that + owns the executed or triggered scriptlet. It can be + `nullptr` if the scriptlet owner is not part of + the transaction (e.g., a package installation triggered + an update of the man database, owned by man-db package). + :param nevra: Nevra of the package that owns the executed or triggered + scriptlet. :param type: Type of the scriptlet """ - print(f"Running rpm scriptlet for {nevra.get_name()}-{nevra.get_epoch()}:{nevra.get_version()}" - f"-{nevra.get_release()}.{nevra.get_arch()}", flush=True) + print( + f"Running rpm scriptlet for {nevra.get_name()}-{nevra.get_epoch()}" + f":{nevra.get_version()}-{nevra.get_release()}.{nevra.get_arch()}", + flush=True, + ) diff --git a/vmupdate/agent/source/dnf/dnf_api.py b/vmupdate/agent/source/dnf/dnf_api.py index 506090b..adc9c23 100644 --- a/vmupdate/agent/source/dnf/dnf_api.py +++ b/vmupdate/agent/source/dnf/dnf_api.py @@ -19,8 +19,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. +import os import subprocess import dnf +import dnf.conf +import dnf.rpm from dnf.yum.rpmtrans import TransactionDisplay from dnf.callback import DownloadProgress import dnf.transaction @@ -28,18 +31,55 @@ from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT from source.common.progress_reporter import ProgressReporter, Progress +from source.common.package_manager import AgentType from .dnf_cli import DNFCLI class DNF(DNFCLI): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) - self.base = dnf.Base() - self.base.conf.read() # load dnf.conf - update = FetchProgress(weight=0, log=self.log) # % of total time - fetch = FetchProgress(weight=50, log=self.log) # % of total time - upgrade = UpgradeProgress(weight=50, log=self.log) # % of total time + PROGRESS_REPORTING = True + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + + if self.type == AgentType.UPDATE_VM: + dnfconf = self.UPDATE_VM_INSTALLROOT + "/etc/dnf/dnf.conf" + else: + dnfconf = None + conf = dnf.conf.Conf() + conf.read(filename=dnfconf) + + if self.type == AgentType.UPDATE_VM: + conf.best = True + conf.plugins = False + conf.installroot = self.UPDATE_VM_INSTALLROOT + for opt in ("cachedir", "logdir", "persistdir"): + conf.prepend_installroot(opt) + conf.reposdir = [self.UPDATE_VM_INSTALLROOT + "/etc/yum.repos.d"] + conf.excludepkgs = ["qubes-template-*"] + + # make sure log file exists + log_dir = self.UPDATE_VM_INSTALLROOT + "/var/log" + log_file = os.path.join(log_dir, "hawkey.log") + os.makedirs(log_dir, exist_ok=True) + if not os.path.exists(log_file): + with open(log_file, "w"): + pass + + # Passing `conf` to `base` causes `releasever` not to be set + subst = conf.substitutions + if "releasever" not in subst: + subst["releasever"] = dnf.rpm.detect_releasever(conf.installroot) + + self.base = dnf.Base(conf) + # Repositories serve as sources of information about packages. + self.base.read_all_repos() + + update = FetchProgress( + weight=10, log=self.log, refresh=True + ) # % of total time + fetch = FetchProgress(weight=45, log=self.log) # % of total time + upgrade = UpgradeProgress(weight=45, log=self.log) # % of total time self.progress = ProgressReporter(update, fetch, upgrade) def refresh(self, hard_fail: bool) -> ProcessResult: @@ -49,16 +89,18 @@ def refresh(self, hard_fail: bool) -> ProcessResult: :param hard_fail: raise error if some repo is unavailable :return: (exit_code, stdout, stderr) """ - self.base.conf.skip_if_unavailable = int(not hard_fail) - result = ProcessResult() + self.base.conf.skip_if_unavailable = True try: self.log.debug("Refreshing available packages...") - # Repositories serve as sources of information about packages. - self.base.read_all_repos() + repos = tuple(self.base.repos.iter_enabled()) + # we do not know the size of the repositories + self.progress.update_progress.start(len(repos), len(repos)) + for repo in repos: + self.progress.update_progress.progress(repo.id, 1) + repo.load() + self.progress.update_progress.end(repo.id, 0, "") updated = self.base.update_cache() - # A sack is needed for querying. - self.base.fill_sack() if updated: self.log.debug("Cache refresh successful.") else: @@ -66,9 +108,9 @@ def refresh(self, hard_fail: bool) -> ProcessResult: result += ProcessResult(EXIT.ERR_VM_REFRESH) except Exception as exc: self.log.error( - "An error occurred while refreshing packages: %s", str(exc)) + "An error occurred while refreshing packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_REFRESH, out="", err=str(exc)) - return result def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: @@ -80,34 +122,38 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult: result = ProcessResult() try: self.log.debug("Performing package upgrade...") + # A sack is needed for querying. + self.base.fill_sack() + self.base.upgrade_all() # fill empty `Command line` column in dnf history self.base.cmds = ["qubes-vm-update"] - self.base.resolve() + self.base.resolve(allow_erasing=self.type == AgentType.UPDATE_VM) trans = self.base.transaction if not trans: self.log.info("No packages to upgrade, quitting.") - return ProcessResult(EXIT.OK, out="", err="") + return ProcessResult(EXIT.OK_NO_UPDATES, out="", err="") self.base.download_packages( - trans.install_set, - progress=self.progress.fetch_progress + trans.install_set, progress=self.progress.fetch_progress ) result += sign_check(self.base, trans.install_set, self.log) - if result.code == EXIT.OK: + if result.code == EXIT.OK and self.type is not AgentType.UPDATE_VM: print("Updating packages.", flush=True) self.log.debug("Committing upgrade...") self.base.do_transaction(self.progress.upgrade_progress) self.log.debug("Package upgrade successful.") - self.log.info("Notifying dom0 about installed applications") - subprocess.call(['/etc/qubes-rpc/qubes.PostInstall']) + if self.type is AgentType.VM: + self.log.info("Notifying dom0 about installed applications") + subprocess.call(["/etc/qubes-rpc/qubes.PostInstall"]) print("Updated", flush=True) except Exception as exc: self.log.error( - "An error occurred while upgrading packages: %s", str(exc)) + "An error occurred while upgrading packages: %s", str(exc) + ) result += ProcessResult(EXIT.ERR_VM_UPDATE, out="", err=str(exc)) finally: self.base.close() @@ -143,10 +189,11 @@ def sign_check(base, packages, log) -> ProcessResult: class FetchProgress(DownloadProgress, Progress): - def __init__(self, weight: int, log): + def __init__(self, weight: int, log, refresh: bool = False): Progress.__init__(self, weight, log) self.bytes_to_fetch = None self.bytes_fetched = 0 + self.action = "refresh" if refresh else "fetch" self.package_bytes = {} def end(self, payload, status, msg): @@ -155,12 +202,19 @@ def end(self, payload, status, msg): :api, `status` is a constant denoting the type of outcome, `err_msg` is an error message in case the outcome was an error. """ - print(f"{payload}: Fetched", flush=True) + if status != 0: + if isinstance(msg, bytes): + msg = msg.decode("ascii", errors="ignore") + if msg: + print(msg, flush=True, file=self._stdout) + else: + print(f"{payload}: {self.action.capitalize()}ed", flush=True) def message(self, msg): if isinstance(msg, bytes): - msg = msg.decode('ascii', errors='ignore') - print(msg, flush=True, file=self._stdout) + msg = msg.decode("ascii", errors="ignore") + if msg: + print(msg, flush=True, file=self._stdout) def progress(self, payload, done): """Update the progress display. :api @@ -181,12 +235,17 @@ def start(self, total_files, total_size, total_drpms=0): `total_size` total size of all files. """ - self.log.info("Fetch started.") + self.log.info(f"{self.action.capitalize()} started.") self.bytes_to_fetch = total_size - print(f"Fetching {total_files} packages " - f"[{self._format_bytes(self.bytes_to_fetch)}]", - flush=True) - self.package_bytes = {} + if self.action == "refresh": + print("Refreshing available packages.", flush=True) + else: + print( + f"Fetching {total_files} packages " + f"[{self._format_bytes(self.bytes_to_fetch)}]", + flush=True, + ) + self.package_bytes = {} self.notify_callback(0) @@ -195,8 +254,7 @@ def __init__(self, weight: int, log): TransactionDisplay.__init__(self) Progress.__init__(self, weight, log) - def progress(self, _package, action, ti_done, ti_total, ts_done, - ts_total): + def progress(self, _package, action, ti_done, ti_total, ts_done, ts_total): """ Report ongoing progress on a transaction item. @@ -207,11 +265,12 @@ def progress(self, _package, action, ti_done, ti_total, ts_done, :param ts_done: number of actions processed in the whole transaction :param ts_total: total number of actions in the whole transaction """ + self.log.info(_package) fetch = 6 install = 7 if action not in (fetch, install): return - percent = ti_done / ti_total * ts_done / ts_total * 100 + percent = (ti_done / ti_total + ts_done - 1) / ts_total * 100 self.notify_callback(percent) def scriptout(self, msgs): @@ -220,7 +279,7 @@ def scriptout(self, msgs): """ if msgs: if isinstance(msgs, bytes): - msgs = msgs.decode('ascii', errors='ignore') + msgs = msgs.decode("ascii", errors="ignore") print(msgs, flush=True) def filelog(self, package, action): @@ -231,6 +290,9 @@ def error(self, message): Write an error message to the fake stderr. """ if isinstance(message, bytes): - message = message.decode('ascii', errors='ignore') - print("Error during installation :" + message, - flush=True, file=self._stderr) + message = message.decode("ascii", errors="ignore") + print( + "Error during installation :" + message, + flush=True, + file=self._stderr, + ) diff --git a/vmupdate/agent/source/dnf/dnf_cli.py b/vmupdate/agent/source/dnf/dnf_cli.py index 20cb0bc..afa8d1d 100644 --- a/vmupdate/agent/source/dnf/dnf_cli.py +++ b/vmupdate/agent/source/dnf/dnf_cli.py @@ -22,21 +22,24 @@ import shutil from typing import List -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class DNFCLI(PackageManager): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) - pck_mng_path = shutil.which('dnf') + PROGRESS_REPORTING = False + UPDATE_VM_INSTALLROOT = "/var/lib/qubes/dom0-updates" + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + pck_mng_path = shutil.which("dnf") if pck_mng_path is not None: - pck_mngr = 'dnf' + pck_mngr = "dnf" else: - pck_mng_path = shutil.which('yum') + pck_mng_path = shutil.which("yum") if pck_mng_path is not None: - pck_mngr = 'yum' + pck_mngr = "yum" else: raise RuntimeError("Package manager not found!") self.package_manager: str = pck_mngr @@ -50,17 +53,23 @@ def refresh(self, hard_fail: bool) -> ProcessResult: """ result = self.expire_cache() - cmd = [self.package_manager, - "-q", - "check-update", - "--assumeyes", - f"--setopt=skip_if_unavailable={int(not hard_fail)}"] - result_check = self.run_cmd(cmd) - # ret_code == 100 is not an error - # It means there are packages to be updated - result_check.code = result_check.code if result_check.code != 100 else 0 - result += result_check - result.error_from_messages() + cmd = [ + self.package_manager, + "-q", + "check-update", + "--assumeyes", + f"--setopt=skip_if_unavailable={int(not hard_fail)}", + ] + if self.type != AgentType.UPDATE_VM: + # In UpdateVM we use preconfigured repos + result_check = self.run_cmd(cmd) + # ret_code == 100 is not an error + # It means there are packages to be updated + result_check.code = ( + result_check.code if result_check.code != 100 else 0 + ) + result += result_check + result.error_from_messages() return result @@ -68,11 +77,12 @@ def expire_cache(self) -> ProcessResult: """ Use package manager to expire cache. """ - cmd = [self.package_manager, - "-q", - "clean", - "expire-cache"] - result = self.run_cmd(cmd) + cmd = [self.package_manager, "-q", "clean", "expire-cache"] + if self.type != AgentType.UPDATE_VM: + result = self.run_cmd(cmd) + else: + # In UpdateVM we use preconfigured repos + result = ProcessResult() return result def get_packages(self): @@ -102,10 +112,35 @@ def get_action(self, remove_obsolete) -> List[str]: """ Disable or enforce obsolete flag in dnf/yum. """ + result = ["-y"] + if self.type is AgentType.UPDATE_VM: + result.extend( + [ + "upgrade", + "--noplugins", + "--best", + "--allowerasing", + "--downloadonly", + "--installroot", + self.UPDATE_VM_INSTALLROOT, + f"--setopt=cachedir={self.UPDATE_VM_INSTALLROOT}/var/cache/dnf", + f"--config={self.UPDATE_VM_INSTALLROOT}/etc/dnf/dnf.conf", + f"--setopt=reposdir={self.UPDATE_VM_INSTALLROOT}/etc/yum.repos.d", + "--exclude=qubes-template-*", + "-y", + ] + ) + return result if remove_obsolete: - return ["-y", "--setopt=obsoletes=1", "upgrade"] - return ["-y", "--setopt=obsoletes=0", - "upgrade" if self.package_manager == "dnf" else "update"] + result.extend(["--setopt=obsoletes=1", "upgrade"]) + else: + result.append("--setopt=obsoletes=0") + if self.package_manager == "dnf": + result.append("upgrade") + else: + # yum + result.append("update") + return result def clean(self) -> int: """ diff --git a/vmupdate/agent/source/log_config.py b/vmupdate/agent/source/log_config.py index 9a4c2be..dcac508 100644 --- a/vmupdate/agent/source/log_config.py +++ b/vmupdate/agent/source/log_config.py @@ -24,18 +24,18 @@ import grp from pathlib import Path -LOGPATH = '/var/log/qubes/qubes-update' -FORMAT_LOG = '%(asctime)s [Agent] %(message)s' -LOG_FILE = 'update-agent.log' +LOGPATH = "/var/log/qubes/qubes-update" +FORMAT_LOG = "%(asctime)s [Agent] %(message)s" +LOG_FILE = "update-agent.log" def init_logs( - directory=LOGPATH, - file=LOG_FILE, - format_=FORMAT_LOG, - level="INFO", - truncate_file=False, - qname=None, + directory=LOGPATH, + file=LOG_FILE, + format_=FORMAT_LOG, + level="INFO", + truncate_file=False, + qname=None, ): Path(directory).mkdir(parents=True, exist_ok=True) log_path = os.path.join(directory, file) @@ -46,14 +46,14 @@ def init_logs( # Persistent logs are at dom0. pass - log_handler = logging.FileHandler(log_path, encoding='utf-8') + log_handler = logging.FileHandler(log_path, encoding="utf-8") log_formatter = logging.Formatter(format_) log_handler.setFormatter(log_formatter) if qname is not None: log = logging.getLogger(qname) else: - log = logging.getLogger('vm-update.agent.PackageManager') + log = logging.getLogger("vm-update.agent.PackageManager") log.addHandler(log_handler) log.propagate = False try: diff --git a/vmupdate/agent/source/pacman/__init__.py b/vmupdate/agent/source/pacman/__init__.py index c8d3ba8..a2d6cf9 100644 --- a/vmupdate/agent/source/pacman/__init__.py +++ b/vmupdate/agent/source/pacman/__init__.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. \ No newline at end of file +# USA. diff --git a/vmupdate/agent/source/pacman/pacman_cli.py b/vmupdate/agent/source/pacman/pacman_cli.py index ad218d5..590ae0e 100644 --- a/vmupdate/agent/source/pacman/pacman_cli.py +++ b/vmupdate/agent/source/pacman/pacman_cli.py @@ -21,16 +21,21 @@ from typing import List, Dict -from source.common.package_manager import PackageManager +from source.common.package_manager import PackageManager, AgentType from source.common.process_result import ProcessResult from source.common.exit_codes import EXIT class PACMANCLI(PackageManager): - def __init__(self, log_handler, log_level): - super().__init__(log_handler, log_level) + PROGRESS_REPORTING = False + + def __init__(self, log_handler, log_level, agent_type: AgentType): + super().__init__(log_handler, log_level, agent_type) + if self.type is AgentType.UPDATE_VM: + raise NotImplementedError("Pacman do not support update proxy VM.") self.package_manager = "pacman" + # pylint: disable=unused-argument def refresh(self, hard_fail: bool) -> ProcessResult: """ Use package manager to refresh available packages. @@ -61,6 +66,7 @@ def get_packages(self) -> Dict[str, List[str]]: return packages + # pylint: disable=unused-argument def get_action(self, remove_obsolete) -> List[str]: """ Pacman will handle obsoletions itself diff --git a/vmupdate/agent/source/plugins/__init__.py b/vmupdate/agent/source/plugins/__init__.py index e254f34..e5e6399 100644 --- a/vmupdate/agent/source/plugins/__init__.py +++ b/vmupdate/agent/source/plugins/__init__.py @@ -1,12 +1,18 @@ import importlib import os.path import glob + modules = sorted(glob.glob(os.path.join(os.path.dirname(__file__), "*.py"))) -__all__ = [os.path.basename(f)[:-3] - for f in modules if os.path.isfile(f) - and not f.endswith('__init__.py')] -modules = [importlib.import_module("source.plugins." + name) - for name in __all__] -entrypoints = [getattr(module, name) - for name, module in zip(__all__, modules) - if callable(getattr(module, name))] +__all__ = [ + os.path.basename(f)[:-3] + for f in modules + if os.path.isfile(f) and not f.endswith("__init__.py") +] +modules = [ + importlib.import_module("source.plugins." + name) for name in __all__ +] +entrypoints = [ + getattr(module, name) + for name, module in zip(__all__, modules) + if callable(getattr(module, name)) +] diff --git a/vmupdate/agent/source/plugins/allow_release_info_change.py b/vmupdate/agent/source/plugins/allow_release_info_change.py index 9816746..6b5ec4c 100644 --- a/vmupdate/agent/source/plugins/allow_release_info_change.py +++ b/vmupdate/agent/source/plugins/allow_release_info_change.py @@ -22,6 +22,7 @@ APT_CONF = "/etc/apt/apt.conf.d/01qubes-update" +# pylint: disable=unused-argument def allow_release_info_change(os_data, log, **kwargs): """ Add apt conf file to disable `AllowReleaseInfoChange` for `buster`. @@ -32,5 +33,5 @@ def allow_release_info_change(os_data, log, **kwargs): if os_data.get("codename", "") == "buster": option = 'Acquire::AllowReleaseInfoChange "false"' log.info("Set %s as workaround for buster.", option) - with open(APT_CONF, "w") as file: - file.write(f'\n{option};\n') + with open(APT_CONF, "wt") as file: + file.write(f"\n{option};\n") diff --git a/vmupdate/agent/source/plugins/bookworm_backports.py b/vmupdate/agent/source/plugins/bookworm_backports.py index fdbe3bb..1abc353 100644 --- a/vmupdate/agent/source/plugins/bookworm_backports.py +++ b/vmupdate/agent/source/plugins/bookworm_backports.py @@ -87,6 +87,7 @@ def check_package_not_from_backports(package): return False +# pylint: disable=unused-argument def bookworm_backports(os_data, log, **kwargs): """ Update firmware and/or pipewire packages from backports repository. diff --git a/vmupdate/agent/source/plugins/disable_deltarpm.py b/vmupdate/agent/source/plugins/disable_deltarpm.py index 4214a21..6bb19af 100644 --- a/vmupdate/agent/source/plugins/disable_deltarpm.py +++ b/vmupdate/agent/source/plugins/disable_deltarpm.py @@ -20,6 +20,7 @@ # USA. +# pylint: disable=unused-argument def disable_deltarpm(os_data, log, dnf_conf="/etc/dnf/dnf.conf", **kwargs): """ Modify dnf.conf file to disable `deltarpm`. diff --git a/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py b/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py index 91254b4..37855bc 100644 --- a/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py +++ b/vmupdate/agent/source/plugins/fix_meminfo_writer_label.py @@ -3,6 +3,7 @@ import signal +# pylint: disable=unused-argument def fix_meminfo_writer_label(os_data, log, **kwargs): """ Fix meminfo-writer SELinux label to make memory ballooning work again @@ -42,7 +43,9 @@ def fix_meminfo_writer_label(os_data, log, **kwargs): if label_changed: try: - with open("/run/meminfo-writer.pid", "r", encoding="utf-8") as f: + with open( + "/run/meminfo-writer.pid", "r", encoding="utf-8" + ) as f: target_pid = int(f.read().strip()) os.kill(target_pid, signal.SIGUSR1) log.info( diff --git a/vmupdate/agent/source/plugins/manage_rpm_macro.py b/vmupdate/agent/source/plugins/manage_rpm_macro.py index dee42eb..d766acf 100644 --- a/vmupdate/agent/source/plugins/manage_rpm_macro.py +++ b/vmupdate/agent/source/plugins/manage_rpm_macro.py @@ -36,8 +36,9 @@ def manage_rpm_macro(os_data, log, **kwargs): if version < 33: log.info("Old fedora version detected.") with open(rpm_macro, "w") as file: - file.write("# CVE-2021-20271 mitigation\n" - "%_pkgverify_level all") + file.write( + "# CVE-2021-20271 mitigation\n" "%_pkgverify_level all" + ) else: if os.path.exists(rpm_macro): os.remove(rpm_macro) diff --git a/vmupdate/agent/source/plugins/pipewire_archlinux.py b/vmupdate/agent/source/plugins/pipewire_archlinux.py index ebf16d2..5bb29bd 100644 --- a/vmupdate/agent/source/plugins/pipewire_archlinux.py +++ b/vmupdate/agent/source/plugins/pipewire_archlinux.py @@ -19,15 +19,18 @@ import subprocess + +# pylint: disable=unused-argument def pipewire_archlinux(os_data, log, **kwargs): """Help with unattended switch from pulseaudio to pipewire-pulse""" # pacman proposes to remove pulseaudio when installing pipewire-pulse, # but the default answer is "n", so the update with --noconfirm fails # workaround it by removing pulseaudio before the update - if os_data["os_family"] != 'ArchLinux': + if os_data["os_family"] != "ArchLinux": return # check if pulseaudio is installed - p = subprocess.call(["pacman", "-Q", "pulseaudio"], + p = subprocess.call( + ["pacman", "-Q", "pulseaudio"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) @@ -36,12 +39,15 @@ def pipewire_archlinux(os_data, log, **kwargs): # ... and whether pipewire-pulse is going to be installed in the update # this will refresh metadata already, before starting progress reporting, # but well... - update_list = subprocess.check_output(["pacman", "-Syup"], - stderr=subprocess.DEVNULL).decode() + update_list = subprocess.check_output( + ["pacman", "-Syup"], stderr=subprocess.DEVNULL + ).decode() if not any("/pipewire-pulse-" in line for line in update_list.splitlines()): return # ... then remove pulseaudio beforehand (temporarily breaking the # dependencies) - log.info("Removing pulseaudio to allow update cleanly migrate to " - "pipewire-pulse") + log.info( + "Removing pulseaudio to allow update cleanly migrate to " + "pipewire-pulse" + ) subprocess.check_call(["pacman", "-Rdd", "--noconfirm", "pulseaudio"]) diff --git a/vmupdate/agent/source/plugins/updatesproxy_fix.py b/vmupdate/agent/source/plugins/updatesproxy_fix.py index 74f387a..cf49488 100644 --- a/vmupdate/agent/source/plugins/updatesproxy_fix.py +++ b/vmupdate/agent/source/plugins/updatesproxy_fix.py @@ -20,10 +20,10 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -import os import pathlib +# pylint: disable=unused-argument def updatesproxy_fix(os_data, log, **kwargs): """ Deploy #9025 fix diff --git a/vmupdate/agent/source/status.py b/vmupdate/agent/source/status.py index b302615..08d0c02 100644 --- a/vmupdate/agent/source/status.py +++ b/vmupdate/agent/source/status.py @@ -59,3 +59,13 @@ def updating(qube, percent: float): @staticmethod def done(qube, status: FinalStatus): return StatusInfo(qube, Status.DONE, info=status) + + +class FormatedLine: + def __init__(self, qube_name, stream: str, message: str): + self.qname = qube_name + self.stream = stream + self.message = message + + def __str__(self): + return f"{self.qname}:{self.stream}: {self.message}" diff --git a/vmupdate/agent/source/utils.py b/vmupdate/agent/source/utils.py index d486954..e64f3e8 100644 --- a/vmupdate/agent/source/utils.py +++ b/vmupdate/agent/source/utils.py @@ -38,9 +38,7 @@ def get_os_data(logger: Optional = None) -> Dict[str, Any]: data = {} os_release = _load_os_release( - "/etc/os-release", - "/usr/lib/os-release", - logger=logger + "/etc/os-release", "/usr/lib/os-release", logger=logger ) data["id"] = os_release.get("ID", "linux").strip() @@ -51,19 +49,25 @@ def get_os_data(logger: Optional = None) -> Dict[str, Any]: if "VERSION_CODENAME" in os_release: data["codename"] = os_release["VERSION_CODENAME"] - family = [os_release.get('ID', 'linux'), - *os_release.get('ID_LIKE', '').split()] + family = [ + os_release.get("ID", "linux"), + *os_release.get("ID_LIKE", "").split(), + ] - data["os_family"] = 'Unknown' + data["os_family"] = "Unknown" - if 'debian' in family: - data["os_family"] = 'Debian' + if "debian" in family: + data["os_family"] = "Debian" - if 'rhel' in family or 'fedora' in family: - data["os_family"] = 'RedHat' + if "rhel" in family or "fedora" in family: + data["os_family"] = "RedHat" - if 'arch' in family: - data["os_family"] = 'ArchLinux' + if "qubes" in family: + # We do not want to use 'RedHat' for dom0 since usually plugins do not apply to dom0 + data["os_family"] = "Qubes" + + if "arch" in family: + data["os_family"] = "ArchLinux" return data @@ -80,19 +84,23 @@ def _load_os_release(*os_release_files, logger: Optional): with open(filename) as file: for line_number, line in enumerate(file): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - match = re.match(r'([A-Z][A-Z_0-9]+)=(.*)', line) + match = re.match(r"([A-Z][A-Z_0-9]+)=(.*)", line) if match: key, val = match.groups() - if val and val[0] in '"\'': + if val and val[0] in "\"'": val = ast.literal_eval(val) result[key] = val else: if logger: - logger.error("%s:%i: error in parsing: %s", - filename, line_number, line) + logger.error( + "%s:%i: error in parsing: %s", + filename, + line_number, + line, + ) break except OSError as exc: if logger: diff --git a/vmupdate/qube_connection.py b/vmupdate/qube_connection.py index 9720130..543974b 100644 --- a/vmupdate/qube_connection.py +++ b/vmupdate/qube_connection.py @@ -21,6 +21,7 @@ import os import shutil import signal +import subprocess import tempfile import concurrent.futures from os.path import join @@ -30,7 +31,7 @@ import qubesadmin from vmupdate.agent.source.args import AgentArgs from vmupdate.agent.source.log_config import LOGPATH, LOG_FILE -from vmupdate.agent.source.status import StatusInfo, FinalStatus +from vmupdate.agent.source.status import StatusInfo, FinalStatus, FormatedLine from vmupdate.agent.source.common.process_result import ProcessResult @@ -48,13 +49,7 @@ class QubeConnection: PYTHON_PATH = "/usr/bin/python3" def __init__( - self, - qube, - dest_dir, - cleanup, - logger, - show_progress, - status_notifier + self, qube, dest_dir, cleanup, logger, show_progress, status_notifier ): self.qube = qube self.dest_dir = dest_dir @@ -83,16 +78,20 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.status_notifier.put(StatusInfo.done(self.qube, self.status)) if self.cleanup: - self.logger.info('Remove %s', self.dest_dir) + self.logger.info("Remove %s", self.dest_dir) try: self._run_shell_command_in_qube( - self.qube, ['rm', '-r', self.dest_dir]) + self.qube, ["rm", "-r", self.dest_dir] + ) except Exception as err: - self.logger.error('Cannot remove %s, because of error: %s', - self.dest_dir, str(err)) + self.logger.error( + "Cannot remove %s, because of error: %s", + self.dest_dir, + str(err), + ) if self.qube.is_running() and not self._initially_running: - self.logger.info('Shutdown %s', self.qube.name) + self.logger.info("Shutdown %s", self.qube.name) self.qube.shutdown() self.__connected = False @@ -112,11 +111,14 @@ def transfer_agent(self, src_dir: str) -> ProcessResult: base_dir = os.path.basename(src_dir.strip(os.sep)) src_arch = join(arch_dir, base_dir + arch_format) dest_arch = join(self.dest_dir, base_dir + arch_format) - shutil.make_archive(base_name=join(arch_dir, base_dir), - format='gztar', root_dir=root_dir, - base_dir=base_dir) + shutil.make_archive( + base_name=join(arch_dir, base_dir), + format="gztar", + root_dir=root_dir, + base_dir=base_dir, + ) - command = ['mkdir', '-p', self.dest_dir] + command = ["mkdir", "-p", self.dest_dir] result = self._run_shell_command_in_qube(self.qube, command) if result: return result @@ -134,12 +136,13 @@ def _copy_file_from_dom0(self, src, dest) -> ProcessResult: command = " ".join(write_dest) self.logger.debug("run command: %s < %s", command, src) try: - with open(src, 'rb') as file: + with open(src, "rb") as file: untrusted_stdout_and_stderr = self.qube.run( - command, user='root', input=file.read() + command, user="root", input=file.read() ) result = ProcessResult.from_untrusted_out_err( - *untrusted_stdout_and_stderr) + *untrusted_stdout_and_stderr + ) if result.code: raise OSError(f"Command returns code: {result.code}") except OSError as exc: @@ -148,7 +151,7 @@ def _copy_file_from_dom0(self, src, dest) -> ProcessResult: return result def run_entrypoint( - self, entrypoint_path: str, agent_args + self, entrypoint_path: str | List, agent_args ) -> ProcessResult: """ Run a script in the qube. @@ -157,15 +160,18 @@ def run_entrypoint( :param agent_args: args for agent entrypoint :return: return code and output of the script """ - # make sure entrypoint is executable - command = ['chmod', 'u+x', entrypoint_path] - result = self._run_shell_command_in_qube(self.qube, command) + if isinstance(entrypoint_path, str): + command = [ + QubeConnection.PYTHON_PATH, + entrypoint_path, + *AgentArgs.to_cli_args(agent_args), + ] + else: + command = entrypoint_path - # run entrypoint - command = [QubeConnection.PYTHON_PATH, entrypoint_path, - *AgentArgs.to_cli_args(agent_args)] - result += self._run_shell_command_in_qube( - self.qube, command, show=self.show_progress) + result = self._run_shell_command_in_qube( + self.qube, command, show=self.show_progress + ) return result @@ -173,31 +179,45 @@ def read_logs(self) -> ProcessResult: """ Read vm logs file. """ - command = ['cat', - str(join(LOGPATH, LOG_FILE))] + command = ["cat", str(join(LOGPATH, LOG_FILE))] result = self._run_shell_command_in_qube(self.qube, command) return result def _run_shell_command_in_qube( - self, target, command: List[str], show: bool = False + self, target, command: List[str], show: bool = False ) -> ProcessResult: - self.logger.debug("run command in %s: %s", - target.name, " ".join(command)) + self.logger.debug( + "run command in %s: %s", target.name, " ".join(command) + ) if not show: return self._run_command_and_wait_for_output(target, command) - else: - return self._run_command_and_actively_report_progress( - target, command) + return self._run_command_and_actively_report_progress(target, command) def _run_command_and_wait_for_output( - self, target, command: List[str] + self, target, command: List[str] ) -> ProcessResult: + self.logger.debug("Wait for output") + result = ProcessResult() try: - untrusted_stdout_and_stderr = target.run_with_args( - *command, user='root' + if self.qube.klass == "AdminVM": + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + untrusted_stdout_and_stderr = proc.communicate() + if proc.returncode: + raise CalledProcessError( + proc.returncode, + command, + output=untrusted_stdout_and_stderr[0] + + untrusted_stdout_and_stderr[1], + ) + else: + untrusted_stdout_and_stderr = target.run_with_args( + *command, user="root" + ) + result += ProcessResult.from_untrusted_out_err( + *untrusted_stdout_and_stderr ) - result = ProcessResult.from_untrusted_out_err( - *untrusted_stdout_and_stderr) except CalledProcessError as err: if err.returncode == 100: self.status = FinalStatus.NO_UPDATES @@ -206,20 +226,35 @@ def _run_command_and_wait_for_output( self.logger.error(str(err)) ret_code = err.returncode result = ProcessResult.from_untrusted_out_err( - err.output, err.output) + err.output, err.output + ) result.code = ret_code except Exception as err: result = ProcessResult(1, "", str(err)) return result def _run_command_and_actively_report_progress( - self, target, command: List[str] + self, target, command: List[str] ) -> ProcessResult: - proc = target.run_service( - 'qubes.VMExec+' + qubesadmin.utils.encode_for_vmexec(command), - user='root', - preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN) - ) + self.logger.debug("Progress reporting enabled.") + if self.qube.klass == "AdminVM": + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + elif "--download-only" in command: + # for download-only commands, run with fakeroot + proc = target.run_service( + "qubes.VMExec+" + + qubesadmin.utils.encode_for_vmexec(["fakeroot"] + command), + user="user", + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN), + ) + else: + proc = target.run_service( + "qubes.VMExec+" + qubesadmin.utils.encode_for_vmexec(command), + user="root", + preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN), + ) self.logger.debug("Fetching agent process stdout/stderr.") with concurrent.futures.ThreadPoolExecutor() as executor: @@ -228,7 +263,8 @@ def _run_command_and_actively_report_progress( future_out = executor.submit(self._collect_stdout, proc=proc) result = ProcessResult.from_untrusted_out_err( - future_out.result(), future_err.result()) + future_out.result(), future_err.result() + ) result.code = proc.wait() self.logger.debug("Agent process finished.") @@ -239,7 +275,7 @@ def _run_command_and_actively_report_progress( def _collect_stderr(self, proc) -> bytes: progress_finished = False - for untrusted_line in iter(proc.stderr.readline, b''): + for untrusted_line in iter(proc.stderr.readline, b""): if not untrusted_line: continue line = ProcessResult.sanitize_output(untrusted_line, single=True) @@ -249,33 +285,41 @@ def _collect_stderr(self, proc) -> bytes: try: progress = float(line) except ValueError: - self._print('err', line) - continue - - if progress == 100.: + try: + progress = float(line.split()[-1]) + except ValueError: + self.status_notifier.put( + FormatedLine(self.qube.name, "err", line) + ) + continue + + if progress == 100.0: progress_finished = True self.status_notifier.put( - StatusInfo.updating(self.qube, progress)) + StatusInfo.updating(self.qube, progress) + ) else: - self._print('err', line) + self.status_notifier.put( + FormatedLine(self.qube.name, "err", line) + ) proc.stderr.close() self.logger.debug("Agent stderr closed.") - return b'' + return b"" def _collect_stdout(self, proc) -> bytes: - for untrusted_line in iter(proc.stdout.readline, b''): + for untrusted_line in iter(proc.stdout.readline, b""): if untrusted_line: line = ProcessResult.sanitize_output( - untrusted_line, single=True) + untrusted_line, single=True + ) if line: - self._print('out', line) + self.status_notifier.put( + FormatedLine(self.qube.name, "out", line) + ) - proc.stderr.close() + proc.stdout.close() self.logger.debug("Agent stdout closed.") - return b'' - - def _print(self, stream: str, line: str): - self.status_notifier.put(f"{self.qube.name}:{stream}: {line}") + return b"" diff --git a/vmupdate/tests/conftest.py b/vmupdate/tests/conftest.py index 3bae319..f6401fb 100644 --- a/vmupdate/tests/conftest.py +++ b/vmupdate/tests/conftest.py @@ -44,7 +44,7 @@ def __init__(self, name, app, klass, template=None, **kwargs): self.app.domains[name] = self self.klass = klass self.running = True - if self.klass in ('AppVM', 'DispVM'): + if self.klass in ("AppVM", "DispVM"): template.derived_vms.append(self) self.derived_vms = [] self.auto_cleanup = False @@ -124,18 +124,18 @@ def test_qapp(): def test_agent(): def closure(results, unexpected): class UpdateAgentManager: - def __init__(self, app, qube, agent_args, show_progress): + def __init__(self, app, qube, agent_args, show_progress, dom0): self.qube = qube def run_agent(self, agent_args, status_notifier, termination): if self.qube.name not in results: status_notifier.put( - StatusInfo.done(self.qube, FinalStatus.UNKNOWN)) + StatusInfo.done(self.qube, FinalStatus.UNKNOWN) + ) unexpected.append(self.qube.name) return ProcessResult(code=99) for status in results[self.qube.name]["statuses"]: - status_notifier.put( - StatusInfo.done(self.qube, status)) + status_notifier.put(StatusInfo.done(self.qube, status)) result = ProcessResult(code=results[self.qube.name]["retcode"]) del results[self.qube.name] return result @@ -149,41 +149,75 @@ def generate_vm_variations(app, variations): """ Generate all possible variations of vms for the given list of features. """ - dom0 = TestVM("dom0", app, klass="AdminVM", updateable=True, running=True, - update_result=FinalStatus.UNKNOWN, - features=Features("dom0", app, {'updates-available': True})) + dom0 = TestVM( + "dom0", + app, + klass="AdminVM", + updateable=True, + running=True, + update_result=FinalStatus.UNKNOWN, + features=Features("dom0", app, {"updates-available": True}), + ) domains = { - "klass": {"TemplateVM": set(), "StandaloneVM": set(), "AppVM": set(), - "DispVM": set()}, + "klass": { + "TemplateVM": set(), + "StandaloneVM": set(), + "AppVM": set(), + "DispVM": set(), + }, "is_running": {False: set(), True: set()}, "servicevm": {False: set(), True: set()}, "auto_cleanup": {False: set(), True: set()}, "updatable": {True: set(), False: set()}, "updates_available": {False: set(), True: set()}, - "last_updates_check": {None: set(), '2020-01-01 00:00:00': set(), - '3020-01-01 00:00:00': set()}, + "last_updates_check": { + None: set(), + "2020-01-01 00:00:00": set(), + "3020-01-01 00:00:00": set(), + }, "qrexec": {False: set(), True: set()}, - "os": {'Linux': set(), 'BSD': set()}, - "updated": {FinalStatus.UNKNOWN: set(), FinalStatus.SUCCESS: set(), - FinalStatus.NO_UPDATES: set(), FinalStatus.ERROR: set(), - FinalStatus.CANCELLED: set(), - }, + "os": {"Linux": set(), "BSD": set()}, + "updated": { + FinalStatus.UNKNOWN: set(), + FinalStatus.SUCCESS: set(), + FinalStatus.NO_UPDATES: set(), + FinalStatus.ERROR: set(), + FinalStatus.CANCELLED: set(), + }, "has_template_updated": { - FinalStatus.SUCCESS: set(), FinalStatus.NO_UPDATES: set(), - FinalStatus.ERROR: set(), FinalStatus.CANCELLED: set(), - FinalStatus.UNKNOWN: set()}, + FinalStatus.SUCCESS: set(), + FinalStatus.NO_UPDATES: set(), + FinalStatus.ERROR: set(), + FinalStatus.CANCELLED: set(), + FinalStatus.UNKNOWN: set(), + }, } - klasses = list(reversed(sorted(list(domains['klass'].keys())))) + klasses = list(reversed(sorted(list(domains["klass"].keys())))) if "klass" not in variations: klasses = klasses[:1] - rest = [list(domains[key].keys()) - if key in variations else list(domains[key].keys())[:1] - for key in domains.keys() if key != "klass"] + rest = [ + ( + list(domains[key].keys()) + if key in variations + else list(domains[key].keys())[:1] + ) + for key in domains.keys() + if key != "klass" + ] for k in klasses: - for (running, servicevm, auto_cleanup, updatable, updates_available, - last_check, qrexec, os, updated, template_updated - ) in itertools.product(*rest): + for ( + running, + servicevm, + auto_cleanup, + updatable, + updates_available, + last_check, + qrexec, + os, + updated, + template_updated, + ) in itertools.product(*rest): if not updatable and (updates_available or last_check): # do not consider features about updates for non-updatable vms @@ -199,19 +233,34 @@ def generate_vm_variations(app, variations): # `template_updated` continue - lc_enc = {None: '0', '2020-01-01 00:00:00': '1', - '3020-01-01 00:00:00': '2'} - os_enc = {'Linux': '0', 'BSD': '1'} - f_map = {FinalStatus.SUCCESS: "0", FinalStatus.ERROR: "1", - FinalStatus.CANCELLED: "2", FinalStatus.NO_UPDATES: "3", - FinalStatus.UNKNOWN: "4"} + lc_enc = { + None: "0", + "2020-01-01 00:00:00": "1", + "3020-01-01 00:00:00": "2", + } + os_enc = {"Linux": "0", "BSD": "1"} + f_map = { + FinalStatus.SUCCESS: "0", + FinalStatus.ERROR: "1", + FinalStatus.CANCELLED: "2", + FinalStatus.NO_UPDATES: "3", + FinalStatus.UNKNOWN: "4", + } txt = lambda x: str(int(x)) - suffix = (txt(running) + txt(servicevm) + lc_enc[last_check] + - txt(updates_available) + txt(qrexec) + os_enc[os] + - txt(updatable) + txt(auto_cleanup)) - if k in ('DispVM', 'AppVM'): + suffix = ( + txt(running) + + txt(servicevm) + + lc_enc[last_check] + + txt(updates_available) + + txt(qrexec) + + os_enc[os] + + txt(updatable) + + txt(auto_cleanup) + ) + if k in ("DispVM", "AppVM"): template = app.domains[ - 'T' + f_map[template_updated] + "4" + suffix[:-1] + "0"] + "T" + f_map[template_updated] + "4" + suffix[:-1] + "0" + ] ext_suffix = f_map[updated] + f_map[template_updated] + suffix update_result = updated else: @@ -221,21 +270,27 @@ def generate_vm_variations(app, variations): features = {} if servicevm: - features['servicevm'] = True + features["servicevm"] = True if updates_available: - features['updates-available'] = True + features["updates-available"] = True if last_check: - features['last-updates-check'] = last_check + features["last-updates-check"] = last_check if qrexec: - features['qrexec'] = qrexec + features["qrexec"] = qrexec if os: - features['os'] = os + features["os"] = os vm = TestVM( - k[0] + ext_suffix, app, klass=k, updateable=updatable, - running=running, auto_cleanup=auto_cleanup, template=template, + k[0] + ext_suffix, + app, + klass=k, + updateable=updatable, + running=running, + auto_cleanup=auto_cleanup, + template=template, features=Features(k[0] + ext_suffix, app, features), - update_result=update_result) + update_result=update_result, + ) domains["klass"][k].add(vm) domains["is_running"][running].add(vm) @@ -246,7 +301,7 @@ def generate_vm_variations(app, variations): domains["last_updates_check"][last_check].add(vm) domains["qrexec"][qrexec].add(vm) domains["os"][os].add(vm) - if k in ('DispVM', 'AppVM'): + if k in ("DispVM", "AppVM"): domains["updated"][updated].add(vm) domains["has_template_updated"][template_updated].add(vm) else: @@ -255,10 +310,15 @@ def generate_vm_variations(app, variations): domains["klass"]["AdminVM"] = {dom0} dom_prop = { - "is_running": True, "servicevm": False, "auto_cleanup": False, - "updatable": True, "updates_available": True, - "last_updates_check": None, "updated": FinalStatus.UNKNOWN, - "has_template_updated": FinalStatus.UNKNOWN} + "is_running": True, + "servicevm": False, + "auto_cleanup": False, + "updatable": True, + "updates_available": True, + "last_updates_check": None, + "updated": FinalStatus.UNKNOWN, + "has_template_updated": FinalStatus.UNKNOWN, + } for key, subkey in dom_prop.items(): try: domains[key][subkey].add(dom0) diff --git a/vmupdate/tests/test_vmupdate.py b/vmupdate/tests/test_vmupdate.py index c6b43c1..7b0b4d5 100644 --- a/vmupdate/tests/test_vmupdate.py +++ b/vmupdate/tests/test_vmupdate.py @@ -32,37 +32,58 @@ from vmupdate import vmupdate -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") def test_no_options_do_nothing(_logger, _log_file, _chmod, _chown, test_qapp): test_qapp.domains = test_qapp.Domains() TestVM("dom0", test_qapp, klass="AdminVM") args = [] assert main(args, test_qapp) == EXIT.OK - args = ['--signal-no-updates'] + args = ["--signal-no-updates"] assert main(args, test_qapp) == EXIT.OK_NO_UPDATES -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +class Subproc: + def __init__(self, returncode=0): + self.returncode = returncode + + def wait(self): + pass + + +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("subprocess.Popen") def test_preselection( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, + dummy_subprocess, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool domains = generate_vm_variations( - test_qapp, ["klass", "updatable", "is_running"]) + test_qapp, ["klass", "updatable", "is_running"] + ) updatable = domains["updatable"][True] is_running = domains["is_running"][True] @@ -72,7 +93,9 @@ def test_preselection( app = domains["klass"]["AppVM"] disp = domains["klass"]["DispVM"] run_app = app & is_running - default = updatable & ((templ | stand) | (is_running & (disp | app))) + default = updatable & ( + (templ | stand | admin) | (is_running & (disp | app)) + ) AdminVM = next(iter(admin)) TemplVM = next(iter(templ)) @@ -85,41 +108,146 @@ def test_preselection( (): default, ("--skip", UpStandVM.name): default - {UpStandVM}, ("--all",): default, - ("--all", "--apps",): default, - ("--all", "--templates",): default, - ("--all", "--standalones",): default, - ("--all", "--skip", UpStandVM.name,): default - {UpStandVM}, - ("--all", "--targets", UpStandVM.name,): default, - ("--all", "--targets", NRunAppVM.name,): default | {NRunAppVM}, - ("--all", "--targets", NUpStandVM.name,): default | {NUpStandVM}, + ( + "--all", + "--apps", + ): default, + ( + "--all", + "--templates", + ): default, + ( + "--all", + "--standalones", + ): default, + ( + "--all", + "--skip", + UpStandVM.name, + ): default + - {UpStandVM}, + ( + "--all", + "--targets", + UpStandVM.name, + ): default, + ( + "--all", + "--targets", + NRunAppVM.name, + ): default + | {NRunAppVM}, + ( + "--all", + "--targets", + NUpStandVM.name, + ): default + | {NUpStandVM}, ("--apps",): app & is_running, ("--templates",): updatable & templ, ("--standalones",): updatable & stand, - ("--templates", "--apps",): (updatable & templ) | run_app, - ("--templates", "--standalones",): updatable & (templ | stand), - ("--templates", "--standalones", "--apps",): - (updatable & (templ | stand)) | (app & is_running), - ("--standalones", "--skip", StandVM.name,): - (updatable & stand) - {StandVM}, - ("--standalones", "--skip", TemplVM.name,): (updatable & stand), - ("--standalones", "--targets", UpStandVM.name,): (updatable & stand), - ("--standalones", "--targets", NUpStandVM.name,): - (updatable & stand) | {NUpStandVM}, - ("--standalones", "--targets", TemplVM.name,): - (updatable & stand) | {TemplVM}, - ("--apps", "--skip", NRunAppVM.name,): run_app, - ("--apps", "--skip", StandVM.name,): run_app, - ("--apps", "--targets", NRunAppVM.name,): run_app | {NRunAppVM}, - ("--apps", "--targets", TemplVM.name,): run_app | {TemplVM}, - ("--targets", NRunAppVM.name,): {NRunAppVM}, - ("--targets", StandVM.name,): {StandVM}, - # dom0 skipped, user warning - ("--targets", AdminVM.name,): EXIT.OK_NO_UPDATES, - ("--targets", "unknown",): EXIT.ERR_USAGE, - ("--targets", f"{TemplVM.name},{StandVM.name}",): {TemplVM, StandVM}, - ("--targets", f"{TemplVM.name},{TemplVM.name}",): EXIT.ERR_USAGE, - ("--targets", TemplVM.name, "--skip", TemplVM.name,): {}, - ("--targets", f"{TemplVM.name},{StandVM.name}", "--skip", TemplVM.name,): {StandVM}, + ( + "--templates", + "--apps", + ): (updatable & templ) + | run_app, + ( + "--templates", + "--standalones", + ): updatable + & (templ | stand), + ( + "--templates", + "--standalones", + "--apps", + ): (updatable & (templ | stand)) + | (app & is_running), + ( + "--standalones", + "--skip", + StandVM.name, + ): (updatable & stand) + - {StandVM}, + ( + "--standalones", + "--skip", + TemplVM.name, + ): (updatable & stand), + ( + "--standalones", + "--targets", + UpStandVM.name, + ): (updatable & stand), + ( + "--standalones", + "--targets", + NUpStandVM.name, + ): (updatable & stand) + | {NUpStandVM}, + ( + "--standalones", + "--targets", + TemplVM.name, + ): (updatable & stand) + | {TemplVM}, + ( + "--apps", + "--skip", + NRunAppVM.name, + ): run_app, + ( + "--apps", + "--skip", + StandVM.name, + ): run_app, + ( + "--apps", + "--targets", + NRunAppVM.name, + ): run_app + | {NRunAppVM}, + ( + "--apps", + "--targets", + TemplVM.name, + ): run_app + | {TemplVM}, + ( + "--targets", + NRunAppVM.name, + ): {NRunAppVM}, + ( + "--targets", + StandVM.name, + ): {StandVM}, + ( + "--targets", + AdminVM.name, + ): {AdminVM}, + ( + "--targets", + "unknown", + ): EXIT.ERR_USAGE, + ( + "--targets", + f"{TemplVM.name},{StandVM.name}", + ): {TemplVM, StandVM}, + ( + "--targets", + f"{TemplVM.name},{TemplVM.name}", + ): EXIT.ERR_USAGE, + ( + "--targets", + TemplVM.name, + "--skip", + TemplVM.name, + ): {}, + ( + "--targets", + f"{TemplVM.name},{StandVM.name}", + "--skip", + TemplVM.name, + ): {StandVM}, } failed = {} @@ -128,9 +256,10 @@ def test_preselection( feed = {} expected_exit = selected else: - feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], - 'retcode': EXIT.OK} - for vm in selected} + feed = { + vm.name: {"statuses": [FinalStatus.SUCCESS], "retcode": EXIT.OK} + for vm in selected + } if feed: expected_exit = EXIT.OK else: @@ -138,47 +267,67 @@ def test_preselection( unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) - retcode = main(("--force-update", "--just-print-progress", *args), test_qapp) + retcode = main( + ("--force-update", "--just-print-progress", *args), test_qapp + ) failed[args] = {} if retcode != expected_exit: failed[args]["unexpected exit code"] = retcode failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - failed[args] = {key: value - for key, value in failed[args].items() if value} + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("subprocess.Popen") def test_selection( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + dummy_subprocess, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool domains = generate_vm_variations( test_qapp, - ["klass", "updates_available", "last_updates_check", "qrexec", "os"]) + ["klass", "updates_available", "last_updates_check", "qrexec", "os"], + ) all = domains["updatable"][True] qlinux = domains["qrexec"][True] & domains["os"]["Linux"] to_update = domains["updates_available"][True] - stale = qlinux & (domains["updates_available"][False] & - (domains["last_updates_check"][None] | - domains["last_updates_check"]['2020-01-01 00:00:00'])) + stale = qlinux & ( + domains["updates_available"][False] + & ( + domains["last_updates_check"][None] + | domains["last_updates_check"]["2020-01-01 00:00:00"] + ) + ) expected = { ("--force-update",): all, @@ -195,14 +344,15 @@ def test_selection( if isinstance(selected, int): feed = {} expected_exit = selected - monkeypatch.setattr( - vmupdate, "preselect_targets", lambda *_: all) + monkeypatch.setattr(vmupdate, "preselect_targets", lambda *_: all) else: - feed = {vm.name: {'statuses': [FinalStatus.SUCCESS], - 'retcode': EXIT.OK} - for vm in selected} + feed = { + vm.name: {"statuses": [FinalStatus.SUCCESS], "retcode": EXIT.OK} + for vm in selected + } monkeypatch.setattr( - vmupdate, "preselect_targets", lambda *_: selected) + vmupdate, "preselect_targets", lambda *_: selected + ) if feed: expected_exit = EXIT.OK else: @@ -217,35 +367,56 @@ def test_selection( failed[args]["unexpected exit code"] = retcode failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - failed[args] = {key: value - for key, value in failed[args].items() if value} + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') -@patch('asyncio.run') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") +@patch("asyncio.run") +@patch("subprocess.Popen") def test_restarting( - arun, mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + dummy_subprocess, + arun, + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): + dummy_subprocess.return_value = Subproc() mp_manager.return_value = test_manager mp_pool.return_value = test_pool domains = generate_vm_variations( test_qapp, - ["klass", "is_running", "servicevm", "auto_cleanup", - "updated", "has_template_updated"]) + [ + "klass", + "is_running", + "servicevm", + "auto_cleanup", + "updated", + "has_template_updated", + ], + ) all = domains["updatable"][True] service = domains["servicevm"][True] @@ -258,33 +429,41 @@ def test_restarting( not_updated = all - domains["updated"][FinalStatus.SUCCESS] running = domains["is_running"][True] template_updated = domains["has_template_updated"][FinalStatus.SUCCESS] - applicable = (derived & not_updated & running & template_updated - ) - (auto_cleanup & disp) + applicable = (derived & not_updated & running & template_updated) - ( + auto_cleanup & disp + ) expected = { - (): {"halted": set(), - "restarted": set(), - "untouched": all}, + (): {"halted": set(), "restarted": set(), "untouched": all}, ("--no-apply",): { "halted": set(), "restarted": set(), - "untouched": all}, + "untouched": all, + }, ("--apply-to-sys",): { "halted": updated & running & templ, "restarted": applicable & service, - "untouched": all - (updated & running & templ) - (applicable & service)}, + "untouched": all + - (updated & running & templ) + - (applicable & service), + }, ("--apply-to-all",): { "halted": (updated & running & templ) | (applicable - service), "restarted": applicable & service, - "untouched": all - (updated & running & templ) - applicable}, + "untouched": all - (updated & running & templ) - applicable, + }, } failed = {} for args, selected in expected.items(): monkeypatch.setattr(vmupdate, "get_targets", lambda *_: all) - feed = {vm.name: {'statuses': [vm.update_result], - 'retcode': None} # we don't care - for vm in all} + feed = { + vm.name: { + "statuses": [vm.update_result], + "retcode": None, + } # we don't care + for vm in all + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) @@ -295,27 +474,35 @@ def test_restarting( failed[args]["unexpected vm"] = unexpected failed[args]["leftover feed"] = feed - halted = {vm for vm in all - if vm.shutdown.called and not vm.start.called} - restarted = {vm for vm in all - if vm.shutdown.called and vm.start.called} - untouched = {vm for vm in all - if not vm.shutdown.called and not vm.start.called} - failed[args]["unexpected restart"] = set(map( - lambda vm: vm.name, restarted - selected["restarted"])) - failed[args]["not restarted"] = set(map( - lambda vm: vm.name, selected["restarted"] - restarted)) - failed[args]["unexpected shutdown"] = set(map( - lambda vm: vm.name, halted - selected["halted"])) - failed[args]["not halted"] = set(map( - lambda vm: vm.name, selected["halted"] - halted)) - failed[args]["unexpected untouched"] = set(map( - lambda vm: vm.name, untouched - selected["untouched"])) - failed[args]["unexpected touched"] = set(map( - lambda vm: vm.name, selected["untouched"] - untouched)) - - failed[args] = {key: value - for key, value in failed[args].items() if value} + halted = { + vm for vm in all if vm.shutdown.called and not vm.start.called + } + restarted = {vm for vm in all if vm.shutdown.called and vm.start.called} + untouched = { + vm for vm in all if not vm.shutdown.called and not vm.start.called + } + failed[args]["unexpected restart"] = set( + map(lambda vm: vm.name, restarted - selected["restarted"]) + ) + failed[args]["not restarted"] = set( + map(lambda vm: vm.name, selected["restarted"] - restarted) + ) + failed[args]["unexpected shutdown"] = set( + map(lambda vm: vm.name, halted - selected["halted"]) + ) + failed[args]["not halted"] = set( + map(lambda vm: vm.name, selected["halted"] - halted) + ) + failed[args]["unexpected untouched"] = set( + map(lambda vm: vm.name, untouched - selected["untouched"]) + ) + failed[args]["unexpected touched"] = set( + map(lambda vm: vm.name, selected["untouched"] - untouched) + ) + + failed[args] = { + key: value for key, value in failed[args].items() if value + } fails = {args: failed[args] for args in failed if failed[args]} assert not fails @@ -325,60 +512,134 @@ def test_restarting( stat = FinalStatus -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") @pytest.mark.parametrize( "tmpl_status, tmpl_retcode, app_status, app_retcode, expected_retcode", -( - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: 2x OK"), - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: tmpl OK"), - pytest.param( - stat.NO_UPDATES, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.OK_NO_UPDATES, id="no updates: app OK"), - pytest.param( - stat.ERROR, EXIT.OK, stat.NO_UPDATES, EXIT.OK, - EXIT.ERR, id="error: tmpl"), - pytest.param( - stat.SUCCESS, EXIT.OK, stat.ERROR, EXIT.OK, - EXIT.ERR, id="error: app"), - pytest.param( - stat.SUCCESS, EXIT.OK_NO_UPDATES, stat.SUCCESS, EXIT.OK, - EXIT.ERR_VM_UNHANDLED, id="unhandled retcode"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM, stat.ERROR, EXIT.ERR_VM_PRE, - EXIT.ERR_VM_PRE, id="vm inside error"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM_UPDATE, stat.ERROR, EXIT.ERR_VM_REFRESH, - EXIT.ERR_VM_UPDATE, id="vm inside error 2"), - pytest.param( - stat.SUCCESS, EXIT.ERR_VM, stat.SUCCESS, EXIT.OK, - EXIT.ERR_VM, id="vm general inside error"), - pytest.param( - stat.CANCELLED, EXIT.OK, stat.SUCCESS, EXIT.OK, - EXIT.SIGINT, id="cancelled"), - pytest.param( - stat.CANCELLED, EXIT.OK, stat.ERROR, EXIT.ERR_VM_UNHANDLED, - EXIT.SIGINT, id="cancelled with error"), - pytest.param( - stat.UNKNOWN, EXIT.OK, stat.SUCCESS, EXIT.OK, - EXIT.ERR_QREXEX, id="communication error"), -)) + ( + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: 2x OK", + ), + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: tmpl OK", + ), + pytest.param( + stat.NO_UPDATES, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.OK_NO_UPDATES, + id="no updates: app OK", + ), + pytest.param( + stat.ERROR, + EXIT.OK, + stat.NO_UPDATES, + EXIT.OK, + EXIT.ERR, + id="error: tmpl", + ), + pytest.param( + stat.SUCCESS, + EXIT.OK, + stat.ERROR, + EXIT.OK, + EXIT.ERR, + id="error: app", + ), + pytest.param( + stat.SUCCESS, + EXIT.OK_NO_UPDATES, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_VM_UNHANDLED, + id="unhandled retcode", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM, + stat.ERROR, + EXIT.ERR_VM_PRE, + EXIT.ERR_VM_PRE, + id="vm inside error", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM_UPDATE, + stat.ERROR, + EXIT.ERR_VM_REFRESH, + EXIT.ERR_VM_UPDATE, + id="vm inside error 2", + ), + pytest.param( + stat.SUCCESS, + EXIT.ERR_VM, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_VM, + id="vm general inside error", + ), + pytest.param( + stat.CANCELLED, + EXIT.OK, + stat.SUCCESS, + EXIT.OK, + EXIT.SIGINT, + id="cancelled", + ), + pytest.param( + stat.CANCELLED, + EXIT.OK, + stat.ERROR, + EXIT.ERR_VM_UNHANDLED, + EXIT.SIGINT, + id="cancelled with error", + ), + pytest.param( + stat.UNKNOWN, + EXIT.OK, + stat.SUCCESS, + EXIT.OK, + EXIT.ERR_QREXEX, + id="communication error", + ), + ), +) def test_return_codes( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch, - tmpl_status, tmpl_retcode, app_status, app_retcode, expected_retcode + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, + tmpl_status, + tmpl_retcode, + app_status, + app_retcode, + expected_retcode, ): mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -388,32 +649,48 @@ def test_return_codes( appvm = TestVM("appvm", test_qapp, klass="AppVM", template=vm) feed = { - vm.name: {'statuses': [tmpl_status], 'retcode': tmpl_retcode}, - appvm.name: {'statuses': [app_status], 'retcode': app_retcode}} + vm.name: {"statuses": [tmpl_status], "retcode": tmpl_retcode}, + appvm.name: {"statuses": [app_status], "retcode": app_retcode}, + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) retcode = main( - ("--just-print-progress", "--all", "--force-update", - "--signal-no-updates"), test_qapp) + ( + "--just-print-progress", + "--all", + "--force-update", + "--signal-no-updates", + ), + test_qapp, + ) assert retcode == expected_retcode -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('vmupdate.update_manager.UpdateAgentManager') -@patch('multiprocessing.Pool') -@patch('multiprocessing.Manager') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("vmupdate.update_manager.UpdateAgentManager") +@patch("multiprocessing.Pool") +@patch("multiprocessing.Manager") def test_error( - mp_manager, mp_pool, agent_mng, - _logger, _log_file, _chmod, _chown, _print, - test_qapp, test_manager, test_pool, test_agent, - monkeypatch + mp_manager, + mp_pool, + agent_mng, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + test_manager, + test_pool, + test_agent, + monkeypatch, ): mp_manager.return_value = test_manager mp_pool.return_value = test_pool @@ -423,47 +700,66 @@ def test_error( appvm = TestVM("appvm", test_qapp, klass="AppVM", template=vm) feed = { - vm.name: {'statuses': [FinalStatus.ERROR], 'retcode': EXIT.OK}, - appvm.name: {'statuses': [FinalStatus.NO_UPDATES], 'retcode': EXIT.OK}} + vm.name: {"statuses": [FinalStatus.ERROR], "retcode": EXIT.OK}, + appvm.name: {"statuses": [FinalStatus.NO_UPDATES], "retcode": EXIT.OK}, + } unexpected = [] agent_mng.side_effect = test_agent(feed, unexpected) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) - retcode = main(( - "--just-print-progress", "--all", "--force-update"), test_qapp) + retcode = main( + ("--just-print-progress", "--all", "--force-update"), test_qapp + ) assert retcode == EXIT.ERR -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') -@patch('asyncio.run') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") +@patch("asyncio.run") @pytest.mark.parametrize( "action, code", -( - pytest.param("template shutdown", EXIT.ERR_SHUTDOWN_TMPL), - pytest.param("app shutdown", EXIT.ERR_SHUTDOWN_APP), - pytest.param("app start", EXIT.ERR_START_APP), -)) + ( + pytest.param("template shutdown", EXIT.ERR_SHUTDOWN_TMPL), + pytest.param("app shutdown", EXIT.ERR_SHUTDOWN_APP), + pytest.param("app start", EXIT.ERR_START_APP), + ), +) def test_error_apply( - _arun, _logger, _log_file, _chmod, _chown, _print, - test_qapp, monkeypatch, action, code + _arun, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, + monkeypatch, + action, + code, ): _dom0 = TestVM("dom0", test_qapp, klass="AdminVM") vm = TestVM("vm", test_qapp, klass="TemplateVM") appvm = TestVM( - "appvm", test_qapp, klass="AppVM", template=vm, - features=Features("appvm", test_qapp, {'servicevm': True})) + "appvm", + test_qapp, + klass="AppVM", + template=vm, + features=Features("appvm", test_qapp, {"servicevm": True}), + ) monkeypatch.setattr(vmupdate, "get_targets", lambda *_: [vm, appvm]) - monkeypatch.setattr(vmupdate, "run_update", - lambda *_: [EXIT.OK, {"vm": FinalStatus.SUCCESS}]) + monkeypatch.setattr( + vmupdate, + "run_update", + lambda *_: [EXIT.OK, {"vm": FinalStatus.SUCCESS}], + ) def raiser(*_args, **_kwargs): raise qubesadmin.exc.QubesVMError("foo") + if action == "template shutdown": vm.shutdown = raiser elif action == "app shutdown": @@ -477,17 +773,23 @@ def raiser(*_args, **_kwargs): assert retcode == code -@patch('vmupdate.update_manager.TerminalMultiBar.print') -@patch('os.chmod') -@patch('os.chown') -@patch('logging.FileHandler') -@patch('logging.getLogger') +@patch("vmupdate.update_manager.TerminalMultiBar.print") +@patch("os.chmod") +@patch("os.chown") +@patch("logging.FileHandler") +@patch("logging.getLogger") def test_error_usage_wrong_param( - _logger, _log_file, _chmod, _chown, _print, test_qapp, + _logger, + _log_file, + _chmod, + _chown, + _print, + test_qapp, ): _dom0 = TestVM("dom0", test_qapp, klass="AdminVM") - retcode = main(( - "--just-print-progress", "--targets", 'vm', "--force-update"), - test_qapp) + retcode = main( + ("--just-print-progress", "--targets", "vm", "--force-update"), + test_qapp, + ) assert retcode == EXIT.ERR_USAGE diff --git a/vmupdate/update_manager.py b/vmupdate/update_manager.py index 0e3aea0..fdabe4e 100644 --- a/vmupdate/update_manager.py +++ b/vmupdate/update_manager.py @@ -2,7 +2,8 @@ # # The Qubes OS Project, http://www.qubes-os.org # -# Copyright (C) 2022 Piotr Bartman +# Copyright (C) 2022-2025 Piotr Bartman-Szwarc +# # Copyright (C) 2022 Marek Marczykowski-Górecki # # @@ -32,11 +33,11 @@ from tqdm import tqdm -from .agent.source.status import StatusInfo, FinalStatus, Status -from .qube_connection import QubeConnection from vmupdate.agent.source.log_config import init_logs from vmupdate.agent.source.common.process_result import ProcessResult from vmupdate.agent.source.common.exit_codes import EXIT +from .agent.source.status import StatusInfo, FinalStatus, Status, FormatedLine +from .qube_connection import QubeConnection class UpdateManager: @@ -44,18 +45,20 @@ class UpdateManager: Update multiple qubes simultaneously. """ - def __init__(self, qubes, args, log): + def __init__(self, qubes, args, log, dom0=False): self.qubes = qubes self.max_concurrency = args.max_concurrency self.show_output = args.show_output self.quiet = args.quiet self.no_progress = args.no_progress self.just_print_progress = args.just_print_progress + self.download_only = args.download_only self.buffered = not args.just_print_progress and not args.no_progress self.buffer = "" self.cleanup = not args.no_cleanup self.ret_code = EXIT.OK self.log = log + self.dom0 = dom0 def run(self, agent_args): """ @@ -67,24 +70,41 @@ def run(self, agent_args): return EXIT.OK, {} show_progress = not self.quiet and not self.no_progress - SimpleTerminalBar.reinit_class() - progress_output = SimpleTerminalBar \ - if self.just_print_progress else tqdm + SimpleTerminalBar.reinit_class(self.download_only) + progress_output = ( + SimpleTerminalBar if self.just_print_progress else tqdm + ) progress_bar = MultipleUpdateMultipleProgressBar( dummy=not show_progress, output=progress_output, max_concurrency=self.max_concurrency, - printer=self.print if self.show_output else None + printer=self.print if self.show_output else None, ) for qube in self.qubes: - progress_bar.add_bar(qube.name) + disp_name = ( + agent_args.display_name + if agent_args.display_name is not None + else qube.name + ) + progress_bar.add_bar(disp_name) progress_bar.pool.apply_async( update_qube, - (qube, agent_args, show_progress, - progress_bar.status_notifier, progress_bar.termination), - callback=self.collect_result, error_callback=print + ( + qube, + agent_args, + show_progress, + progress_bar.status_notifier, + progress_bar.termination, + self.dom0, + ), + callback=self.collect_result, + error_callback=print, ) + if qube.klass == "AdminVM" and show_progress: + # progress of AdminVM is continuation of different process, + # so we want to skip 0 value at beginning + progress_bar.progress_bars[qube.name].progress = None progress_bar.pool.close() progress_bar.feeding() @@ -121,21 +141,25 @@ def collect_result(self, result_tuple: Tuple[str, ProcessResult]): self.ret_code = max(self.ret_code, vm_code) if self.show_output: - for line in result.out.split('\n'): - self.print(qube_name + ":out:", line) - for line in result.err.split('\n'): - self.print(qube_name + ":err:", line) + for line in result.out.split("\n"): + self.print(FormatedLine(qube_name, "out", line)) + for line in result.err.split("\n"): + self.print(FormatedLine(qube_name, "err", line)) elif not self.quiet and self.no_progress: self.print(result.out) def print(self, *args): if self.buffered: - self.buffer += ' '.join(args) + '\n' + self.buffer += " ".join(map(str, args)) + "\n" else: print(*args, file=sys.stdout, flush=True) class TerminalMultiBar: + """ + Handles multiple progress bars in terminal. + """ + def __init__(self): self.progresses = [] @@ -145,7 +169,12 @@ def print(self): class SimpleTerminalBar: + """ + Simple progress bar for terminal output. Could be used by TerminalMultiBar. + """ + PARENT_MULTI_BAR = None + DOWNLOAD_ONLY = False def __init__(self, total, position, desc): assert position == len(SimpleTerminalBar.PARENT_MULTI_BAR.progresses) @@ -156,19 +185,27 @@ def __init__(self, total, position, desc): def __str__(self): info = None - name, status = self.desc.split(' ', 1) + name, status = self.desc.split(" ", 1) status = status[1:-1] # remove brackets - if status in (FinalStatus.SUCCESS.value, - FinalStatus.ERROR.value, - FinalStatus.CANCELLED.value, - FinalStatus.NO_UPDATES.value): + if status in ( + FinalStatus.SUCCESS.value, + FinalStatus.ERROR.value, + FinalStatus.CANCELLED.value, + FinalStatus.NO_UPDATES.value, + ): + if SimpleTerminalBar.DOWNLOAD_ONLY: + return "" info = status.replace(" ", "_") status = "done" if status == Status.UPDATING.value: + if self.progress is None: + return "" info = self.progress return f"{name} {status} {info}" def update(self, progress): + if self.progress is None: + self.progress = 0 self.progress += progress SimpleTerminalBar.PARENT_MULTI_BAR.print() @@ -178,11 +215,11 @@ def set_description(self, desc: str): def close(self): """Implementation of tqdm API""" - pass @staticmethod - def reinit_class(): + def reinit_class(download_only=False): SimpleTerminalBar.PARENT_MULTI_BAR = TerminalMultiBar() + SimpleTerminalBar.DOWNLOAD_ONLY = download_only class MultipleUpdateMultipleProgressBar: @@ -194,7 +231,7 @@ def __init__(self, dummy, output, max_concurrency, printer: Optional): self.dummy = dummy self.manager = multiprocessing.Manager() - self.termination = self.manager.Value('b', False) + self.termination = self.manager.Value("b", False) self.status_notifier = self.manager.Queue() # save original signal handler for SIGINT @@ -221,8 +258,9 @@ def add_bar(self, qname: str): self.progresses[qname] = 0 self.progress_bars[qname] = self.output_class( - total=100, position=len(self.progress_bars), - desc=f"{qname} ({Status.PENDING.value})" + total=100, + position=len(self.progress_bars), + desc=f"{qname} ({Status.PENDING.value})", ) def feeding(self): @@ -237,18 +275,20 @@ def feeding(self): left_to_finish = len(self.progresses) while left_to_finish: try: - feed: Optional[StatusInfo, str] = \ - self.status_notifier.get(block=True) + feed: Optional[StatusInfo, str] = self.status_notifier.get( + block=True + ) if feed is None: continue - elif isinstance(feed, StatusInfo): + if isinstance(feed, StatusInfo): status_name = feed.status.value if feed.status == Status.DONE: left_to_finish -= 1 status_name = feed.info.value self.statuses[feed.qname] = FinalStatus(status_name) self.progress_bars[feed.qname].set_description( - f"{feed.qname} ({status_name})") + f"{feed.qname} ({status_name})" + ) if feed.status == Status.UPDATING: self._update(feed.qname, feed.info) elif self.print is not None: @@ -279,7 +319,7 @@ def close(self): def update_qube( - qube, agent_args, show_progress, status_notifier, termination + qube, agent_args, show_progress, status_notifier, termination, dom0 ) -> Tuple[str, ProcessResult]: """ Create and run `UpdateAgentManager` for qube. @@ -289,29 +329,37 @@ def update_qube( :param show_progress: if progress should be printed in real time :param status_notifier: an object to be fed with the progress data :param termination: signal to gracefully terminate subprocess + :param dom0: whether to use qubes-dom0-update (do download&install) + or just update agent to install prepared updates :return: """ + if agent_args.display_name is not None: + status_notifier = StatusNotifierWrapper( + status_notifier, agent_args.display_name + ) + if termination.value: status_notifier.put(StatusInfo.done(qube, FinalStatus.CANCELLED)) return qube.name, ProcessResult(EXIT.SIGINT, "Canceled") - status_notifier.put(StatusInfo.updating(qube, 0)) try: runner = UpdateAgentManager( qube.app, qube, agent_args=agent_args, - show_progress=show_progress + show_progress=show_progress, + dom0=dom0, ) result = runner.run_agent( agent_args=agent_args, status_notifier=status_notifier, - termination=termination + termination=termination, ) except Exception as exc: # pylint: disable=broad-except status_notifier.put(StatusInfo.done(qube, FinalStatus.ERROR)) return qube.name, ProcessResult( - EXIT.ERR_VM_UNHANDLED, f"ERROR (exception {str(exc)})") + EXIT.ERR_VM_UNHANDLED, f"ERROR (exception {str(exc)})" + ) return qube.name, result @@ -319,21 +367,27 @@ class UpdateAgentManager: """ Send update agent files and run it in the qube. """ + AGENT_RELATIVE_DIR = "agent" ENTRYPOINT = AGENT_RELATIVE_DIR + "/entrypoint.py" - LOGPATH = '/var/log/qubes' - FORMAT_LOG = '%(asctime)s %(message)s' + LOGPATH = "/var/log/qubes" + FORMAT_LOG = "%(asctime)s %(message)s" WORKDIR = "/run/qubes-update/" - def __init__( - self, app, qube, agent_args, show_progress): + def __init__(self, app, qube, agent_args, show_progress, dom0): self.qube = qube self.app = app - - (self.log, self.log_handler, log_level, - self.log_path, self.log_formatter) = init_logs( - directory=UpdateAgentManager.LOGPATH, - file=f'update-{qube.name}.log', + self.dom0 = dom0 + + ( + self.log, + self.log_handler, + _log_level, + self.log_path, + self.log_formatter, + ) = init_logs( + directory=self.LOGPATH, + file=f"update-{qube.name}.log", format_=UpdateAgentManager.FORMAT_LOG, level=agent_args.log, truncate_file=False, @@ -344,67 +398,127 @@ def __init__( self.show_progress = show_progress def run_agent( - self, agent_args, status_notifier, termination + self, agent_args, status_notifier, termination ) -> ProcessResult: """ Copy agent file to dest vm, run entrypoint, collect output and logs. """ - result = self._run_agent( - agent_args, status_notifier, termination) - output = result.out.split("\n") + result.err.split("\n") - for line in output: - self.log.debug('agent output: %s', line) - self.log.info('agent exit code: %d', result.code) - if not agent_args.show_output or not output: - result.out = "OK" if result.code == EXIT.OK else \ - f"ERROR (exit code {result.code}, details in {self.log_path})" + if self.qube.klass == "AdminVM" and not self.dom0: + # using UpdateVM to download updates + status_notifier = StatusNotifierWrapper(status_notifier, "dom0") + + result = self._run_agent(agent_args, status_notifier, termination) + + self._log_output(result, agent_args.show_output) return result def _run_agent( - self, agent_args, status_notifier, termination + self, agent_args, status_notifier, termination ) -> ProcessResult: - self.log.info('Running update agent for %s', self.qube.name) - dest_dir = UpdateAgentManager.WORKDIR - dest_agent = os.path.join(dest_dir, UpdateAgentManager.ENTRYPOINT) - this_dir = os.path.dirname(os.path.realpath(__file__)) - src_dir = join(this_dir, UpdateAgentManager.AGENT_RELATIVE_DIR) + self.log.info("Running update agent for %s", self.qube.name) + dest_dir = None + src_dir = None + cleanup = False + if self.qube.klass == "AdminVM": + if self.dom0: + entrypoint = ["sudo", "qubes-dom0-update", "-y"] + if agent_args.just_print_progress or self.show_progress: + entrypoint.append("--just-print-progress") + if agent_args.quiet: + # silent is equivalent to quiet for dom0-update + entrypoint.append("--silent") + else: + this_dir = os.path.dirname(os.path.realpath(__file__)) + entrypoint = join(this_dir, UpdateAgentManager.ENTRYPOINT) + else: + cleanup = self.cleanup + dest_dir = UpdateAgentManager.WORKDIR + entrypoint = os.path.join(dest_dir, UpdateAgentManager.ENTRYPOINT) + this_dir = os.path.dirname(os.path.realpath(__file__)) + src_dir = join(this_dir, UpdateAgentManager.AGENT_RELATIVE_DIR) with QubeConnection( - self.qube, - dest_dir, - self.cleanup, - self.log, - self.show_progress, - status_notifier + self.qube, + dest_dir, + cleanup, + self.log, + self.show_progress, + status_notifier, ) as qconn: - self.log.info( - "Transferring files to destination qube: %s", self.qube.name) - result = qconn.transfer_agent(src_dir) - if result: - self.log.error('Qube communication error code: %i', result.code) - return result + result = self._transfer_agent(qconn, src_dir) if termination.value: qconn.status = FinalStatus.CANCELLED return ProcessResult(EXIT.SIGINT, "", "Cancelled") + result += self._run_entrypoint(qconn, entrypoint, agent_args) + + self._read_logs(qconn) + + return result + + def _transfer_agent(self, qconn, src_dir) -> ProcessResult: + result = ProcessResult() + if self.qube.klass != "AdminVM": self.log.info( - "The agent is starting the task in qube: %s", self.qube.name) - result += qconn.run_entrypoint(dest_agent, agent_args) - if not result and qconn.status != FinalStatus.NO_UPDATES: - qconn.status = FinalStatus.SUCCESS - - result_logs = qconn.read_logs() - if result_logs: - self.log.error( - "Problem with collecting logs from %s, return code: %i", - self.qube.name, result_logs.code) - # agent logs already have timestamp - self.log_handler.setFormatter(logging.Formatter('%(message)s')) - # critical -> always write agent logs - for log_line in result_logs.out.split("\n"): - if log_line: - self.log.critical("%s", log_line) - self.log_handler.setFormatter(self.log_formatter) + "Transferring files to destination qube: %s", self.qube.name + ) + result += qconn.transfer_agent(src_dir) + if result: + self.log.error("Qube communication error code: %i", result.code) + return result + return result + def _run_entrypoint(self, qconn, entrypoint, agent_args) -> ProcessResult: + result = ProcessResult() + self.log.info( + "The agent is starting the task in qube: %s", self.qube.name + ) + self.log.debug("%s", entrypoint) + result += qconn.run_entrypoint(entrypoint, agent_args) + if not result and qconn.status != FinalStatus.NO_UPDATES: + qconn.status = FinalStatus.SUCCESS return result + + def _read_logs(self, qconn): + result_logs = qconn.read_logs() + if result_logs: + self.log.error( + "Problem with collecting logs from %s, return code: %i", + self.qube.name, + result_logs.code, + ) + # agent logs already have timestamp + self.log_handler.setFormatter(logging.Formatter("%(message)s")) + # critical -> always write agent logs + for log_line in result_logs.out.split("\n"): + if log_line: + self.log.critical("%s", log_line) + self.log_handler.setFormatter(self.log_formatter) + + def _log_output(self, result: ProcessResult, show_output: bool): + output = result.out.split("\n") + result.err.split("\n") + for line in output: + self.log.debug("agent output: %s", line) + self.log.info("agent exit code: %d", result.code) + if not show_output or not output: + result.out = ( + "OK" + if result.code == EXIT.OK + else f"ERROR (exit code {result.code}, details in {self.log_path})" + ) + + +class StatusNotifierWrapper: + """ + Masks proxy VM with display name. + """ + + def __init__(self, status_notifier, qube_name): + self.status_notifier = status_notifier + self.qube_name = qube_name + + def put(self, message): + if isinstance(message, (StatusInfo, FormatedLine)): + message.qname = self.qube_name + self.status_notifier.put(message) diff --git a/vmupdate/vmupdate.py b/vmupdate/vmupdate.py index b2e2fe5..edc6cb8 100644 --- a/vmupdate/vmupdate.py +++ b/vmupdate/vmupdate.py @@ -2,6 +2,7 @@ """ Update qubes. """ + import argparse import asyncio import logging @@ -20,23 +21,22 @@ from .agent.source.args import AgentArgs DEFAULT_UPDATE_IF_STALE = 7 -LOGPATH = '/var/log/qubes/qubes-vm-update.log' -LOG_FORMAT = '%(asctime)s %(message)s' +LOGPATH = "/var/log/qubes/qubes-vm-update.log" +LOG_FORMAT = "%(asctime)s %(message)s" class ArgumentError(Exception): - """Nonsense arguments - """ + """Nonsense arguments""" def main(args=None, app=qubesadmin.Qubes()): args = parse_args(args, app) - log_handler = logging.FileHandler(LOGPATH, encoding='utf-8') + log_handler = logging.FileHandler(LOGPATH, encoding="utf-8") log_formatter = logging.Formatter(LOG_FORMAT) log_handler.setFormatter(log_formatter) - log = logging.getLogger('vm-update') + log = logging.getLogger("vm-update") log.setLevel(args.log) log.addHandler(log_handler) try: @@ -59,99 +59,185 @@ def main(args=None, app=qubesadmin.Qubes()): print("No qube selected for update") return EXIT.OK_NO_UPDATES if args.signal_no_updates else EXIT.OK - independent = [target for target in targets if target.klass in ( - 'TemplateVM', 'StandaloneVM')] - derived = [target for target in targets if target.klass not in ( - 'TemplateVM', 'StandaloneVM')] + admin = [target for target in targets if target.klass == "AdminVM"] + independent = [ + target + for target in targets + if target.klass in ("TemplateVM", "StandaloneVM") + ] + derived = [ + target + for target in targets + if target.klass not in ("AdminVM", "TemplateVM", "StandaloneVM") + ] + + no_updates = True + ret_code_admin = EXIT.OK + if admin: + message = f"The admin VM ({admin[0].name}) will be updated." + else: + message = "The admin VM will not be updated." + if args.dry_run: + print(message) + elif admin: + log.debug(message) + if args.just_print_progress and args.no_refresh: + # internal usage just for installing ready updates, use carefully + ret_code_admin, admin_status = run_update( + admin, args, log, "admin VM" + ) + else: + # use qubes-dom0-update to update dom0 + ret_code_admin, admin_status = run_update( + admin, args, log, "admin VM", dom0=True + ) + no_updates = all( + stat == FinalStatus.NO_UPDATES for stat in admin_status.values() + ) # independent qubes first (TemplateVMs, StandaloneVMs) ret_code_independent, templ_statuses = run_update( - independent, args, log, "templates and standalones") - no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in templ_statuses.values()) + independent, args, log, "templates and standalones" + ) + no_updates = ( + all(stat == FinalStatus.NO_UPDATES for stat in templ_statuses.values()) + and no_updates + ) # then derived qubes (AppVMs...) ret_code_appvm, app_statuses = run_update(derived, args, log) - no_updates = all(stat == FinalStatus.NO_UPDATES - for stat in app_statuses.values()) and no_updates + no_updates = ( + all(stat == FinalStatus.NO_UPDATES for stat in app_statuses.values()) + and no_updates + ) ret_code_restart = apply_updates_to_appvm( - args, independent, templ_statuses, app_statuses, log) + args, independent, templ_statuses, app_statuses, log + ) - ret_code = max(ret_code_independent, ret_code_appvm, ret_code_restart) + ret_code = max( + ret_code_admin, ret_code_independent, ret_code_appvm, ret_code_restart + ) if ret_code == EXIT.OK and no_updates and args.signal_no_updates: return EXIT.OK_NO_UPDATES + if ret_code == EXIT.OK_NO_UPDATES and not args.signal_no_updates: + return EXIT.OK return ret_code def parse_args(args, app): parser = argparse.ArgumentParser() try: - default_update_if_stale = int(app.domains["dom0"].features.get( - "qubes-vm-update-update-if-stale", DEFAULT_UPDATE_IF_STALE)) + default_update_if_stale = int( + app.domains["dom0"].features.get( + "qubes-vm-update-update-if-stale", DEFAULT_UPDATE_IF_STALE + ) + ) except qubesadmin.exc.QubesDaemonAccessError: default_update_if_stale = DEFAULT_UPDATE_IF_STALE - parser.add_argument('--max-concurrency', '-x', - action='store', - help='Maximum number of VMs configured simultaneously ' - '(default: number of cpus)', - type=int) - parser.add_argument('--dry-run', action='store_true', - help='Just print what happens.') parser.add_argument( - '--signal-no-updates', action='store_true', - help='Return exit code 100 instead of 0 ' - 'if there is no updates available.') + "--max-concurrency", + "-x", + action="store", + help="Maximum number of VMs configured simultaneously " + "(default: number of cpus)", + type=int, + ) + parser.add_argument( + "--dry-run", action="store_true", help="Just print what happens." + ) + parser.add_argument( + "--signal-no-updates", + action="store_true", + help="Return exit code 100 instead of 0 " + "if there is no updates available.", + ) restart = parser.add_mutually_exclusive_group() restart.add_argument( - '--apply-to-sys', '--restart', '-r', - action='store_true', - help='Restart not updated ServiceVMs whose template has been updated.') + "--apply-to-sys", + "--restart", + "-r", + action="store_true", + help="Restart not updated ServiceVMs whose template has been updated.", + ) restart.add_argument( - '--apply-to-all', '-R', action='store_true', - help='Restart not updated ServiceVMs and shutdown not updated AppVMs ' - 'whose template has been updated.') + "--apply-to-all", + "-R", + action="store_true", + help="Restart not updated ServiceVMs and shutdown not updated AppVMs " + "whose template has been updated.", + ) restart.add_argument( - '--no-apply', action='store_true', - help='DEFAULT. Do not restart/shutdown any AppVMs.') + "--no-apply", + action="store_true", + help="DEFAULT. Do not restart/shutdown any AppVMs.", + ) update_state = parser.add_mutually_exclusive_group() update_state.add_argument( - '--force-update', action='store_true', - help='Attempt to update all targeted VMs ' - 'even if no updates are available') + "--force-update", + action="store_true", + help="Attempt to update all targeted VMs " + "even if no updates are available", + ) update_state.add_argument( - '--update-if-stale', action='store', - help='DEFAULT. ' - 'Attempt to update targeted VMs with known updates available ' - 'or for which last update check was more than N days ago. ' - '(default: %(default)d)', - type=int, default=default_update_if_stale) + "--update-if-stale", + action="store", + help="DEFAULT. " + "Attempt to update targeted VMs with known updates available " + "or for which last update check was more than N days ago. " + "(default: %(default)d)", + type=int, + default=default_update_if_stale, + ) update_state.add_argument( - '--update-if-available', action='store_true', - help='Update targeted VMs with known updates available.') + "--update-if-available", + action="store_true", + help="Update targeted VMs with known updates available.", + ) parser.add_argument( - '--skip', action='store', - help='Comma separated list of VMs to be skipped, ' - 'works with all other options.', default="") + "--skip", + action="store", + help="Comma separated list of VMs to be skipped, " + "works with all other options.", + default="", + ) parser.add_argument( - '--targets', action='store', - help='Comma separated list of VMs to target. Ignores conditions.') + "--targets", + action="store", + help="Comma separated list of VMs to target. Ignores conditions.", + ) parser.add_argument( - '--templates', '-T', action='store_true', - help='Target all updatable TemplateVMs.') + "--templates", + "-T", + action="store_true", + help="Target all updatable TemplateVMs.", + ) parser.add_argument( - '--standalones', '-S', action='store_true', - help='Target all updatable StandaloneVMs.') + "--standalones", + "-S", + action="store_true", + help="Target all updatable StandaloneVMs.", + ) parser.add_argument( - '--apps', '-A', action='store_true', - help='Target running updatable AppVMs to update in place.') + "--apps", + "-A", + action="store_true", + help="Target running updatable AppVMs to update in place.", + ) parser.add_argument( - '--all', action='store_true', - help='DEFAULT. Target all updatable VMs except AdminVM. ' - 'Use explicitly with "--targets" to include both.') + "--all", + action="store_true", + help="DEFAULT. Target all updatable VMs except AdminVM. " + 'Use explicitly with "--targets" to include both.', + ) + + # for internal usage, e.g., download updates via proxy vm + parser.add_argument( + "--display-name", action="store", help=argparse.SUPPRESS + ) AgentArgs.add_arguments(parser) args = parser.parse_args(args) @@ -170,28 +256,40 @@ def get_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: def preselect_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: targets = set() - updatable = {vm for vm in app.domains if getattr(vm, 'updateable', False)} - default_targeting = (not args.templates and not args.standalones and - not args.apps and not args.targets) + updatable = {vm for vm in app.domains if getattr(vm, "updateable", False)} + default_targeting = ( + not args.templates + and not args.standalones + and not args.apps + and not args.targets + ) if args.all or default_targeting: # filter out stopped AppVMs and DispVMs (?) - targets = {vm for vm in updatable - if vm.klass not in ("AppVM", "DispVM") or vm.is_running()} + targets = { + vm + for vm in updatable + if vm.klass not in ("AppVM", "DispVM") or vm.is_running() + } else: # if not all updatable are included, target a specific classes if args.templates: - targets.update([vm for vm in updatable - if vm.klass == 'TemplateVM']) + targets.update([vm for vm in updatable if vm.klass == "TemplateVM"]) if args.standalones: - targets.update([vm for vm in updatable - if vm.klass == 'StandaloneVM']) + targets.update( + [vm for vm in updatable if vm.klass == "StandaloneVM"] + ) if args.apps: - targets.update({vm for vm in app.domains - if vm.klass == 'AppVM' and vm.is_running()}) + targets.update( + { + vm + for vm in app.domains + if vm.klass == "AppVM" and vm.is_running() + } + ) # user can target non-updatable vm if she like if args.targets: - names = args.targets.split(',') + names = args.targets.split(",") explicit_targets = {vm for vm in app.domains if vm.name in names} if len(names) != len(explicit_targets): target_names = {q.name for q in explicit_targets} @@ -204,16 +302,16 @@ def preselect_targets(args, app) -> Set[qubesadmin.vm.QubesVM]: targets.update(explicit_targets) # remove skipped qubes and dom0 - not a target - to_skip = args.skip.split(',') - if 'dom0' in targets and not args.quiet: - print("Skipping dom0. To update AdminVM use `qubes-dom0-update`") - targets = {vm for vm in targets - if vm.name != 'dom0' and vm.name not in to_skip} + to_skip = args.skip.split(",") + targets = {vm for vm in targets if vm.name not in to_skip} # exclude vms with `skip-update` feature, but allow --targets to override it if not args.targets: - targets = {vm for vm in targets - if not bool(vm.features.get('skip-update', False))} + targets = { + vm + for vm in targets + if not bool(vm.features.get("skip-update", False)) + } return targets @@ -226,9 +324,20 @@ def select_targets(targets, args) -> Set[qubesadmin.vm.QubesVM]: selected = set() for vm in targets: try: - to_update = vm.features.get('updates-available', False) + to_update = vm.features.get("updates-available", False) except qubesadmin.exc.QubesDaemonCommunicationError: to_update = False + try: + prohibit_start = vm.features.get("prohibit-start", False) + except qubesadmin.exc.QubesDaemonCommunicationError: + prohibit_start = False + try: + skip_update = vm.features.get("skip-update", False) + except qubesadmin.exc.QubesDaemonCommunicationError: + skip_update = False + + if skip_update or prohibit_start: + continue # there are updates available => select if to_update: @@ -249,13 +358,15 @@ def select_targets(targets, args) -> Set[qubesadmin.vm.QubesVM]: def is_stale(vm, expiration_period): today = datetime.today() try: - if not ('qrexec' in vm.features.keys() - and vm.features.get('os', '') == 'Linux'): + if not ( + "qrexec" in vm.features.keys() + and vm.features.get("os", "") == "Linux" + ): return False last_update_str = vm.features.check_with_template( - 'last-updates-check', - datetime.fromtimestamp(0).strftime('%Y-%m-%d %H:%M:%S') + "last-updates-check", + datetime.fromtimestamp(0).strftime("%Y-%m-%d %H:%M:%S"), ) last_update = datetime.fromisoformat(last_update_str) if (today - last_update).days > expiration_period: @@ -266,25 +377,34 @@ def is_stale(vm, expiration_period): def run_update( - targets, args, log, qube_klass="qubes" + targets, args, log, qube_klass="qubes", dom0=False ) -> Tuple[int, Dict[str, FinalStatus]]: - if not targets: - return EXIT.OK, {} - - message = f"Following {qube_klass} will be updated:" + \ - ",".join((target.name for target in targets)) + if targets: + message = f"Following {qube_klass} will be updated: " + ", ".join( + (target.name for target in targets) + ) + else: + if qube_klass == "qubes": + message = "" # no need to inform about app VMs etc. + else: + message = f"No {qube_klass} will be updated." if args.dry_run: - print(message) + if message: + print(message) return EXIT.OK, {target.name: FinalStatus.SUCCESS for target in targets} - else: - log.debug(message) + log.debug(message) + + if not targets: + return EXIT.OK, {} - runner = update_manager.UpdateManager(targets, args, log=log) + runner = update_manager.UpdateManager(targets, args, log=log, dom0=dom0) ret_code, statuses = runner.run(agent_args=args) if ret_code: log.error("Updating fails with code: %d", ret_code) - log.debug("Updating report: %s", - ", ".join((k + ":" + v.value for k, v in statuses.items()))) + log.debug( + "Updating report: %s", + ", ".join((k + ":" + v.value for k, v in statuses.items())), + ) return ret_code, statuses @@ -309,11 +429,11 @@ def get_boolean_feature(vm, feature_name, default=False): def apply_updates_to_appvm( - args, - vm_updated: Iterable, - template_statuses: Dict[str, FinalStatus], - derived_statuses: Dict[str, FinalStatus], - log + args, + vm_updated: Iterable, + template_statuses: Dict[str, FinalStatus], + derived_statuses: Dict[str, FinalStatus], + log, ) -> int: """ Shutdown running templates and then restart/shutdown derived AppVMs. @@ -328,22 +448,31 @@ def apply_updates_to_appvm( return EXIT.OK updated_tmpls = [ - vm for vm in vm_updated - if bool(template_statuses[vm.name]) and vm.klass == 'TemplateVM' + vm + for vm in vm_updated + if bool(template_statuses[vm.name]) and vm.klass == "TemplateVM" ] to_restart, to_shutdown = get_derived_vm_to_apply( - updated_tmpls, derived_statuses) - templates_to_shutdown = [template for template in updated_tmpls - if template.is_running()] + updated_tmpls, derived_statuses + ) + templates_to_shutdown = [ + template for template in updated_tmpls if template.is_running() + ] if args.dry_run: - print("Following templates will be shutdown:", - ",".join((target.name for target in templates_to_shutdown))) + print( + "Following templates will be shutdown:", + ",".join((target.name for target in templates_to_shutdown)), + ) # we do not check if any volume is outdated, we expect it will be. - print("Following qubes CAN be restarted:", - ",".join((target.name for target in to_restart))) - print("Following qubes CAN be shutdown:", - ",".join((target.name for target in to_shutdown))) + print( + "Following qubes CAN be restarted:", + ",".join((target.name for target in to_restart)), + ) + print( + "Following qubes CAN be shutdown:", + ",".join((target.name for target in to_shutdown)), + ) return EXIT.OK # first shutdown templates to apply changes to the root volume @@ -354,14 +483,17 @@ def apply_updates_to_appvm( log.error("Shutdown of some templates fails with code %d", ret_code) log.warning( "Derived VMs of the following templates will be omitted: %s", - ", ".join((t.name for t in updated_tmpls if t.is_running()))) + ", ".join((t.name for t in updated_tmpls if t.is_running())), + ) ret_code = EXIT.ERR_SHUTDOWN_TMPL # Some templates are not down dur to errors, there is no point in # restarting their derived AppVMs - ready_templates = [tmpl for tmpl in updated_tmpls - if not tmpl.is_running()] + ready_templates = [ + tmpl for tmpl in updated_tmpls if not tmpl.is_running() + ] to_restart, to_shutdown = get_derived_vm_to_apply( - ready_templates, derived_statuses) + ready_templates, derived_statuses + ) # both flags `restart` and `apply-to-all` include service vms ret_code_ = restart_vms(to_restart, log) @@ -383,10 +515,12 @@ def get_derived_vm_to_apply(templates, derived_statuses): to_shutdown = set() for vm in possibly_changed_vms: - if (not bool(derived_statuses.get(vm.name, False)) - and vm.is_running() - and (vm.klass != 'DispVM' or not vm.auto_cleanup)): - if get_boolean_feature(vm, 'servicevm', False): + if ( + not bool(derived_statuses.get(vm.name, False)) + and vm.is_running() + and (vm.klass != "DispVM" or not vm.auto_cleanup) + ): + if get_boolean_feature(vm, "servicevm", False): to_restart.add(vm) else: to_shutdown.add(vm) @@ -430,5 +564,5 @@ def restart_vms(to_restart, log): return ret_code -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main())