From 6f1c0b1dd065e255e6df534993ddc024b2a40106 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 7 Apr 2026 07:55:54 +0200 Subject: [PATCH 1/3] cbscommon: introduce shared library for unified process and git utilities Create the `cbscommon` library to centralize foundational logic used across CBS components (e.g., `cbscore`, `crt`). This initial implementation provides: - `cbscommon.process`: A robust, asynchronous subprocess wrapper with automatic log sanitization and environment isolation. - `cbscommon.git`: A unified, async-first interface for git operations, consolidating scattered git utilities into a single module. - `cbscommon.exceptions`: Shared exception types for consistent error reporting. This change also integrates `cbscommon` as a workspace member in the root `pyproject.toml`. --- cbscommon/.python-version | 1 + cbscommon/README.md | 48 + cbscommon/pyproject.toml | 15 + cbscommon/src/cbscommon/__init__.py | 12 + cbscommon/src/cbscommon/exceptions.py | 30 + cbscommon/src/cbscommon/git/__init__.py | 6 + cbscommon/src/cbscommon/git/cmds.py | 1209 +++++++++++++++++++ cbscommon/src/cbscommon/git/exceptions.py | 137 +++ cbscommon/src/cbscommon/git/types.py | 43 + cbscommon/src/cbscommon/process/__init__.py | 6 + cbscommon/src/cbscommon/process/cmds.py | 147 +++ cbscommon/src/cbscommon/process/types.py | 19 + cbscommon/src/cbscommon/py.typed | 2 + pyproject.toml | 9 +- uv.lock | 45 +- 15 files changed, 1693 insertions(+), 36 deletions(-) create mode 100644 cbscommon/.python-version create mode 100644 cbscommon/README.md create mode 100644 cbscommon/pyproject.toml create mode 100644 cbscommon/src/cbscommon/__init__.py create mode 100644 cbscommon/src/cbscommon/exceptions.py create mode 100644 cbscommon/src/cbscommon/git/__init__.py create mode 100644 cbscommon/src/cbscommon/git/cmds.py create mode 100644 cbscommon/src/cbscommon/git/exceptions.py create mode 100644 cbscommon/src/cbscommon/git/types.py create mode 100644 cbscommon/src/cbscommon/process/__init__.py create mode 100644 cbscommon/src/cbscommon/process/cmds.py create mode 100644 cbscommon/src/cbscommon/process/types.py create mode 100644 cbscommon/src/cbscommon/py.typed diff --git a/cbscommon/.python-version b/cbscommon/.python-version new file mode 100644 index 00000000..24ee5b1b --- /dev/null +++ b/cbscommon/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/cbscommon/README.md b/cbscommon/README.md new file mode 100644 index 00000000..de090ffe --- /dev/null +++ b/cbscommon/README.md @@ -0,0 +1,48 @@ +# cbscommon + +[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](../COPYING-GPL3) + +`cbscommon` is a shared Python library providing foundational utilities and unified modules for the Clyso Enterprise Storage Build Service (CBS) project. + +## Overview + +This package centralizes common logic used across various CBS components (e.g., `crt`, `cbscore`, `cbsd`), ensuring consistency in process execution, git operations, and error handling. It is designed for asynchronous environments and prioritizes security and isolation. + +## Key Features + +- **Asynchronous by Design**: Built on top of `asyncio` for non-blocking I/O operations. +- **Secure Process Execution**: Subprocess runners include automatic log sanitization to prevent leaking sensitive data (passwords, tokens). +- **Environment Isolation**: Support for scrubbing Python environment variables to prevent host/venv leakage into subprocesses. +- **Unified Git Interface**: A comprehensive, async-first wrapper around the `git` CLI, eliminating the need for heavyweight libraries like `GitPython`. + +## Modules + +### `cbscommon.git` +Provides a high-level asynchronous API for git operations. +- **Commands**: Support for `clone`, `checkout`, `fetch`, `push`, `status`, `am`, `worktree`, and more. +- **Parsing**: Replicates logic for parsing git porcelain output (e.g., push status). +- **Exceptions**: A dedicated hierarchy of git-related errors (`GitError`, `GitPushError`, etc.). + +### `cbscommon.process` +A robust wrapper for `asyncio.create_subprocess_exec`. +- **Log Sanitization**: Automatically redacts `--passphrase`, `--pass`, and custom `SecureArg` values from logs. +- **Python Env Reset**: Capability to reset the `PATH` and other variables to ensure subprocesses use the intended system Python instead of the calling virtual environment. +- **Streaming**: Supports real-time log streaming via callbacks. + +### `cbscommon.exceptions` +Defines the base `CBSCommonError` and other shared exception types used throughout the library to provide consistent error reporting. + +## Installation + +This package is intended for internal use within the CBS workspace. It can be added as a dependency in other CBS components via `uv` or standard Python packaging tools. + +```toml +[tool.uv.sources] +cbscommon = { workspace = true } +``` + +## Development + +- **Python Version**: 3.13+ +- **Type Checking**: Type hints are provided and verified with `basedpyright` (via `py.typed`). +- **Coding Style**: Follows the workspace-wide `ruff` configuration. diff --git a/cbscommon/pyproject.toml b/cbscommon/pyproject.toml new file mode 100644 index 00000000..f0359fdf --- /dev/null +++ b/cbscommon/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "cbscommon" +version = "0.1.0" +description = "Common utilities and shared modules for cbs" +readme = "README.md" +authors = [ + { name = "Uwe Schwaeke", email = "uwe.schwaeke@clyso.com" } +] +license = "GPL-3.0-or-later" +requires-python = ">=3.13" +dependencies = [] + +[build-system] +requires = ["uv_build>=0.10.2,<0.11.0"] +build-backend = "uv_build" diff --git a/cbscommon/src/cbscommon/__init__.py b/cbscommon/src/cbscommon/__init__.py new file mode 100644 index 00000000..ae88f92e --- /dev/null +++ b/cbscommon/src/cbscommon/__init__.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + +import logging + +logger = logging.getLogger("cbscommon") +logger.setLevel(logging.ERROR) + + +def logger_set_handler(handler: logging.Handler) -> None: + logger.propagate = False + logger.addHandler(handler) diff --git a/cbscommon/src/cbscommon/exceptions.py b/cbscommon/src/cbscommon/exceptions.py new file mode 100644 index 00000000..896c2f53 --- /dev/null +++ b/cbscommon/src/cbscommon/exceptions.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + + +import errno +from typing import override + + +class CBSCommonError(Exception): + msg: str | None + ec: int | None + + def __init__(self, msg: str | None = None, *, ec: int | None = None): + super().__init__() + self.msg = msg + self.ec = ec + + @override + def __str__(self) -> str: + ec_name = ( + errno.errorcode[self.ec] if self.ec and self.ec in errno.errorcode else None + ) + return ( + "CBS error" + + (f" ({ec_name})" if ec_name else "") + + (f": {self.msg}" if self.msg else "") + ) + + def with_maybe_msg(self, prefix: str) -> str: + return prefix + f": {self.msg}" if self.msg else "" diff --git a/cbscommon/src/cbscommon/git/__init__.py b/cbscommon/src/cbscommon/git/__init__.py new file mode 100644 index 00000000..01fb78ab --- /dev/null +++ b/cbscommon/src/cbscommon/git/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + +from cbscommon import logger as parent_logger + +logger = parent_logger.getChild("git") diff --git a/cbscommon/src/cbscommon/git/cmds.py b/cbscommon/src/cbscommon/git/cmds.py new file mode 100644 index 00000000..ca97e793 --- /dev/null +++ b/cbscommon/src/cbscommon/git/cmds.py @@ -0,0 +1,1209 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH +import errno +import re +import secrets +import shutil +import tempfile +from pathlib import Path +from typing import IO, Any, cast + +from cbscommon.process.cmds import async_run_cmd, sanitize_cmd +from cbscommon.process.types import AsyncRunCmdOutCallback, CmdArgs, MaybeSecure + +from . import logger +from .exceptions import ( + GitAMApplyError, + GitCherryPickConflictError, + GitCherryPickError, + GitConfigNotSetError, + GitCreateHeadExistsError, + GitEmptyPatchDiffError, + GitError, + GitFetchError, + GitFetchHeadNotFoundError, + GitHeadNotFoundError, + GitIsTagError, + GitMissingBranchError, + GitMissingRemoteError, + GitPatchDiffError, + GitPushError, +) +from .types import SHA, PushInfoLine, PushInfoLineStatus + + +async def git_reset_state(repo_path: Path, branch: str = "main"): + try: + await git_cleanup_repo(repo_path) + await git_switch(repo_path, branch, discard_changes=True) + + except GitError as e: + msg = ( + f"unable to reset state of repository {repo_path} to branch {branch}: '{e}'" + ) + logger.error(msg) + raise GitError(msg) from None + + +async def git_check_patches_diff( + ceph_git_path: Path, + upstream_ref: str | SHA, + head_ref: str | SHA, + *, + limit: str | SHA | None = None, +) -> tuple[list[str], list[str]]: + logger.debug( + f"check ref '{head_ref}' against upstream '{upstream_ref}', limit '{limit}'" + ) + + cmd: CmdArgs = ["cherry", upstream_ref, head_ref] + if limit: + cmd.append(limit) + + try: + res = await _run_git(cmd, path=ceph_git_path) + except GitError as e: + msg = ( + f"unable to check patch diff between '{upstream_ref}' and '{head_ref}': {e}" + ) + logger.error(msg) + raise GitPatchDiffError(msg=msg) from None + + if not res: + logger.warning(f"empty diff between '{upstream_ref}' and '{head_ref}") + raise GitEmptyPatchDiffError() + + patches_res = res.splitlines() + patches_add: list[str] = [] + patches_drop: list[str] = [] + + entry_re = re.compile(r"^([-+])\s+(.*)$") + for entry in patches_res: + m = re.match(entry_re, entry) + if not m: + logger.error(f"unexpected entry format: {entry}") + continue + + action = cast(str, m.group(1)) + sha = cast(str, m.group(2)) + + match action: + case "+": + patches_add.append(sha) + case "-": + patches_drop.append(sha) + case _: + logger.error(f"unexpected patch action '{action}' for sha '{sha}'") + + logger.debug(f"ref '{head_ref}' add {patches_add}") + logger.debug(f"ref '{head_ref}' drop {patches_drop}") + + return (patches_add, patches_drop) + + +async def git_patches_in_interval( + repo_path: Path, from_ref: SHA, to_ref: SHA +) -> list[tuple[SHA, str]]: + logger.debug(f"get patch interval from '{from_ref}' to '{to_ref}'") + + cmd: CmdArgs = [ + "rev-list", + "--ancestry-path", + "--pretty=oneline", + f"{from_ref}~1..{to_ref}", + ] + try: + res = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = f"unable to obtain patch interval: {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + def _split(ln: str) -> tuple[str, str]: + sha, title = ln.split(maxsplit=1) + return (sha, title) + + return list( + map(_split, [line.strip() for line in res.splitlines() if line.strip()]) + ) + + +async def git_get_patch_sha_title(repo_path: Path, sha: SHA) -> tuple[str, str]: + logger.debug(f"get patch sha and title for '{sha}'") + + try: + res = await _git_show(repo_path, sha, format="%H %s", no_patch=True) + except GitError as e: + msg = f"unable to obtain patch sha and title for '{sha}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + logger.debug(res) + lst = [line.strip() for line in res.splitlines() if line.strip()] + if len(lst) > 1: + raise GitError(msg=f"unexpected multiple lines for patch '{sha}'") + logger.debug(lst) + patch_sha, patch_title = next(iter(lst)).split(maxsplit=1) + return (patch_sha, patch_title) + + +async def git_status(repo_path: Path) -> list[tuple[str, str]]: + cmd: CmdArgs = ["status", "--porcelain"] + try: + res = await _run_git(cmd, path=repo_path) + except GitError: + msg = f"unable to run git status on '{repo_path}'" + logger.error(msg) + raise GitError(msg=msg) from None + + status_lst: list[tuple[str, str]] = [] + for entry in res.splitlines(): + status, file = entry.split(maxsplit=1) + status_lst.append((status, file)) + + return status_lst + + +async def git_am_apply(repo_path: Path, patch_path: Path) -> None: + try: + _ = await _git_am(repo_path, patch_path) + except GitError: + msg = f"unable to apply patch '{patch_path}'" + logger.error(msg) + raise GitAMApplyError(msg=msg) from None + + +async def git_am_abort(repo_path: Path) -> None: + try: + _ = await _git_am(repo_path, abort=True) + except GitError: + logger.error("found error aborting git-am") + + +async def git_cleanup_repo(repo_path: Path) -> None: + try: + _ = await _git_submodule(repo_path, "deinit", all=True, force=True) + cmd: CmdArgs = ["clean", "-ffdx"] + _ = await _run_git(cmd, path=repo_path) + _ = await _git_reset(repo_path, hard=True) + except GitError as e: + msg = f"unable to clean up repository '{repo_path}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + +async def git_prepare_remote( + repo_path: Path, remote_uri: str, remote_name: str, token: str +) -> None: + logger.info(f"prepare remote '{remote_name}' uri '{remote_uri}'") + remote_url = f"https://crt:{token}@{remote_uri}" + + try: + _ = await _git_remote(repo_path, "add", remote_name, remote_url) + except GitError as e: + # git remote add returns ESRCH (3), if remote_name already exists. + if e.ec != errno.ESRCH: + msg = f"error occured during git remote add: {e}" + logger.error(msg) + raise GitError(msg=msg, ec=e.ec) from e + else: + logger.debug(f"created remote '{remote_name}' url '{remote_url}'") + + logger.info(f"update remote '{remote_name}'") + try: + _ = await _git_remote(repo_path, "update", remote_name) + except GitError: + logger.error(f"unable to update remote '{remote_name}'") + raise GitError(msg=f"unable to update remote '{remote_name}'") from None + + +async def git_remote_exists(repo_path: Path, remote_name: str) -> bool: + logger.info(f"does remote '{remote_name}' exist.") + res = await _git_remote(repo_path) + return remote_name in res.splitlines() + + +def _get_remote_ref_name( + remote_name: str, remote_ref: str, *, ref_name: str | None = None +) -> tuple[str, str] | None: + ref_re = re.compile(rf"^{remote_name}/(.*)$") + if m := re.match(ref_re, remote_ref): + name = cast(str, m.group(1)) + if ref_name and ref_name != name: + return None + + return (remote_name, m.group(1)) + return None + + +async def git_remote_ref_exists( + repo_path: Path, ref_name: str, remote_name: str +) -> bool: + if not await git_remote_exists(repo_path, remote_name): + logger.error(f"remote '{remote_name}' not found") + raise GitMissingRemoteError(remote_name) from None + + res = await _git_branch(repo_path, remote=True, list=f"{remote_name}/*") + for ref in res.splitlines(): + ref = ref.strip() + if _get_remote_ref_name(remote_name, ref, ref_name=ref_name): + return True + + return False + + +async def _git_pull_ref( + repo_path: Path, from_ref: str, to_ref: str, remote_name: str +) -> bool: + active_branch = await _git_branch(repo_path, show_current=True) + if active_branch != to_ref: + return False + + if not await git_remote_ref_exists(repo_path, from_ref, remote_name): + logger.warning(f"ref '{from_ref}' not found in remote '{remote_name}'") + return False + + cmd: CmdArgs = ["pull", remote_name, f"{from_ref}:{to_ref}"] + try: + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + logger.error( + f"unable to pull from '{remote_name}' ref '{from_ref}' to '{to_ref}'" + ) + logger.error(e.msg) + raise GitFetchError(remote_name, from_ref, to_ref) from None + + return True + + +async def _is_tag(repo_path: Path, tag_name: str) -> bool: + res = await _git_tag(repo_path, list=tag_name) + return bool(res.strip()) + + +async def git_branch_from(repo_path: Path, src_ref: str, dst_branch: str) -> None: + """Create a new branch `dst_branch` from `src_ref`.""" + logger.debug(f"create branch '{dst_branch}' from '{src_ref}'") + active_branch = await _git_branch(repo_path, show_current=True) + logger.debug(f"repo active branch: {active_branch}") + + if await git_local_head_exists(repo_path, dst_branch): + msg = f"unable to create branch '{dst_branch}', already exists" + logger.error(msg) + raise GitCreateHeadExistsError(dst_branch) + + if await _is_tag(repo_path, src_ref): + logger.debug(f"source ref '{src_ref}' is a tag") + src_ref = f"refs/tags/{src_ref}" + + try: + _ = await _git_branch(repo_path, dst_branch, src_ref) + except GitError as e: + msg = f"unable to create branch '{dst_branch}' from '{src_ref}': {e}" + logger.error(msg) + logger.error(e.msg) + raise GitError(msg) from None + + +async def git_fetch_ref( + repo_path: Path, from_ref: str, to_ref: str, remote_name: str +) -> bool: + """ + Fetch a reference from a remote into a given branch. + + If the target branch is already checked out, perform a `git pull` instead. + If the source ref is a tag, do not fetch. + + Will raise if `from_ref` is a tag, or if it doesn't exist in the specified remote. + Might raise in other `git fetch` error conditions. + """ + logger.debug(f"fetch from '{remote_name}' ref '{from_ref}' to '{to_ref}'") + active_branch = await _git_branch(repo_path, show_current=True) + logger.debug(f"repo active branch: {active_branch}") + + if active_branch == to_ref: + logger.warning(f"checked out branch is '{to_ref}', pull instead.") + return await _git_pull_ref(repo_path, from_ref, to_ref, remote_name) + + # check whether 'from_ref' is a tag + if await _is_tag(repo_path, from_ref): + logger.warning(f"can't fetch tag '{from_ref}' from remote '{remote_name}'") + raise GitIsTagError(from_ref) + + # check whether 'from_ref' is a remote head + if not await git_remote_ref_exists(repo_path, from_ref, remote_name): + logger.warning(f"unable to find ref '{from_ref}' in remote '{remote_name}'") + raise GitFetchHeadNotFoundError(remote_name, from_ref) + + if not await git_remote_exists(repo_path, remote_name): + msg = f"unexpected error obtaining remote '{remote_name}'" + logger.error(msg) + raise GitError(msg) from None + + try: + cmd: CmdArgs = ["fetch", remote_name, f"{from_ref}:{to_ref}"] + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + logger.error( + f"unable to fetch from remote '{remote_name}' " + + f"ref '{from_ref}' to '{to_ref}'" + ) + logger.error(e.msg) + raise GitFetchError(remote_name, from_ref, to_ref) from None + + return True + + +async def git_checkout( + repo_path: Path, + ref: str, + *, + to_branch: str | None = None, + remote_name: str | None = None, + update_from_remote: bool = False, + fetch_if_not_exists: bool = False, + clean: bool = False, + update_submodules: bool = False, +) -> None: + """ + Check out a reference, possibly to a new branch. + + 1. Resolves target branch. + 2. Handles remote fetching/updating if requested. + 3. Creates local branch from ref if it doesn't exist. + 4. Switches to the branch. + 5. Optionally cleans the repo and updates submodules. + """ + target_branch = to_branch if to_branch else ref + + # check if 'target_branch' exists as a branch locally + exists_locally = await git_local_head_exists(repo_path, target_branch) + + if exists_locally: + if update_from_remote and remote_name: + logger.debug(f"update '{target_branch}' from remote '{remote_name}'") + try: + _ = await git_fetch_ref( + repo_path, target_branch, target_branch, remote_name + ) + except Exception as e: + logger.error( + f"unable to update '{target_branch}' " + + f"from remote '{remote_name}': {e}" + ) + else: + # local head does not exist + if fetch_if_not_exists: + if not remote_name: + msg = f"unable to fetch ref '{ref}', no remote given" + logger.error(msg) + raise GitError(msg) from None + + is_tag = False + try: + _ = await git_fetch_ref(repo_path, ref, target_branch, remote_name) + except GitIsTagError: + logger.debug(f"ref '{ref}' is a tag, must checkout instead.") + is_tag = True + except GitFetchHeadNotFoundError as e: + logger.error(f"ref '{ref}' not found in remote.") + raise e from None + except GitError as e: + logger.error(f"error occurred fetching ref '{ref}': {e}") + raise e from None + + if is_tag: + try: + cmd: CmdArgs = ["checkout", ref, "-b", target_branch] + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = f"unable to checkout ref '{ref}' to '{target_branch}': {e}" + logger.error(msg) + logger.error(e.msg) + raise GitError(msg) from None + # If it was a tag and we checked it out, switch is done. + else: + # Not fetching. Maybe create from local ref? + if await git_local_head_exists(repo_path, ref) or await _is_tag( + repo_path, ref + ): + await git_branch_from(repo_path, ref, target_branch) + else: + logger.debug(f"not fetching '{ref}' as specified") + raise GitMissingBranchError(ref) + + # Perform the switch (if not already handled by manual tag checkout) + await git_switch(repo_path, target_branch, discard_changes=True) + + if clean: + await git_cleanup_repo(repo_path) + + if update_submodules: + await git_update_submodules(repo_path) + + +async def git_worktree(repo_path: Path, ref: str, worktrees_base_path: Path) -> Path: + """ + Checkout a reference pointed to by `ref`, in repository `repo_path`. + + Uses git worktrees to checkout the reference into a new worktree under + `worktrees_base_path`. + + Returns the path to the checked out worktree. + """ + try: + worktrees_base_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + msg = f"unable to create worktrees base path at '{worktrees_base_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + worktree_rnd_suffix = secrets.token_hex(5) + worktree_name = ref.replace("/", "--") + f".{worktree_rnd_suffix}" + worktree_path = worktrees_base_path / worktree_name + logger.info(f"checkout ref '{ref}' into worktree at '{worktree_path}'") + + try: + _ = await _run_git( + [ + "worktree", + "add", + "--track", + "-b", + worktree_name, + "--quiet", + worktree_path.resolve().as_posix(), + ref, + ], + path=repo_path, + ) + except GitError as e: + msg = f"unable to checkout ref '{ref}' in repository '{repo_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + return worktree_path + + +async def git_branch_delete(repo_path: Path, branch: str) -> None: + """Delete a local branch.""" + active_branch = await _git_branch(repo_path, show_current=True) + if active_branch == branch: + await git_reset_state(repo_path) + _ = await _git_branch(repo_path, branch_name=branch, delete=True, force=True) + + +async def git_push( + repo_path: Path, + ref: str, + remote_name: str, + *, + ref_to: str | None = None, +) -> tuple[bool, list[str], list[str]]: + """Pushes either a local head of branch or a local tag to the remote.""" + dst_ref = ref_to if ref_to else ref + + if await _is_tag(repo_path, ref): + ref = f"refs/tags/{ref}" + dst_ref = f"refs/tags/{dst_ref}" + elif not await git_local_head_exists(repo_path, ref): + # ref is neither a local branch nor tag + logger.error(f"unable to find ref '{ref}' to push") + raise GitHeadNotFoundError(ref) + + if not await git_remote_exists(repo_path, remote_name): + logger.error(f"unable to find remote '{remote_name}'") + raise GitMissingRemoteError(remote_name) from None + + try: + info = await _git_push( + repo_path, remote_name, f"{ref}:{dst_ref}", porcelain=True + ) + # skip first line because it is To remote url + # and skip last line because it is Done + info = info.splitlines()[1:-1] + info = [PushInfoLine(line) for line in info] + except GitError as e: + msg = f"unable to push '{ref}' to '{dst_ref}': {e}" + logger.error(msg) + logger.error(e.msg) + raise GitPushError(ref, dst_ref, remote_name) from None + + updated: list[str] = [] + rejected: list[str] = [] + failed = len(info) == 0 + + for entry in info: + logger.debug(f"entry '{entry.remote_ref_name}' flags '{entry.flag}'") + if entry.status == PushInfoLineStatus.REJECTED: + logger.debug(f"rejected head: {entry.remote_ref_name}") + rejected.append(entry.remote_ref_name) + elif entry.status == PushInfoLineStatus.UPDATED: + logger.debug(f"updated head: {entry.remote_ref_name}") + updated.append(entry.remote_ref_name) + + return (failed, updated, rejected) + + +async def git_tag( + repo_path: Path, + tag_name: str, + ref: str, + *, + msg: str | None = None, + push_to: str | None = None, +) -> None: + logger.debug(f"create tag '{tag_name}' at ref '{ref}'") + try: + _ = await _git_tag(repo_path, tag_name, ref, m=msg) + except GitError as e: + msg = f"unable to create tag '{tag_name}' at ref '{ref}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + if push_to: + logger.debug(f"push tag '{tag_name}' to remote '{push_to}'") + try: + _ = await _git_push(repo_path, push_to, tag=tag_name) + except GitError as e: + msg = f"unable to push tag '{tag_name}' to remote '{push_to}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + +async def git_patch_id(repo_path: Path, sha: SHA) -> str: + with tempfile.TemporaryFile() as tmp: + + async def _write_to_tmp(line: str): + _ = tmp.write(line.encode("utf-8")) + _ = tmp.write(b"\n") + + try: + _ = await _git_show(repo_path, sha, outcb=_write_to_tmp) + except GitError: + msg = f"unable to find patch sha '{sha}'" + logger.error(msg) + raise GitError(msg=msg) from None + + _ = tmp.seek(0) + cmd: CmdArgs = ["patch-id", "--stable"] + res = await _run_git(cmd, path=repo_path, stdin=tmp) + + if not res: + raise GitError(msg="unable to obtain git patch id") + return res.split()[0] + + +async def git_revparse(repo_path: Path, commitish: SHA | str) -> str: + try: + res = await _git_rev_parse(repo_path, commitish) + except GitError as e: + msg = f"unable to obtain revision for '{commitish}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + return res + + +async def git_format_patch( + repo_path: Path, rev: SHA, *, base_rev: SHA | None = None +) -> str: + cmd: CmdArgs = ["format-patch", "--stdout"] + + if not base_rev: + cmd.append("-1") + + rev_str = f"{base_rev}..{rev}" if base_rev else rev + cmd.append(rev_str) + + try: + res = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = f"unable to obtain format patch for '{rev_str}': {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + return res + + +async def git_tag_exists_in_remote( + repo_path: Path, remote_name: str, tag_name: str +) -> bool: + cmd: CmdArgs = ["ls-remote", "--tags", remote_name, f"refs/tags/{tag_name}"] + try: + raw_tag = await _run_git(cmd, path=repo_path) + return bool(raw_tag.strip()) + except GitError as e: + msg = ( + f"unable to execute git ls-remote --tags {remote_name} " + f"refs/tags/{tag_name}: {e}" + ) + logger.error(msg) + raise GitError(msg) from None + + +async def git_remote_ref_names(repo_path: Path, remote_name: str) -> list[str]: + try: + res = await _git_branch( + repo_path, remote=True, list=remote_name, format="%(refname)" + ) + lines = res.splitlines() + return [line.removeprefix("refs/remotes/") for line in lines] + except GitError as e: + msg = f"unable to list remote names of remote '{remote_name}': '{e}'" + logger.error(msg) + raise GitError(msg) from None + + +async def git_update_submodules(repo_path: Path) -> None: + logger.debug("update submodules") + try: + _ = await _git_submodule(repo_path, "update", init=True, recursive=True) + except GitError as e: + msg = f"unable to update repository's submodules: {e}" + logger.error(msg) + raise GitError(msg=msg) from None + + +async def git_local_head_exists(repo_path: Path, name: str) -> bool: + res = await _git_branch(repo_path, list=name) + return bool(res.strip()) + + +async def _run_git( + args: CmdArgs, + *, + path: Path | None = None, + outcb: AsyncRunCmdOutCallback | None = None, + stdin: int | IO[Any] | None = None, # pyright: ignore[reportExplicitAny] +) -> str: + """ + Run a git command within the repository. + + If `path` is provided, run the command in `path`. Otherwise, run in the current + directory. + """ + cmd: CmdArgs = ["git"] + if path is not None: + cmd.extend(["-C", path.resolve().as_posix()]) + + cmd.extend(args) + logger.debug(f"run {sanitize_cmd(cmd)}") + try: + rc, stdout, stderr = await async_run_cmd(cmd, outcb=outcb, stdin=stdin) + except Exception as e: + msg = f"unexpected error running command: {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + if rc != 0: + logger.error( + f"unable to obtain result from git '{sanitize_cmd(args)}': {stderr}" + ) + raise GitError(stderr, ec=rc) + + return stdout + + +async def get_git_user(repo_path: Path | None = None) -> tuple[str, str]: + """Obtain the current repository's git user and email, returned as a tuple.""" + + async def _run_git_config_for(v: str) -> str: + cmd: CmdArgs = ["config", v] + val = await _run_git(cmd, path=repo_path) + if len(val) == 0: + logger.error(f"'{v}' not set in git config") + raise GitConfigNotSetError(v) + + return val.strip() + + user_name = await _run_git_config_for("user.name") + user_email = await _run_git_config_for("user.email") + + return (user_name, user_email) + + +async def get_git_repo_root() -> Path: + """Obtain the root of the current git repository.""" + val = await _git_rev_parse(show_toplevel=True) + if len(val) == 0: + logger.error("unable to obtain toplevel git directory path") + raise GitError("top-level git directory not found", ec=errno.ENOENT) + + return Path(val.strip()) + + +async def _clone(repo: MaybeSecure, dest_path: Path) -> None: + """Clones a repository from `repo` to `dest_path`.""" + try: + cmd: CmdArgs = [ + "clone", + "--mirror", + "--quiet", + repo, + dest_path.resolve().as_posix(), + ] + _ = await _run_git(cmd) + except GitError as e: + msg = f"unable to clone '{repo}' to '{dest_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + +async def _update(repo: MaybeSecure, repo_path: Path) -> None: + """Update a git repository in `repo_path` from its upstream at `repo`.""" + try: + _ = await _git_remote(repo_path, "set-url", "origin", repo) + _ = await _git_remote(repo_path, "update") + except GitError as e: + msg = f"unable to update '{repo_path}': {e}'" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + +async def git_remove_worktree(repo_path: Path, worktree_path: Path) -> None: + """Remove a git worktree at `worktree_path` from repository `repo_path`.""" + logger.info(f"removing worktree at '{worktree_path}' from repository '{repo_path}'") + try: + _ = await _run_git( + ["worktree", "remove", "--force", worktree_path.resolve().as_posix()], + path=repo_path, + ) + except GitError as e: + msg = f"unable to remove worktree at '{worktree_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + +async def git_clone(repo: MaybeSecure, base_path: Path, repo_name: str) -> Path: + """ + Clone a mirror git repository if it doesn't exist; update otherwise. + + Clone a git repository from `repo` to a directory `repo_name` in `base_path`. If a + git repository exists at the destination, ensure the repository is updated instead. + + Returns the path to the repository. + """ + logger.info(f"cloning '{repo}' to new destination '{base_path}' name '{repo_name}'") + if not base_path.exists(): + logger.warning(f"base path at '{base_path}' does not exist -- creating") + try: + base_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + msg = f"unable to create base path at '{base_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + dest_path = base_path / f"{repo_name}.git" + + if dest_path.exists(): + if not dest_path.is_dir() or not dest_path.joinpath("HEAD").exists(): + logger.warning( + f"destination path at '{dest_path}' exists, " + + "but is not a valid git repository -- nuke it!" + ) + try: + shutil.rmtree(dest_path) + except Exception as e: + msg = f"unable to remove invalid git repository at '{dest_path}': {e}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + # propagate exception to caller + await _update(repo, dest_path) + return dest_path + + # propagate exception to caller + await _clone(repo, dest_path) + return dest_path + + +async def git_apply(repo_path: Path, patch_path: Path) -> None: + """Apply a patch onto the repository specified by `repo_path`.""" + try: + _ = await _run_git(["apply", patch_path.resolve().as_posix()], path=repo_path) + except GitError as e: + msg = f"error applying patch '{patch_path}' to '{repo_path}': {e}" + logger.exception(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + pass + + +async def git_get_sha1(repo_path: Path) -> str: + """For the repository in `repo_path`, obtain its currently checked out SHA1.""" + val = await _git_rev_parse(repo_path, "HEAD") + if len(val) == 0: + msg = f"unable to obtain current SHA1 on repository '{repo_path}" + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) + + return val.strip() + + +async def _git_show( + repo_path: Path, + object: SHA, + *, + format: str | None = None, + no_patch: bool = False, + outcb: AsyncRunCmdOutCallback | None = None, +) -> str: + cmd: CmdArgs = ["show", object] + if format: + cmd.append(f"--format={format}") + if no_patch: + cmd.append("--no-patch") + + return await _run_git(cmd, path=repo_path, outcb=outcb) + + +async def _git_am( + repo_path: Path, mbox: Path | None = None, *, abort: bool = False +) -> str: + cmd: CmdArgs = ["am"] + if abort: + cmd.append("--abort") + if mbox: + cmd.append(str(mbox)) + return await _run_git(cmd, path=repo_path) + + +async def _git_submodule( + repo_path: Path, + verb: str, + *, + init: bool = False, + recursive: bool = False, + all: bool = False, + force: bool = False, +) -> str: + cmd: CmdArgs = ["submodule", verb] + if init: + cmd.append("--init") + if recursive: + cmd.append("--recursive") + if all: + cmd.append("--all") + if force: + cmd.append("--force") + return await _run_git(cmd, path=repo_path) + + +async def _git_reset( + repo_path: Path, commit: str | None = None, *, hard: bool = False +) -> str: + cmd: CmdArgs = ["reset"] + if hard: + cmd.append("--hard") + if commit: + cmd.append(commit) + return await _run_git(cmd, path=repo_path) + + +async def _git_remote( + repo_path: Path, + verb: str | None = None, + name: str | None = None, + url: MaybeSecure | None = None, +) -> str: + cmd: CmdArgs = ["remote"] + if verb: + cmd.append(verb) + if name: + cmd.append(name) + if url: + cmd.append(url) + return await _run_git(cmd, path=repo_path) + + +async def _git_branch( + repo_path: Path, + branch_name: str | None = None, + start_point: str | None = None, + *, + remote: bool = False, + list: str | None = None, + show_current: bool = False, + delete: bool = False, + force: bool = False, + format: str | None = None, +) -> str: + cmd: CmdArgs = ["branch"] + if branch_name: + cmd.append(branch_name) + if start_point: + cmd.append(start_point) + if remote: + cmd.append("--remotes") + if list: + cmd.append("--list") + cmd.append(list) + if show_current: + cmd.append("--show-current") + if delete: + cmd.append("--delete") + if force: + cmd.append("--force") + if format: + cmd.append("--format") + cmd.append(format) + + return await _run_git(cmd, path=repo_path) + + +async def _git_tag( + repo_path: Path, + tagname: str | None = None, + commit: str | None = None, + *, + list: str | None = None, + m: str | None = None, +) -> str: + cmd: CmdArgs = ["tag"] + if tagname: + cmd.append(tagname) + if commit: + cmd.append(commit) + if list: + cmd.append("--list") + cmd.append(list) + if m: + cmd.append("-m") + cmd.append(m) + + return await _run_git(cmd, path=repo_path) + + +async def git_switch(repo_path: Path, branch: str, *, discard_changes: bool = False): + cmd: CmdArgs = ["switch", branch] + if discard_changes: + cmd.append("--discard-changes") + try: + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = ( + f"unable to switch to branch '{branch}' with discard-changes=" + f"{discard_changes}: {e}" + ) + logger.error(msg) + raise GitError(msg) from None + + +async def _git_push( + repo_path: Path, + repository: str, + refspec: str | None = None, + *, + tag: str | None = None, + porcelain: bool = False, +) -> str: + cmd: CmdArgs = ["push", repository] + if refspec: + cmd.append(refspec) + if tag: + cmd.append("tag") + cmd.append(tag) + if porcelain: + cmd.append("--porcelain") + + return await _run_git(cmd, path=repo_path) + + +async def _git_rev_parse( + repo_path: Path | None = None, + arg: SHA | str | None = None, + *, + show_toplevel: bool = False, + abbrev_ref: bool = False, +) -> str: + cmd: CmdArgs = ["rev-parse"] + if arg: + cmd.append(arg) + if show_toplevel: + cmd.append("--show-toplevel") + if abbrev_ref: + cmd.append("--abbrev-ref") + + return await _run_git(cmd, path=repo_path) + + +# unused functions + + +async def _git_cherry_pick(repo_path: Path, sha: SHA) -> None: # pyright: ignore[reportUnusedFunction, reportRedeclaration] + cmd: CmdArgs = ["cherry-pick", "-x", "-s", sha] + try: + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = f"unable to cherry-pick patch sha '{sha}'" + logger.error(msg) + + status_files = await git_status(repo_path) + conflicts: list[str] = [f for s, f in status_files if s == "UU"] + + if conflicts: + raise GitCherryPickConflictError(sha, conflicts) from None + + logger.error(e.msg) + raise GitCherryPickError(msg=msg) from None + + +async def _git_abort_cherry_pick(repo_path: Path) -> None: # pyright: ignore[reportUnusedFunction] + cmd: CmdArgs = ["cherry-pick", "--abort"] + try: + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + logger.error(f"found error aborting cherry-pick: {e.msg}") + + +async def _get_git_modified_paths( # pyright: ignore[reportUnusedFunction] + base_sha: str, + ref: str, + *, + in_repo_path: str | None = None, + repo_path: Path | None = None, +) -> tuple[list[Path], list[Path]]: + """ + Obtain all modifications since `ref` on the repository. + + If `path` is specified, perform the action within the context of `path`. Otherwise, + on the git repository existing in current directory. + """ + try: + cmd: CmdArgs = [ + "diff-tree", + "--diff-filter=ACDMR", + "--ignore-all-space", + "--no-commit-id", + "--name-status", + "-r", + base_sha, + ref, + ] + + if in_repo_path: + cmd.extend(["--", in_repo_path]) + + val = await _run_git(cmd, path=repo_path) + except GitError as e: + logger.error(f"error: unable to obtain latest patch: {e}") + raise GitError( + f"unable to obtain patches between {base_sha} and {ref}", + ec=errno.ENOTRECOVERABLE, + ) from e + + if len(val) == 0: + logger.debug(f"no relevant patches found between {base_sha} and {ref}") + return [], [] + + descs_deleted: list[Path] = [] + descs_modified: list[Path] = [] + + lines = val.splitlines() + regex = ( + re.compile(rf"^\s*([ACDMR])\s+({repo_path}.*)\s*$") + if repo_path is not None + else re.compile(r"\s*([ACDMR])\s+([^\s]+)\s*$") + ) + for line in lines: + m = re.match(regex, line) + if m is None: + logger.debug(f"'{line}' does not match") + continue + + action = m.group(1) + target = m.group(2) + logger.debug(f"action: {action}, target: {target}") + + match action: + case "D": + descs_deleted.append(Path(target)) + case "A" | "C" | "M" | "R": + descs_modified.append(Path(target)) + case _: + logger.error( + f"unexpected action '{action}' on '{target}', line: '{line}'" + ) + raise GitError( + f"unexpected action '{action}' on '{target}'", + ec=errno.ENOTRECOVERABLE, + ) + + return descs_modified, descs_deleted + + +async def _git_pull( # pyright: ignore[reportUnusedFunction] + remote: MaybeSecure, + *, + from_branch: str | None = None, + to_branch: str | None = None, + repo_path: Path | None = None, +) -> None: + """Pull commits from `remote`.""" + logger.debug(f"Pull from '{remote}' (from: {from_branch}, to: {to_branch})") + try: + cmd: CmdArgs = ["pull", remote] + branches: str | None = None + if from_branch: + branches = from_branch + if to_branch: + branches = f"{branches}:{to_branch}" + if branches: + cmd.append(branches) + _ = await _run_git(cmd, path=repo_path) + except GitError as e: + msg = f"unable to pull from '{remote}': {e}" + logger.exception(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + +async def _git_cherry_pick( # pyright: ignore[reportUnusedFunction] + sha: str, *, sha_end: str | None = None, repo_path: Path | None = None +) -> None: + """ + Cherry-picks a given SHA to the currently checked out branch. + + If `sha_end` is provided, will cherry-pick the patches `[sha~1, sha_end]`. + If `repo_path` is provided, run the command in said repository; otherwise, run + in the current directory. + """ + commit_to_pick = sha if not sha_end else f"{sha}~1..{sha_end}" + logger.debug(f"cherry-pick commit '{commit_to_pick}'") + try: + _ = await _run_git(["cherry-pick", "-x", commit_to_pick], path=repo_path) + except GitError as e: + msg = f"unable to cherry-pick '{commit_to_pick}': {e}" + logger.exception(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e + + +async def _git_get_current_branch(repo_path: Path) -> str: # pyright: ignore[reportUnusedFunction] + """Obtain the name of the currently checked out branch.""" + val = await _git_rev_parse(repo_path, "HEAD", abbrev_ref=True) + if not val: + msg = ( + "unable to obtain current checked out branch's " + + f"name on repository '{repo_path}'" + ) + logger.error(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) + + return val.strip() + + +async def _git_fetch( # pyright: ignore[reportUnusedFunction] + remote: str, from_ref: str, to_branch: str, *, repo_path: Path | None = None +) -> None: + """ + Fetch a reference from a remote to a new branch. + + Fetches the reference pointed to by `from_ref` from remote `remote` to a new branch + `to_branch`. If `repo_path` is specified, run the command in said path; otherwise, + run in current directory. + """ + logger.debug(f"fetch from '{remote}', source: {from_ref}, dest: {to_branch}") + try: + _ = await _run_git(["fetch", remote, f"{from_ref}:{to_branch}"], path=repo_path) + except GitError as e: + msg = f"unable to fetch '{from_ref}' from '{remote}' to '{to_branch}': {e}" + logger.exception(msg) + raise GitError(msg, ec=errno.ENOTRECOVERABLE) from e diff --git a/cbscommon/src/cbscommon/git/exceptions.py b/cbscommon/src/cbscommon/git/exceptions.py new file mode 100644 index 00000000..169d099e --- /dev/null +++ b/cbscommon/src/cbscommon/git/exceptions.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + + +import errno +from typing import override + +from ..exceptions import CBSCommonError +from .types import SHA + + +class GitError(CBSCommonError): + @override + def __str__(self) -> str: + return "git error" + (f": {self.msg}" if self.msg else "") + + +class GitIsTagError(GitError): + def __init__(self, tag: str) -> None: + super().__init__(msg=tag) + + @override + def __str__(self) -> str: + return self.with_maybe_msg("found unexpected tag") + + +class GitPatchDiffError(GitError): + @override + def __str__(self) -> str: + return "patches diff error" + (f": {self.msg}" if self.msg else "") + + +class GitEmptyPatchDiffError(GitPatchDiffError): + pass + + +class GitCherryPickError(GitError): + @override + def __str__(self) -> str: + return "cherry-pick error" + (f": {self.msg}" if self.msg else "") + + +class GitCherryPickConflictError(GitCherryPickError): + sha: SHA + conflicts: list[str] + + def __init__(self, sha: SHA, files: list[str] | None = None) -> None: + super().__init__(msg=f"conflict occurred on sha '{sha}'") + self.sha = sha + self.conflicts = files if files else [] + + +class GitAMApplyError(GitError): + pass + + +class GitMissingRemoteError(GitError): + remote_name: str + + def __init__(self, remote_name: str) -> None: + super().__init__() + self.remote_name = remote_name + + @override + def __str__(self) -> str: + return f"missing remote '{self.remote_name}'" + + +class GitMissingBranchError(GitError): + def __init__(self, branch_name: str) -> None: + super().__init__(msg=branch_name) + + @override + def __str__(self) -> str: + return self.with_maybe_msg("missing branch") + + +class GitCreateHeadExistsError(GitError): + def __init__(self, name: str) -> None: + super().__init__(msg=name) + + @override + def __str__(self) -> str: + return self.with_maybe_msg("head already exists") + + +class GitHeadNotFoundError(GitError): + def __init__(self, name: str) -> None: + super().__init__(msg=name) + + @override + def __str__(self) -> str: + return self.with_maybe_msg("head not found") + + +class GitFetchError(GitError): + remote_name: str + from_ref: str + to_ref: str + + def __init__(self, remote_name: str, from_ref: str, to_ref: str) -> None: + super().__init__() + self.remote_name = remote_name + self.from_ref = from_ref + self.to_ref = to_ref + + @override + def __str__(self) -> str: + return ( + f"failed fetching from remote '{self.remote_name}': " + + f"from '{self.from_ref}' to '{self.to_ref}'" + ) + + +class GitFetchHeadNotFoundError(GitFetchError): + def __init__(self, remote_name: str, head: str) -> None: + super().__init__(remote_name, head, "") + + @override + def __str__(self) -> str: + return f"head '{self.from_ref}' not found in remote '{self.remote_name}'" + + +class GitPushError(GitError): + def __init__(self, branch: str, dst_branch: str, remote_name: str) -> None: + super().__init__( + msg=f"from branch '{branch}' to '{dst_branch}' on remote '{remote_name}'" + ) + + @override + def __str__(self) -> str: + return self.with_maybe_msg("unable to push") + + +class GitConfigNotSetError(GitError): + def __init__(self, what: str) -> None: + super().__init__(f"{what} not set in config", ec=errno.ENOENT) diff --git a/cbscommon/src/cbscommon/git/types.py b/cbscommon/src/cbscommon/git/types.py new file mode 100644 index 00000000..7c42d3ed --- /dev/null +++ b/cbscommon/src/cbscommon/git/types.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + + +import re +from enum import Enum +from typing import final + +SHA = str + + +class PushInfoLineStatus(Enum): + UPDATED = 1 + REJECTED = 2 + OTHER = 3 + + +@final +class PushInfoLine: + _NEW_HEAD = "*" + _FAST_FORWARD = " " + _ERROR = "!" + + def __init__(self, line: str) -> None: + self.status = PushInfoLineStatus.OTHER + # each line is in form: \t : \t () + flag, from_to, summary = line.split("\t", 2) + self.flag = flag + + # from git_utils only if new head or fast forward than its status is updated + if flag == self._NEW_HEAD or flag == self._FAST_FORWARD: + self.status = PushInfoLineStatus.UPDATED + + # change status to rejected if status flag is "!" + # or if summary contains [no match] see GitPython + if flag == self._ERROR: + self.status = PushInfoLineStatus.REJECTED + if summary.startswith("[") and "[no match]" in summary: + self.status = PushInfoLineStatus.REJECTED + + _, to_ref = from_to.split(":", 1) + # remove /refs/heads/ or /refs/tags/ from the beginning + self.remote_ref_name = re.sub("^/refs/(heads|tags)/", "", to_ref) diff --git a/cbscommon/src/cbscommon/process/__init__.py b/cbscommon/src/cbscommon/process/__init__.py new file mode 100644 index 00000000..197a3968 --- /dev/null +++ b/cbscommon/src/cbscommon/process/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + +from cbscommon import logger as parent_logger + +logger = parent_logger.getChild("process") diff --git a/cbscommon/src/cbscommon/process/cmds.py b/cbscommon/src/cbscommon/process/cmds.py new file mode 100644 index 00000000..ae8e9600 --- /dev/null +++ b/cbscommon/src/cbscommon/process/cmds.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + +import asyncio +import os +import re +import shutil +from asyncio.streams import StreamReader +from io import StringIO +from pathlib import Path +from typing import IO, Any + +from cbscommon.process.types import AsyncRunCmdOutCallback, CmdArgs, SecureArg + +from . import logger + + +def sanitize_cmd(cmd: CmdArgs) -> list[str]: + sub_pattern = re.compile(r"(.*)(?:(--pass(?:phrase)[\s=]+)[^\s]+)") + gh_token_pattern = re.compile("crt:[^@]*") + sanitized: list[str] = [] + next_secure = False + for c in cmd: + if isinstance(c, SecureArg): + sanitized.append(str(c)) + continue + + if c == "--passphrase" or c == "--pass": + next_secure = True + sanitized.append(c) + continue + + if next_secure: + sanitized.append("****") + next_secure = False + continue + + s = re.sub(sub_pattern, r"\1\2****", c) + s = re.sub(gh_token_pattern, "crt:token", s) + sanitized.append(s) + + return sanitized + + +def _reset_python_env(env: dict[str, str]) -> dict[str, str]: + logger.debug("reset python env for command") + + python3_loc = shutil.which("python3") + if not python3_loc: + print("python3 executable not found") + return env + + logger.debug(f"python3 location: {python3_loc}") + + python3_path = Path(python3_loc) + if python3_path.parent.full_match("/usr/bin"): + logger.debug("nothing to do to python3 path") + return env + + orig_path = env.get("PATH") + assert orig_path + paths = orig_path.split(":") + + new_paths: list[str] = [] + for p in paths: + if Path(p).full_match(python3_path.parent): + continue + new_paths.append(p) + + env["PATH"] = ":".join(new_paths) + return env + + +def get_unsecured_cmd(orig: CmdArgs) -> list[str]: + cmd: list[str] = [] + for c in orig: + if isinstance(c, SecureArg): + cmd.append(c.value) + else: + cmd.append(c) + return cmd + + +async def async_run_cmd( + cmd: CmdArgs, + *, + outcb: AsyncRunCmdOutCallback | None = None, + timeout: float | None = None, + cwd: Path | None = None, + reset_python_env: bool = False, + extra_env: dict[str, str] | None = None, + stdin: int | IO[Any] | None = None, # pyright: ignore[reportExplicitAny] +) -> tuple[int, str, str]: + logger.debug(f"async run '{sanitize_cmd(cmd)}'") + + env: dict[str, str] = os.environ.copy() + if reset_python_env: + env = _reset_python_env(env) + + if extra_env: + env.update(extra_env) + + logger.debug(f"run async subprocess, cwd: {cwd}, cmd: {cmd}") + p = await asyncio.create_subprocess_exec( + *(get_unsecured_cmd(cmd)), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + stdin=stdin, + cwd=cwd, + env=env, + ) + + async def read_stream(stream: StreamReader | None) -> str: + collected = StringIO() + + if not stream: + return "" + + async for line in stream: + ln = line.decode("utf-8") + if outcb: + await outcb(ln) + else: + _ = collected.write(ln) + + return collected.getvalue() + + async def monitor(): + return await asyncio.gather( + read_stream(p.stdout), + read_stream(p.stderr), + ) + + try: + retcode, (stdout, stderr) = await asyncio.wait_for( + asyncio.gather(p.wait(), monitor()), timeout=timeout + ) + except (TimeoutError, asyncio.CancelledError): + # FIXME: evaluate all callers for whether they are properly handling these + # exceptions, vs handling a 'CommandError'. Or implement specific + # 'CommandError' exceptions for timeout and cancellation. + logger.error("async subprocess timed out or was cancelled") + p.kill() + _ = await p.wait() + raise + + return retcode, stdout, stderr diff --git a/cbscommon/src/cbscommon/process/types.py b/cbscommon/src/cbscommon/process/types.py new file mode 100644 index 00000000..c4cf2385 --- /dev/null +++ b/cbscommon/src/cbscommon/process/types.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH + +import abc +from collections.abc import Callable, Coroutine +from typing import Any + +AsyncRunCmdOutCallback = Callable[[str], Coroutine[Any, Any, None]] # pyright: ignore[reportExplicitAny] + + +class SecureArg(abc.ABC): + @property + @abc.abstractmethod + def value(self) -> str: + pass + + +MaybeSecure = str | SecureArg +CmdArgs = list[MaybeSecure] diff --git a/cbscommon/src/cbscommon/py.typed b/cbscommon/src/cbscommon/py.typed new file mode 100644 index 00000000..d21c7651 --- /dev/null +++ b/cbscommon/src/cbscommon/py.typed @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (c) 2026 Clyso GmbH \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 95282d8a..404a270b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,14 @@ requires-python = ">=3.13" dependencies = [] [tool.uv.workspace] -members = ["cbscore", "cbsdcore", "cbsd", "cbc", "crt"] +members = [ + "cbscore", + "cbsdcore", + "cbsd", + "cbc", + "crt", + "cbscommon", +] [dependency-groups] dev = ["basedpyright==1.37.1", "lefthook==2.0.15", "ruff==0.14.13"] diff --git a/uv.lock b/uv.lock index 54e45094..30e39a33 100644 --- a/uv.lock +++ b/uv.lock @@ -9,6 +9,7 @@ resolution-markers = [ [manifest] members = [ "cbc", + "cbscommon", "cbscore", "cbsd", "cbsdcore", @@ -366,6 +367,11 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "pyinstaller", specifier = ">=6.18.0" }] +[[package]] +name = "cbscommon" +version = "0.1.0" +source = { editable = "cbscommon" } + [[package]] name = "cbscore" version = "1.0.1" @@ -373,6 +379,7 @@ source = { editable = "cbscore" } dependencies = [ { name = "aioboto3" }, { name = "aiofiles" }, + { name = "cbscommon" }, { name = "click" }, { name = "hvac" }, { name = "pydantic" }, @@ -384,6 +391,7 @@ dependencies = [ requires-dist = [ { name = "aioboto3", specifier = "==13.3.0" }, { name = "aiofiles", specifier = ">=25.1.0" }, + { name = "cbscommon", editable = "cbscommon" }, { name = "click", specifier = ">=8.1.8" }, { name = "hvac", specifier = ">=2.3.0" }, { name = "pydantic", specifier = ">=2.11.6" }, @@ -706,8 +714,8 @@ version = "0.1.0" source = { editable = "crt" } dependencies = [ { name = "boto3" }, + { name = "cbscommon" }, { name = "click" }, - { name = "gitpython" }, { name = "httpx" }, { name = "pydantic" }, { name = "rich" }, @@ -717,8 +725,8 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "boto3", specifier = "==1.35.81" }, + { name = "cbscommon", editable = "cbscommon" }, { name = "click", specifier = ">=8.1.8" }, - { name = "gitpython", specifier = ">=3.1.44" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "pydantic", specifier = ">=2.10.6" }, { name = "rich", specifier = ">=14.0.0" }, @@ -882,30 +890,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] -[[package]] -name = "gitdb" -version = "4.0.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "smmap" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, -] - -[[package]] -name = "gitpython" -version = "3.1.46" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "gitdb" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/b5/59d16470a1f0dfe8c793f9ef56fd3826093fc52b3bd96d6b9d6c26c7e27b/gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f", size = 215371, upload-time = "2026-01-01T15:37:32.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -1672,15 +1656,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "smmap" -version = "5.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, -] - [[package]] name = "starlette" version = "0.50.0" From ce63f00d1a771b2dc962aaea232053d72eb91479 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 7 Apr 2026 08:13:27 +0200 Subject: [PATCH 2/3] cbscore: migrate git and process utilities to cbscommon Migrate the git and process execution logic from `cbscore` to the shared `cbscommon` library to allow for better code reuse across the CBS workspace. - Remove `cbscore/src/cbscore/utils/git.py` and replace its usage with `cbscommon.git`. - Refactor `cbscore/src/cbscore/utils/__init__.py` to delegate core process execution, command sanitization, and secure argument handling to `cbscommon.process`. - Update all internal modules in `cbscore` to use the unified interfaces provided by `cbscommon`. - Add `cbscommon` as a workspace dependency in `cbscore/pyproject.toml`. This consolidation reduces duplication and ensures consistent behavior for git operations and asynchronous command execution across CBS tools. --- cbscore/pyproject.toml | 4 + cbscore/src/cbscore/builder/prepare.py | 33 +- cbscore/src/cbscore/builder/rpmbuild.py | 5 +- cbscore/src/cbscore/builder/signing.py | 5 +- cbscore/src/cbscore/builder/upload.py | 4 +- cbscore/src/cbscore/builder/utils.py | 5 +- cbscore/src/cbscore/cmds/__init__.py | 2 + cbscore/src/cbscore/cmds/versions.py | 3 +- cbscore/src/cbscore/images/desc.py | 2 +- cbscore/src/cbscore/images/signing.py | 5 +- cbscore/src/cbscore/images/skopeo.py | 3 +- cbscore/src/cbscore/logger.py | 5 + cbscore/src/cbscore/releases/utils.py | 5 +- cbscore/src/cbscore/runner.py | 2 +- cbscore/src/cbscore/utils/__init__.py | 167 +--------- cbscore/src/cbscore/utils/buildah.py | 11 +- cbscore/src/cbscore/utils/git.py | 401 ----------------------- cbscore/src/cbscore/utils/podman.py | 4 +- cbscore/src/cbscore/utils/secrets/git.py | 4 +- cbscore/src/cbscore/utils/secrets/mgr.py | 3 +- 20 files changed, 83 insertions(+), 590 deletions(-) delete mode 100644 cbscore/src/cbscore/utils/git.py diff --git a/cbscore/pyproject.toml b/cbscore/pyproject.toml index 6212e77a..e1bc11d3 100644 --- a/cbscore/pyproject.toml +++ b/cbscore/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.13" dependencies = [ "aioboto3==13.3.0", "aiofiles>=25.1.0", + "cbscommon", "click>=8.1.8", "hvac>=2.3.0", "pydantic>=2.11.6", @@ -23,3 +24,6 @@ cbsbuild = "cbscore.__main__:main" [build-system] requires = ["uv_build>=0.9.1,<0.10.0"] build-backend = "uv_build" + +[tool.uv.sources] +cbscommon = { workspace = true } diff --git a/cbscore/src/cbscore/builder/prepare.py b/cbscore/src/cbscore/builder/prepare.py index 7c6c1119..bf9dbfd8 100644 --- a/cbscore/src/cbscore/builder/prepare.py +++ b/cbscore/src/cbscore/builder/prepare.py @@ -20,12 +20,21 @@ from pathlib import Path import pydantic +from cbscommon.git.cmds import ( + git_apply, + git_clone, + git_get_sha1, + git_remove_worktree, + git_worktree, +) +from cbscommon.git.exceptions import GitError +from cbscommon.process.cmds import async_run_cmd from cbscore.builder import BuilderError from cbscore.builder import logger as parent_logger from cbscore.builder.utils import get_component_version from cbscore.core.component import CoreComponentLoc -from cbscore.utils import CommandError, async_run_cmd, git +from cbscore.utils import CommandError from cbscore.utils.secrets.mgr import SecretsMgr from cbscore.versions.desc import VersionComponent from cbscore.versions.utils import get_major_version, get_minor_version @@ -218,12 +227,12 @@ async def _clone_repo(comp: VersionComponent) -> Path: start = dt.now(tz=datetime.UTC) try: with secrets.git_url_for(comp.repo) as comp_url: - cloned_path = await git.git_clone( + cloned_path = await git_clone( comp_url, git_repos_path, comp.name, ) - except git.GitError as e: + except GitError as e: msg = f"error cloning '{comp.repo}' to '{git_repos_path}': {e}" logger.error(msg) raise BuilderError(msg) from e @@ -240,12 +249,12 @@ async def _checkout_ref(repo: Path, ref: str) -> Path: """Checkout given ref in repository located at `repo`.""" logger.info(f"checkout ref '{ref}' in repository at '{repo}'") try: - worktree_path = await git.git_checkout( + worktree_path = await git_worktree( repo, ref, git_worktrees_path / comp.name, ) - except git.GitError as e: + except GitError as e: msg = f"unable to checkout ref '{ref}' in repository at '{repo}': {e}" logger.exception(msg) raise BuilderError(msg) from e @@ -270,8 +279,8 @@ async def _apply_patches(comp: VersionComponent, repo: Path) -> None: for patch_path in patches_to_apply: logger.info(f"applying patch from '{patch_path}'") try: - await git.git_apply(repo, patch_path) - except git.GitError as e: + await git_apply(repo, patch_path) + except GitError as e: msg = f"unable to apply patch from '{patch_path}' to '{repo}': {e}" logger.exception(msg) raise BuilderError(msg) from e @@ -303,8 +312,8 @@ async def _get_component_info( component. """ try: - sha1 = await git.git_get_sha1(worktree_path) - except (git.GitError, Exception) as e: + sha1 = await git_get_sha1(worktree_path) + except (GitError, Exception) as e: msg = f"error obtaining SHA1 for worktree '{worktree_path}': {e}" logger.error(msg) raise BuilderError(msg) from e @@ -377,7 +386,7 @@ async def _finalize() -> BuildComponentInfo: return await _finalize() except BuilderError as e: # remove worktree - await git.git_remove_worktree(repo_path, worktree_path) + await git_remove_worktree(repo_path, worktree_path) # propagate exception raise e from None @@ -427,8 +436,8 @@ async def cleanup_components(components: dict[str, BuildComponentInfo]) -> None: for comp_name, comp in components.items(): logger.info(f"cleanup component '{comp_name}'") try: - await git.git_remove_worktree(comp.repo_path, comp.worktree_path) - except git.GitError as e: + await git_remove_worktree(comp.repo_path, comp.worktree_path) + except GitError as e: logger.warning( f"unable to cleanup component '{comp_name}' at " + f"'{comp.worktree_path}' and '{comp.repo_path}': {e}" diff --git a/cbscore/src/cbscore/builder/rpmbuild.py b/cbscore/src/cbscore/builder/rpmbuild.py index 84da7d94..7f4c15a2 100644 --- a/cbscore/src/cbscore/builder/rpmbuild.py +++ b/cbscore/src/cbscore/builder/rpmbuild.py @@ -17,11 +17,14 @@ from datetime import datetime as dt from pathlib import Path +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import CmdArgs + from cbscore.builder import BuilderError from cbscore.builder import logger as parent_logger from cbscore.builder.prepare import BuildComponentInfo from cbscore.core.component import CoreComponentLoc -from cbscore.utils import CmdArgs, CommandError, async_run_cmd +from cbscore.utils import CommandError logger = parent_logger.getChild("rpmbuild") diff --git a/cbscore/src/cbscore/builder/signing.py b/cbscore/src/cbscore/builder/signing.py index ac3a128f..8624bd74 100644 --- a/cbscore/src/cbscore/builder/signing.py +++ b/cbscore/src/cbscore/builder/signing.py @@ -17,10 +17,13 @@ from datetime import datetime as dt from pathlib import Path +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import CmdArgs + from cbscore.builder import BuilderError from cbscore.builder import logger as parent_logger from cbscore.builder.rpmbuild import ComponentBuild -from cbscore.utils import CmdArgs, CommandError, async_run_cmd +from cbscore.utils import CommandError from cbscore.utils.secrets.mgr import SecretsMgr logger = parent_logger.getChild("signing") diff --git a/cbscore/src/cbscore/builder/upload.py b/cbscore/src/cbscore/builder/upload.py index 94eda8e4..2d55c9d5 100644 --- a/cbscore/src/cbscore/builder/upload.py +++ b/cbscore/src/cbscore/builder/upload.py @@ -15,10 +15,12 @@ import shutil from pathlib import Path +from cbscommon.process.cmds import async_run_cmd + from cbscore.builder import BuilderError from cbscore.builder import logger as parent_logger from cbscore.builder.rpmbuild import ComponentBuild -from cbscore.utils import CommandError, async_run_cmd +from cbscore.utils import CommandError from cbscore.utils.s3 import S3Error, S3FileLocator, s3_upload_files from cbscore.utils.secrets.mgr import SecretsMgr diff --git a/cbscore/src/cbscore/builder/utils.py b/cbscore/src/cbscore/builder/utils.py index 8379a201..f32e7e6c 100644 --- a/cbscore/src/cbscore/builder/utils.py +++ b/cbscore/src/cbscore/builder/utils.py @@ -13,10 +13,13 @@ from pathlib import Path +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import CmdArgs + from cbscore.builder import BuilderError, MissingScriptError from cbscore.builder import logger as parent_logger from cbscore.core.component import CoreComponentLoc -from cbscore.utils import CmdArgs, CommandError, async_run_cmd +from cbscore.utils import CommandError logger = parent_logger.getChild("utils") diff --git a/cbscore/src/cbscore/cmds/__init__.py b/cbscore/src/cbscore/cmds/__init__.py index d39a0c0e..a5a5e214 100644 --- a/cbscore/src/cbscore/cmds/__init__.py +++ b/cbscore/src/cbscore/cmds/__init__.py @@ -20,6 +20,7 @@ import click +from cbscommon import logger as cbscommon_logger from cbscore.config import Config from cbscore.logger import logger as root_logger @@ -61,3 +62,4 @@ def inner(*args: P.args, **kwargs: P.kwargs) -> R: def set_log_level(lvl: int) -> None: root_logger.setLevel(lvl) + cbscommon_logger.setLevel(lvl) diff --git a/cbscore/src/cbscore/cmds/versions.py b/cbscore/src/cbscore/cmds/versions.py index 162fc367..ee30431c 100644 --- a/cbscore/src/cbscore/cmds/versions.py +++ b/cbscore/src/cbscore/cmds/versions.py @@ -17,6 +17,8 @@ from pathlib import Path import click +from cbscommon.git.cmds import get_git_repo_root, get_git_user +from cbscommon.git.exceptions import GitError from cbscore.cmds import logger as parent_logger from cbscore.cmds import with_config @@ -25,7 +27,6 @@ from cbscore.images.desc import get_image_desc from cbscore.releases import ReleaseError from cbscore.releases.s3 import list_releases -from cbscore.utils.git import GitError, get_git_repo_root, get_git_user from cbscore.utils.secrets import SecretsMgrError from cbscore.utils.secrets.mgr import SecretsMgr from cbscore.versions.create import version_create_helper diff --git a/cbscore/src/cbscore/images/desc.py b/cbscore/src/cbscore/images/desc.py index c83a7542..566db718 100644 --- a/cbscore/src/cbscore/images/desc.py +++ b/cbscore/src/cbscore/images/desc.py @@ -15,6 +15,7 @@ from pathlib import Path import pydantic +from cbscommon.git.cmds import get_git_repo_root from cbscore.errors import ( MalformedVersionError, @@ -24,7 +25,6 @@ from cbscore.images.errors import ( ImageDescriptorError, ) -from cbscore.utils.git import get_git_repo_root logger = parent_logger.getChild("descriptors") diff --git a/cbscore/src/cbscore/images/signing.py b/cbscore/src/cbscore/images/signing.py index a2373246..f690d289 100644 --- a/cbscore/src/cbscore/images/signing.py +++ b/cbscore/src/cbscore/images/signing.py @@ -14,13 +14,14 @@ import os from typing import override +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import CmdArgs + from cbscore.errors import CESError from cbscore.images import logger as parent_logger from cbscore.utils import ( - CmdArgs, CommandError, PasswordArg, - async_run_cmd, run_cmd, ) from cbscore.utils.containers import get_container_image_base_uri diff --git a/cbscore/src/cbscore/images/skopeo.py b/cbscore/src/cbscore/images/skopeo.py index a00064cd..6e78a9bc 100644 --- a/cbscore/src/cbscore/images/skopeo.py +++ b/cbscore/src/cbscore/images/skopeo.py @@ -19,13 +19,14 @@ import re import pydantic +from cbscommon.process.types import CmdArgs from cbscore.errors import CESError, UnknownRepositoryError from cbscore.images import get_image_name from cbscore.images import logger as parent_logger from cbscore.images.errors import ImageNotFoundError, SkopeoError from cbscore.images.signing import can_sign, sign -from cbscore.utils import CmdArgs, Password, run_cmd +from cbscore.utils import Password, run_cmd from cbscore.utils.containers import get_container_image_base_uri from cbscore.utils.secrets import SecretsMgrError from cbscore.utils.secrets.mgr import SecretsMgr diff --git a/cbscore/src/cbscore/logger.py b/cbscore/src/cbscore/logger.py index c2b10589..789dbeff 100644 --- a/cbscore/src/cbscore/logger.py +++ b/cbscore/src/cbscore/logger.py @@ -13,10 +13,15 @@ import logging +from cbscommon import logger as cbscommon_logger + logging.basicConfig(level=logging.INFO) + + logger = logging.getLogger("cbscore") def set_debug_logging() -> None: """Set debug logging for cbscore library.""" logger.setLevel(logging.DEBUG) + cbscommon_logger.setLevel(logging.DEBUG) diff --git a/cbscore/src/cbscore/releases/utils.py b/cbscore/src/cbscore/releases/utils.py index 7f615096..0378646a 100644 --- a/cbscore/src/cbscore/releases/utils.py +++ b/cbscore/src/cbscore/releases/utils.py @@ -12,10 +12,13 @@ # GNU General Public License for more details. +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import CmdArgs + from cbscore.core.component import CoreComponentLoc from cbscore.releases import ReleaseError from cbscore.releases import logger as parent_logger -from cbscore.utils import CmdArgs, CommandError, async_run_cmd +from cbscore.utils import CommandError logger = parent_logger.getChild("utils") diff --git a/cbscore/src/cbscore/runner.py b/cbscore/src/cbscore/runner.py index 1a37df68..440690f1 100644 --- a/cbscore/src/cbscore/runner.py +++ b/cbscore/src/cbscore/runner.py @@ -23,12 +23,12 @@ from typing import override import aiofiles +from cbscommon.process.types import AsyncRunCmdOutCallback from cbscore.builder.report import BuildArtifactReport from cbscore.config import Config, ConfigError, LoggingConfig from cbscore.errors import CESError from cbscore.logger import logger as root_logger -from cbscore.utils import AsyncRunCmdOutCallback from cbscore.utils.podman import PodmanError, podman_run, podman_stop from cbscore.utils.secrets import SecretsError from cbscore.versions.desc import VersionDescriptor diff --git a/cbscore/src/cbscore/utils/__init__.py b/cbscore/src/cbscore/utils/__init__.py index 4d5190eb..803ad626 100644 --- a/cbscore/src/cbscore/utils/__init__.py +++ b/cbscore/src/cbscore/utils/__init__.py @@ -11,17 +11,15 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -import abc -import asyncio -import os -import re -import shutil import subprocess -from asyncio.streams import StreamReader -from collections.abc import Callable, Coroutine -from io import StringIO -from pathlib import Path -from typing import Any, override +from typing import override + +from cbscommon.process.cmds import get_unsecured_cmd, sanitize_cmd +from cbscommon.process.types import ( + CmdArgs, + MaybeSecure, + SecureArg, +) from cbscore.errors import CESError from cbscore.logger import logger as root_logger @@ -35,13 +33,6 @@ def __str__(self) -> str: return "Command error" + f": {self.msg}" if self.msg else "" -class SecureArg(abc.ABC): - @property - @abc.abstractmethod - def value(self) -> str: - pass - - class Password(SecureArg): _value: str @@ -109,159 +100,23 @@ def _get_value(self, v: str | SecureArg) -> str: return v if isinstance(v, str) else v.value -MaybeSecure = str | SecureArg -CmdArgs = list[MaybeSecure] - - def get_maybe_secure_arg(value: MaybeSecure) -> str: return value if isinstance(value, str) else value.value -def _sanitize_cmd(cmd: CmdArgs) -> list[str]: - sub_pattern = re.compile(r"(.*)(?:(--pass(?:phrase)[\s=]+)[^\s]+)") - - sanitized: list[str] = [] - next_secure = False - for c in cmd: - if isinstance(c, SecureArg): - sanitized.append(str(c)) - continue - - if c == "--passphrase" or c == "--pass": - next_secure = True - sanitized.append(c) - continue - - if next_secure: - sanitized.append("****") - next_secure = False - continue - - s = re.sub(sub_pattern, r"\1\2****", c) - sanitized.append(s) - - return sanitized - - -def get_unsecured_cmd(orig: CmdArgs) -> list[str]: - cmd: list[str] = [] - for c in orig: - if isinstance(c, SecureArg): - cmd.append(c.value) - else: - cmd.append(c) - return cmd - - def run_cmd(cmd: CmdArgs, env: dict[str, str] | None = None) -> tuple[int, str, str]: - logger.debug(f"sync run '{_sanitize_cmd(cmd)}'") + logger.debug(f"sync run '{sanitize_cmd(cmd)}'") try: p = subprocess.run(get_unsecured_cmd(cmd), env=env, capture_output=True) # noqa: S603 except OSError as e: - logger.exception(f"error running '{_sanitize_cmd(cmd)}'") + logger.exception(f"error running '{sanitize_cmd(cmd)}'") raise CESError() from e if p.returncode != 0: logger.error( - f"error running '{_sanitize_cmd(cmd)}': " + f"error running '{sanitize_cmd(cmd)}': " + f"retcode = {p.returncode}, res: {p.stderr}" ) return (p.returncode, "", p.stderr.decode("utf-8")) return (0, p.stdout.decode("utf-8"), p.stderr.decode("utf-8")) - - -def _reset_python_env(env: dict[str, str]) -> dict[str, str]: - logger.debug("reset python env for command") - - python3_loc = shutil.which("python3") - if not python3_loc: - print("python3 executable not found") - return env - - logger.debug(f"python3 location: {python3_loc}") - - python3_path = Path(python3_loc) - if python3_path.parent.full_match("/usr/bin"): - logger.debug("nothing to do to python3 path") - return env - - orig_path = env.get("PATH") - assert orig_path - paths = orig_path.split(":") - - new_paths: list[str] = [] - for p in paths: - if Path(p).full_match(python3_path.parent): - continue - new_paths.append(p) - - env["PATH"] = ":".join(new_paths) - return env - - -AsyncRunCmdOutCallback = Callable[[str], Coroutine[Any, Any, None]] # pyright: ignore[reportExplicitAny] - - -async def async_run_cmd( - cmd: CmdArgs, - *, - outcb: AsyncRunCmdOutCallback | None = None, - timeout: float | None = None, - cwd: Path | None = None, - reset_python_env: bool = False, - extra_env: dict[str, str] | None = None, -) -> tuple[int, str, str]: - logger.debug(f"async run '{_sanitize_cmd(cmd)}'") - - env: dict[str, str] = os.environ.copy() - if reset_python_env: - env = _reset_python_env(env) - - if extra_env: - env.update(extra_env) - - logger.debug(f"run async subprocess, cwd: {cwd}, cmd: {cmd}") - p = await asyncio.create_subprocess_exec( - *(get_unsecured_cmd(cmd)), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=cwd, - env=env, - ) - - async def read_stream(stream: StreamReader | None) -> str: - collected = StringIO() - - if not stream: - return "" - - async for line in stream: - ln = line.decode("utf-8") - if outcb: - await outcb(ln) - else: - _ = collected.write(ln) - - return collected.getvalue() - - async def monitor(): - return await asyncio.gather( - read_stream(p.stdout), - read_stream(p.stderr), - ) - - try: - retcode, (stdout, stderr) = await asyncio.wait_for( - asyncio.gather(p.wait(), monitor()), timeout=timeout - ) - except (TimeoutError, asyncio.CancelledError): - # FIXME: evaluate all callers for whether they are properly handling these - # exceptions, vs handling a 'CommandError'. Or implement specific - # 'CommandError' exceptions for timeout and cancellation. - logger.error("async subprocess timed out or was cancelled") - p.kill() - _ = await p.wait() - raise - - return retcode, stdout, stderr diff --git a/cbscore/src/cbscore/utils/buildah.py b/cbscore/src/cbscore/utils/buildah.py index efd6d084..c40d9b2d 100644 --- a/cbscore/src/cbscore/utils/buildah.py +++ b/cbscore/src/cbscore/utils/buildah.py @@ -19,15 +19,12 @@ from pathlib import Path from typing import override +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import AsyncRunCmdOutCallback, CmdArgs + from cbscore.errors import CESError from cbscore.images.signing import SigningError, async_sign -from cbscore.utils import ( - AsyncRunCmdOutCallback, - CmdArgs, - CommandError, - Password, - async_run_cmd, -) +from cbscore.utils import CommandError, Password from cbscore.utils import logger as parent_logger from cbscore.utils.containers import ( get_container_canonical_uri, diff --git a/cbscore/src/cbscore/utils/git.py b/cbscore/src/cbscore/utils/git.py deleted file mode 100644 index 77e0e43c..00000000 --- a/cbscore/src/cbscore/utils/git.py +++ /dev/null @@ -1,401 +0,0 @@ -# CES library - git utilities -# Copyright (C) 2025 Clyso GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - - -import errno -import re -import secrets -import shutil -from pathlib import Path -from typing import override - -from cbscore.errors import CESError -from cbscore.utils import CmdArgs, MaybeSecure, async_run_cmd -from cbscore.utils import logger as parent_logger - -logger = parent_logger.getChild("git") - - -class GitError(CESError): - retcode: int - - def __init__(self, retcode: int, msg: str) -> None: - super().__init__(msg) - self.retcode = retcode - - @override - def __str__(self) -> str: - return f"git error: {self.msg} (retcode: {self.retcode})" - - -class GitConfigNotSetError(GitError): - def __init__(self, what: str) -> None: - super().__init__(errno.ENOENT, f"{what} not set in config") - - -async def run_git(args: CmdArgs, *, path: Path | None = None) -> str: - """ - Run a git command within the repository. - - If `path` is provided, run the command in `path`. Otherwise, run in the current - directory. - """ - cmd: CmdArgs = ["git"] - if path is not None: - cmd.extend(["-C", path.resolve().as_posix()]) - - cmd.extend(args) - logger.debug(f"run {cmd}") - try: - rc, stdout, stderr = await async_run_cmd(cmd) - except Exception as e: - msg = f"unexpected error running command: {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - if rc != 0: - logger.error(f"unable to obtain result from git '{args}': {stderr}") - raise GitError(rc, stderr) - - return stdout - - -async def get_git_user() -> tuple[str, str]: - """Obtain the current repository's git user and email, returned as a tuple.""" - - async def _run_git_config_for(v: str) -> str: - val = await run_git(["config", v]) - if len(val) == 0: - logger.error(f"'{v}' not set in git config") - raise GitConfigNotSetError(v) - - return val.strip() - - user_name = await _run_git_config_for("user.name") - user_email = await _run_git_config_for("user.email") - assert len(user_name) > 0 and len(user_email) > 0 - return (user_name, user_email) - - -async def get_git_repo_root() -> Path: - """Obtain the root of the current git repository.""" - val = await run_git(["rev-parse", "--show-toplevel"]) - if len(val) == 0: - logger.error("unable to obtain toplevel git directory path") - raise GitError(errno.ENOENT, "top-level git directory not found") - - return Path(val.strip()) - - -async def get_git_modified_paths( - base_sha: str, - ref: str, - *, - in_repo_path: str | None = None, - repo_path: Path | None = None, -) -> tuple[list[Path], list[Path]]: - """ - Obtain all modifications since `ref` on the repository. - - If `path` is specified, perform the action within the context of `path`. Otherwise, - on the git repository existing in current directory. - """ - try: - cmd: CmdArgs = [ - "diff-tree", - "--diff-filter=ACDMR", - "--ignore-all-space", - "--no-commit-id", - "--name-status", - "-r", - base_sha, - ref, - ] - - if in_repo_path: - cmd.extend(["--", in_repo_path]) - - val = await run_git(cmd, path=repo_path) - except GitError as e: - logger.error(f"error: unable to obtain latest patch: {e}") - raise GitError( - errno.ENOTRECOVERABLE, - f"unable to obtain patches between {base_sha} and {ref}", - ) from e - - if len(val) == 0: - logger.debug(f"no relevant patches found between {base_sha} and {ref}") - return [], [] - - descs_deleted: list[Path] = [] - descs_modified: list[Path] = [] - - lines = val.splitlines() - regex = ( - re.compile(rf"^\s*([ACDMR])\s+({repo_path}.*)\s*$") - if repo_path is not None - else re.compile(r"\s*([ACDMR])\s+([^\s]+)\s*$") - ) - for line in lines: - m = re.match(regex, line) - if m is None: - logger.debug(f"'{line}' does not match") - continue - - action = m.group(1) - target = m.group(2) - logger.debug(f"action: {action}, target: {target}") - - match action: - case "D": - descs_deleted.append(Path(target)) - case "A" | "C" | "M" | "R": - descs_modified.append(Path(target)) - case _: - logger.error( - f"unexpected action '{action}' on '{target}', line: '{line}'" - ) - raise GitError( - errno.ENOTRECOVERABLE, f"unexpected action '{action}' on '{target}'" - ) - - return descs_modified, descs_deleted - - -async def _clone(repo: MaybeSecure, dest_path: Path) -> None: - """Clones a repository from `repo` to `dest_path`.""" - try: - _ = await run_git( - [ - "clone", - "--mirror", - "--quiet", - repo, - dest_path.resolve().as_posix(), - ] - ) - except GitError as e: - msg = f"unable to clone '{repo}' to '{dest_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def _update(repo: MaybeSecure, repo_path: Path) -> None: - """Update a git repository in `repo_path` from its upstream at `repo`.""" - try: - _ = await run_git(["remote", "set-url", "origin", repo], path=repo_path) - _ = await run_git(["remote", "update"], path=repo_path) - except GitError as e: - msg = f"unable to update '{repo_path}': {e}'" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def git_checkout(repo_path: Path, ref: str, worktrees_base_path: Path) -> Path: - """ - Checkout a reference pointed to by `ref`, in repository `repo_path`. - - Uses git worktrees to checkout the reference into a new worktree under - `worktrees_base_path`. - - Returns the path to the checked out worktree. - """ - try: - worktrees_base_path.mkdir(parents=True, exist_ok=True) - except Exception as e: - msg = f"unable to create worktrees base path at '{worktrees_base_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - worktree_rnd_suffix = secrets.token_hex(5) - worktree_name = ref.replace("/", "--") + f".{worktree_rnd_suffix}" - worktree_path = worktrees_base_path / worktree_name - logger.info(f"checkout ref '{ref}' into worktree at '{worktree_path}'") - - try: - _ = await run_git( - [ - "worktree", - "add", - "--track", - "-b", - worktree_name, - "--quiet", - worktree_path.resolve().as_posix(), - ref, - ], - path=repo_path, - ) - except GitError as e: - msg = f"unable to checkout ref '{ref}' in repository '{repo_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - return worktree_path - - -async def git_remove_worktree(repo_path: Path, worktree_path: Path) -> None: - """Remove a git worktree at `worktree_path` from repository `repo_path`.""" - logger.info(f"removing worktree at '{worktree_path}' from repository '{repo_path}'") - try: - _ = await run_git( - ["worktree", "remove", "--force", worktree_path.resolve().as_posix()], - path=repo_path, - ) - except GitError as e: - msg = f"unable to remove worktree at '{worktree_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def git_fetch( - remote: str, from_ref: str, to_branch: str, *, repo_path: Path | None = None -) -> None: - """ - Fetch a reference from a remote to a new branch. - - Fetches the reference pointed to by `from_ref` from remote `remote` to a new branch - `to_branch`. If `repo_path` is specified, run the command in said path; otherwise, - run in current directory. - """ - logger.debug(f"fetch from '{remote}', source: {from_ref}, dest: {to_branch}") - try: - _ = await run_git(["fetch", remote, f"{from_ref}:{to_branch}"], path=repo_path) - except GitError as e: - msg = f"unable to fetch '{from_ref}' from '{remote}' to '{to_branch}': {e}" - logger.exception(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def git_pull( - remote: MaybeSecure, - *, - from_branch: str | None = None, - to_branch: str | None = None, - repo_path: Path | None = None, -) -> None: - """Pull commits from `remote`.""" - logger.debug(f"Pull from '{remote}' (from: {from_branch}, to: {to_branch})") - try: - cmd: CmdArgs = ["pull", remote] - branches: str | None = None - if from_branch: - branches = from_branch - if to_branch: - branches = f"{branches}:{to_branch}" - if branches: - cmd.append(branches) - _ = await run_git(cmd, path=repo_path) - except GitError as e: - msg = f"unable to pull from '{remote}': {e}" - logger.exception(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def git_cherry_pick( - sha: str, *, sha_end: str | None = None, repo_path: Path | None = None -) -> None: - """ - Cherry-picks a given SHA to the currently checked out branch. - - If `sha_end` is provided, will cherry-pick the patches `[sha~1, sha_end]`. - If `repo_path` is provided, run the command in said repository; otherwise, run - in the current directory. - """ - commit_to_pick = sha if not sha_end else f"{sha}~1..{sha_end}" - logger.debug(f"cherry-pick commit '{commit_to_pick}'") - try: - _ = await run_git(["cherry-pick", "-x", commit_to_pick], path=repo_path) - except GitError as e: - msg = f"unable to cherry-pick '{commit_to_pick}': {e}" - logger.exception(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - -async def git_clone(repo: MaybeSecure, base_path: Path, repo_name: str) -> Path: - """ - Clone a mirror git repository if it doesn't exist; update otherwise. - - Clone a git repository from `repo` to a directory `repo_name` in `base_path`. If a - git repository exists at the destination, ensure the repository is updated instead. - - Returns the path to the repository. - """ - logger.info(f"cloning '{repo}' to new destination '{base_path}' name '{repo_name}'") - if not base_path.exists(): - logger.warning(f"base path at '{base_path}' does not exist -- creating") - try: - base_path.mkdir(parents=True, exist_ok=True) - except Exception as e: - msg = f"unable to create base path at '{base_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - dest_path = base_path / f"{repo_name}.git" - - if dest_path.exists(): - if not dest_path.is_dir() or not dest_path.joinpath("HEAD").exists(): - logger.warning( - f"destination path at '{dest_path}' exists, " - + "but is not a valid git repository -- nuke it!" - ) - try: - shutil.rmtree(dest_path) - except Exception as e: - msg = f"unable to remove invalid git repository at '{dest_path}': {e}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - - # propagate exception to caller - await _update(repo, dest_path) - return dest_path - - # propagate exception to caller - await _clone(repo, dest_path) - return dest_path - - -async def git_apply(repo_path: Path, patch_path: Path) -> None: - """Apply a patch onto the repository specified by `repo_path`.""" - try: - _ = await run_git(["apply", patch_path.resolve().as_posix()], path=repo_path) - except GitError as e: - msg = f"error applying patch '{patch_path}' to '{repo_path}': {e}" - logger.exception(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) from e - pass - - -async def git_get_sha1(repo_path: Path) -> str: - """For the repository in `repo_path`, obtain its currently checked out SHA1.""" - val = await run_git(["rev-parse", "HEAD"], path=repo_path) - if len(val) == 0: - msg = f"unable to obtain current SHA1 on repository '{repo_path}" - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) - - return val.strip() - - -async def git_get_current_branch(repo_path: Path) -> str: - """Obtain the name of the currently checked out branch.""" - val = await run_git(["rev-parse", "--abbrev-ref", "HEAD"], path=repo_path) - if not val: - msg = ( - "unable to obtain current checked out branch's " - + f"name on repository '{repo_path}'" - ) - logger.error(msg) - raise GitError(errno.ENOTRECOVERABLE, msg) - - return val.strip() diff --git a/cbscore/src/cbscore/utils/podman.py b/cbscore/src/cbscore/utils/podman.py index 827a2b8b..d1f11ad3 100644 --- a/cbscore/src/cbscore/utils/podman.py +++ b/cbscore/src/cbscore/utils/podman.py @@ -17,9 +17,11 @@ from pathlib import Path from typing import override +from cbscommon.process.cmds import async_run_cmd +from cbscommon.process.types import AsyncRunCmdOutCallback, CmdArgs + from cbscore.errors import CESError -from . import AsyncRunCmdOutCallback, CmdArgs, async_run_cmd from . import logger as parent_logger logger = parent_logger.getChild("podman") diff --git a/cbscore/src/cbscore/utils/secrets/git.py b/cbscore/src/cbscore/utils/secrets/git.py index 23dbad8e..bea27800 100644 --- a/cbscore/src/cbscore/utils/secrets/git.py +++ b/cbscore/src/cbscore/utils/secrets/git.py @@ -20,7 +20,9 @@ from pathlib import Path from typing import cast -from cbscore.utils import MaybeSecure, Password, SecureURL +from cbscommon.process.types import MaybeSecure + +from cbscore.utils import Password, SecureURL from cbscore.utils.secrets import SecretsMgrError from cbscore.utils.secrets import logger as parent_logger from cbscore.utils.secrets.models import ( diff --git a/cbscore/src/cbscore/utils/secrets/mgr.py b/cbscore/src/cbscore/utils/secrets/mgr.py index b05595cc..6fb48b7d 100644 --- a/cbscore/src/cbscore/utils/secrets/mgr.py +++ b/cbscore/src/cbscore/utils/secrets/mgr.py @@ -18,8 +18,9 @@ from contextlib import contextmanager from pathlib import Path +from cbscommon.process.types import MaybeSecure + from cbscore.config import Config, ConfigError, VaultConfig -from cbscore.utils import MaybeSecure from cbscore.utils.secrets import SecretsMgrError from cbscore.utils.secrets import logger as parent_logger from cbscore.utils.secrets.git import git_url_for From bbf0b5c25875b79873e86a4453e32248ca381587 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 7 Apr 2026 08:19:51 +0200 Subject: [PATCH 3/3] crt: migrate git utilities to cbscommon and remove GitPython Migrate `crt` to use the unified `cbscommon.git` interface, allowing for the removal of the `GitPython` dependency and the local `crtlib/git_utils.py` module. - Remove `crt/src/crt/crtlib/git_utils.py` and delete its definitions. - Drop `gitpython` from `crt/pyproject.toml` and add `cbscommon` as a workspace dependency. - Update `crt` commands and library modules (manifest, patch, patchset) to use `asyncio.run()` with `cbscommon.git` asynchronous methods. - Standardize on `cbscommon` exception types and types (e.g., `SHA`) across the `crt` codebase. This migration completes the consolidation of git logic into `cbscommon` and removes the need for a heavyweight git library in `crt`. --- crt/pyproject.toml | 5 +- crt/src/crt/cmds/__init__.py | 3 + crt/src/crt/cmds/manifest.py | 62 +- crt/src/crt/cmds/patch.py | 21 +- crt/src/crt/cmds/patchset.py | 42 +- crt/src/crt/cmds/release.py | 118 ++-- crt/src/crt/crtlib/apply.py | 131 ++-- crt/src/crt/crtlib/errors/manifest.py | 3 +- crt/src/crt/crtlib/git_utils.py | 869 -------------------------- crt/src/crt/crtlib/logger.py | 3 + crt/src/crt/crtlib/manifest.py | 83 +-- crt/src/crt/crtlib/models/patch.py | 2 +- crt/src/crt/crtlib/models/patchset.py | 2 +- crt/src/crt/crtlib/patch.py | 14 +- crt/src/crt/crtlib/patchset.py | 58 +- 15 files changed, 288 insertions(+), 1128 deletions(-) delete mode 100644 crt/src/crt/crtlib/git_utils.py diff --git a/crt/pyproject.toml b/crt/pyproject.toml index 86216f9a..2235d2c4 100644 --- a/crt/pyproject.toml +++ b/crt/pyproject.toml @@ -6,8 +6,8 @@ authors = [{ name = "Joao Eduardo Luis", email = "joao@clyso.com" }] requires-python = ">=3.13" dependencies = [ "boto3==1.35.81", + "cbscommon", "click>=8.1.8", - "gitpython>=3.1.44", "httpx>=0.28.1", "pydantic>=2.10.6", "rich>=14.0.0", @@ -20,3 +20,6 @@ crt = "crt.cmds.crt:cmd_crt" [build-system] requires = ["uv_build>=0.9.1,<0.10.0"] build-backend = "uv_build" + +[tool.uv.sources] +cbscommon = { workspace = true } diff --git a/crt/src/crt/cmds/__init__.py b/crt/src/crt/cmds/__init__.py index 564932b8..a37f863b 100644 --- a/crt/src/crt/cmds/__init__.py +++ b/crt/src/crt/cmds/__init__.py @@ -25,6 +25,7 @@ from rich.highlighter import RegexHighlighter from rich.theme import Theme +from cbscommon import logger as cbscommon_logger from crt.crtlib.logger import logger as parent_logger logger = parent_logger.getChild("cmds") @@ -125,10 +126,12 @@ def rprint(s: str) -> None: def set_debug_logging() -> None: parent_logger.setLevel(logging.DEBUG) + cbscommon_logger.setLevel(logging.DEBUG) def set_verbose_logging() -> None: parent_logger.setLevel(logging.INFO) + cbscommon_logger.setLevel(logging.INFO) class Symbols(enum.StrEnum): diff --git a/crt/src/crt/cmds/manifest.py b/crt/src/crt/cmds/manifest.py index a0169f9c..31cc27c6 100644 --- a/crt/src/crt/cmds/manifest.py +++ b/crt/src/crt/cmds/manifest.py @@ -12,6 +12,7 @@ # GNU General Public License for more details. +import asyncio import datetime import errno import re @@ -23,6 +24,14 @@ import click import rich.box +from cbscommon.git.cmds import ( + git_prepare_remote, + git_push, + git_remote_exists, + git_remote_ref_exists, + git_tag_exists_in_remote, +) +from cbscommon.git.exceptions import GitError from cbscore.versions.utils import parse_version from rich.console import Group, RenderableType from rich.padding import Padding @@ -39,14 +48,6 @@ ) from crt.crtlib.errors.patchset import NoSuchPatchSetError, PatchSetError from crt.crtlib.errors.release import NoSuchReleaseError, ReleaseError -from crt.crtlib.git_utils import ( - GitError, - git_get_remote_ref, - git_prepare_remote, - git_push, - git_remote, - git_tag_exists_in_remote, -) from crt.crtlib.github import gh_get_pr from crt.crtlib.manifest import ( ManifestExecuteResult, @@ -1156,30 +1157,41 @@ def _prepare_release_repo( release_name: str, token: str, ) -> None: - try: - release_branch_name = manifest.base_ref - release_tag_name = release_branch_name.replace("/", "-") - remote_name = manifest.dst_repo - if ( - git_remote(ceph_repo_path, remote_name) - and git_get_remote_ref(ceph_repo_path, release_branch_name, remote_name) - and git_tag_exists_in_remote(ceph_repo_path, remote_name, release_tag_name) - ): - pinfo("release repo already prepared") - return + async def _check_release_ready() -> bool: + return ( + await git_remote_exists(ceph_repo_path, remote_name) + and await git_remote_ref_exists( + ceph_repo_path, release_branch_name, remote_name + ) + and await git_tag_exists_in_remote( + ceph_repo_path, remote_name, release_tag_name + ) + ) + async def _push_release(): release = load_release(ces_patch_path, release_name) release_repo_name = release.base_repo - - git_prepare_remote( - ceph_repo_path, f"github.com/{release_repo_name}", release_repo_name, token + await git_prepare_remote( + ceph_repo_path, + f"github.com/{release_repo_name}", + release_repo_name, + token, ) if remote_name != release_repo_name: - git_prepare_remote( + await git_prepare_remote( ceph_repo_path, f"github.com/{remote_name}", remote_name, token ) - git_push(ceph_repo_path, release_branch_name, remote_name) - git_push(ceph_repo_path, release_tag_name, remote_name) + _ = await git_push(ceph_repo_path, release_branch_name, remote_name) + _ = await git_push(ceph_repo_path, release_tag_name, remote_name) + + try: + release_branch_name = manifest.base_ref + release_tag_name = release_branch_name.replace("/", "-") + remote_name = manifest.dst_repo + if asyncio.run(_check_release_ready()): + pinfo("release repo already prepared") + return + asyncio.run(_push_release()) except GitError as e: perror(f"unable to prepare release repository: {e}") sys.exit(e.ec if e.ec else errno.ENOTRECOVERABLE) diff --git a/crt/src/crt/cmds/patch.py b/crt/src/crt/cmds/patch.py index 4a458186..d52125b6 100644 --- a/crt/src/crt/cmds/patch.py +++ b/crt/src/crt/cmds/patch.py @@ -11,6 +11,7 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +import asyncio import errno import re import sys @@ -18,12 +19,16 @@ from pathlib import Path import click +from cbscommon.git.cmds import ( + git_prepare_remote, + git_revparse, +) +from cbscommon.git.exceptions import GitError from crt.cmds import Ctx, pass_ctx, perror, psuccess, pwarn, with_patches_repo_path from crt.cmds import logger as parent_logger from crt.crtlib.apply import ApplyError, patches_apply_to_manifest from crt.crtlib.errors.manifest import MalformedManifestError, NoSuchManifestError -from crt.crtlib.git_utils import GitError, git_prepare_remote, git_revparse from crt.crtlib.manifest import load_manifest_by_name_or_uuid, store_manifest from crt.crtlib.patch import ( PatchError, @@ -164,18 +169,20 @@ def _check_repo(repo_path: Path, what: str) -> None: if not ctx.run_locally: # update remote repo, maybe patches are not yet in the current repo state try: - _ = git_prepare_remote( - src_ceph_repo_path, - f"github.com/{src_gh_repo}", - src_gh_repo, - ctx.github_token, + asyncio.run( + git_prepare_remote( + src_ceph_repo_path, + f"github.com/{src_gh_repo}", + src_gh_repo, + ctx.github_token, + ) ) except Exception as e: perror(f"unable to update remote '{src_gh_repo}': {e}") sys.exit(errno.ENOTRECOVERABLE) try: - shas = [git_revparse(src_ceph_repo_path, sha) for sha in patch_sha] + shas = [asyncio.run(git_revparse(src_ceph_repo_path, sha)) for sha in patch_sha] except GitError as e: perror(f"unable to obtain sha: {e}") sys.exit(errno.EINVAL) diff --git a/crt/src/crt/cmds/patchset.py b/crt/src/crt/cmds/patchset.py index e6540a6f..75a1ac1c 100644 --- a/crt/src/crt/cmds/patchset.py +++ b/crt/src/crt/cmds/patchset.py @@ -11,6 +11,7 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +import asyncio import datetime import errno import re @@ -23,6 +24,16 @@ import click import pydantic import rich.box +from cbscommon.git.cmds import ( + git_branch_delete, + git_branch_from, + git_fetch_ref, + git_get_patch_sha_title, + git_patches_in_interval, + git_prepare_remote, +) +from cbscommon.git.exceptions import GitError +from cbscommon.git.types import SHA from rich.console import Group, RenderableType from rich.padding import Padding from rich.table import Table @@ -42,15 +53,6 @@ from crt.crtlib.errors.patchset import ( MalformedPatchSetError, ) -from crt.crtlib.git_utils import ( - SHA, - GitError, - git_branch_delete, - git_branch_from, - git_get_patch_sha_title, - git_patches_in_interval, - git_prepare_remote, -) from crt.crtlib.models.common import ( AuthorData, ManifestPatchEntry, @@ -575,8 +577,10 @@ def cmd_patchset_add( # obtain the right shas progress.new_task("prepare remote") try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + ) ) except Exception as e: progress.stop_error() @@ -588,7 +592,9 @@ def cmd_patchset_add( progress.new_task("fetch patches") try: - _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") + _ = asyncio.run( + git_fetch_ref(ceph_repo_path, patches_branch, dst_branch, gh_repo) + ) except Exception as e: progress.stop_error() perror(f"unable to fetch branch '{patches_branch}': {e}") @@ -596,7 +602,7 @@ def cmd_patchset_add( else: progress.new_task("create branch patches") try: - git_branch_from(ceph_repo_path, patches_branch, dst_branch) + asyncio.run(git_branch_from(ceph_repo_path, patches_branch, dst_branch)) except GitError as e: progress.stop_error() perror( @@ -606,7 +612,7 @@ def cmd_patchset_add( def _cleanup() -> None: try: - git_branch_delete(ceph_repo_path, dst_branch) + asyncio.run(git_branch_delete(ceph_repo_path, dst_branch)) except Exception as e: progress.stop_error() perror(f"unable to delete temporary branch '{dst_branch}': {e}") @@ -636,7 +642,9 @@ def _cleanup() -> None: if sha_end: try: for sha, title in reversed( - git_patches_in_interval(ceph_repo_path, sha_begin, sha_end) + asyncio.run( + git_patches_in_interval(ceph_repo_path, sha_begin, sha_end) + ) ): if sha not in existing_shas: patches_lst.append((sha, title)) @@ -649,7 +657,9 @@ def _cleanup() -> None: sys.exit(errno.ENOTRECOVERABLE) else: try: - sha, title = git_get_patch_sha_title(ceph_repo_path, sha_begin) + sha, title = asyncio.run( + git_get_patch_sha_title(ceph_repo_path, sha_begin) + ) if sha not in existing_shas: patches_lst.append((sha, title)) else: diff --git a/crt/src/crt/cmds/release.py b/crt/src/crt/cmds/release.py index 62825346..8db6d269 100644 --- a/crt/src/crt/cmds/release.py +++ b/crt/src/crt/cmds/release.py @@ -11,6 +11,7 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +import asyncio import errno import re import sys @@ -19,28 +20,26 @@ import click import rich.box +from cbscommon.git.cmds import ( + git_branch_from, + git_fetch_ref, + git_local_head_exists, + git_prepare_remote, + git_push, + git_remote_exists, + git_remote_ref_exists, + git_remote_ref_names, + git_reset_state, + git_tag, +) +from cbscommon.git.exceptions import GitError, GitFetchHeadNotFoundError, GitIsTagError from cbscore.versions.utils import parse_version -from git import GitError from rich.padding import Padding from rich.table import Table from crt.cmds._common import CRTExitError, CRTProgress from crt.crtlib.errors.manifest import NoSuchManifestError from crt.crtlib.errors.release import NoSuchReleaseError -from crt.crtlib.git_utils import ( - GitFetchHeadNotFoundError, - GitIsTagError, - git_branch_from, - git_cleanup_repo, - git_fetch_ref, - git_get_local_head, - git_get_remote_ref, - git_prepare_remote, - git_push, - git_remote, - git_reset_head, - git_tag, -) from crt.crtlib.manifest import load_manifest_by_name_or_uuid from crt.crtlib.models.release import Release from crt.crtlib.release import load_release, release_exists, store_release @@ -74,8 +73,7 @@ def _prepare_release_repo( run_locally: bool = False, ) -> None: try: - git_cleanup_repo(ceph_repo_path) - git_reset_head(ceph_repo_path, "main") + asyncio.run(git_reset_state(ceph_repo_path)) except GitError as e: perror(f"failed to cleanup ceph repo at '{ceph_repo_path}': {e}") raise _ExitError(errno.ENOTRECOVERABLE) from e @@ -85,12 +83,16 @@ def _prepare_release_repo( return try: - _ = git_prepare_remote( - ceph_repo_path, f"github.com/{src_repo}", src_repo, token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{src_repo}", src_repo, token + ) ) if src_repo != dst_repo: - _ = git_prepare_remote( - ceph_repo_path, f"github.com/{dst_repo}", dst_repo, token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{dst_repo}", dst_repo, token + ) ) except GitError as e: perror(f"failed to prepare git remotes: {e}") @@ -108,11 +110,11 @@ def _prepare_release_branches( ) -> None: try: if run_locally: - if git_get_local_head(ceph_repo_path, dst_branch): + if asyncio.run(git_local_head_exists(ceph_repo_path, dst_branch)): perror(f"destination branch '{dst_branch}' already exists locally") sys.exit(errno.EEXIST) else: - if git_get_remote_ref(ceph_repo_path, dst_branch, dst_repo): + if asyncio.run(git_remote_ref_exists(ceph_repo_path, dst_branch, dst_repo)): perror( f"destination branch '{dst_branch}' already exists in '{dst_repo}'" ) @@ -124,7 +126,7 @@ def _prepare_release_branches( is_tag = False if run_locally: try: - git_branch_from(ceph_repo_path, src_ref, dst_branch) + asyncio.run(git_branch_from(ceph_repo_path, src_ref, dst_branch)) except GitError as e: perror(f"failed to create branch from source ref '{src_ref}': {e}") raise _ExitError(errno.ENOTRECOVERABLE) from e @@ -132,7 +134,9 @@ def _prepare_release_branches( return else: try: - _ = git_fetch_ref(ceph_repo_path, src_ref, dst_branch, src_repo) + _ = asyncio.run( + git_fetch_ref(ceph_repo_path, src_ref, dst_branch, src_repo) + ) except GitIsTagError: logger.debug(f"source ref '{src_ref}' is a tag, fetching as branch") is_tag = True @@ -145,7 +149,7 @@ def _prepare_release_branches( if is_tag: try: - git_branch_from(ceph_repo_path, src_ref, dst_branch) + asyncio.run(git_branch_from(ceph_repo_path, src_ref, dst_branch)) except GitError as e: perror(f"failed to create branch '{dst_branch}' from tag '{src_ref}': {e}") raise _ExitError(errno.ENOTRECOVERABLE) from e @@ -320,8 +324,8 @@ def cmd_release_start( progress.done_task() progress.new_task("prepare release branches") - if not ctx.run_locally and git_get_remote_ref( - ceph_repo_path, f"release/{release_name}", dst_repo + if not ctx.run_locally and asyncio.run( + git_remote_ref_exists(ceph_repo_path, f"release/{release_name}", dst_repo) ): progress.stop_error() perror(f"release '{release_name}' already marked released in '{dst_repo}'") @@ -346,7 +350,7 @@ def cmd_release_start( if not ctx.run_locally: try: - _ = git_push(ceph_repo_path, release_base_branch, dst_repo) + _ = asyncio.run(git_push(ceph_repo_path, release_base_branch, dst_repo)) except GitError as e: progress.stop_error() perror(f"failed to push release branch '{release_base_branch}': {e}") @@ -359,12 +363,14 @@ def cmd_release_start( sys.exit(errno.ENOTRECOVERABLE) try: - git_tag( - ceph_repo_path, - release_base_tag, - release_base_branch, - msg=f"Base release for {release_name}", - push_to=dst_repo if not ctx.run_locally else None, + asyncio.run( + git_tag( + ceph_repo_path, + release_base_tag, + release_base_branch, + msg=f"Base release for {release_name}", + push_to=dst_repo if not ctx.run_locally else None, + ) ) except GitError as e: progress.stop_error() @@ -456,7 +462,7 @@ def cmd_release_list( if ctx.run_locally: progress.new_task("get remote") - if not (remote := git_remote(ceph_repo_path, dst_repo)): + if not asyncio.run(git_remote_exists(ceph_repo_path, dst_repo)): pinfo(f"remote {dst_repo} doesn't exist locally") console.print(Padding(table, (1, 0, 1, 0))) progress.done_task() @@ -467,8 +473,10 @@ def cmd_release_list( else: progress.new_task("prepare remote") try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{dst_repo}", dst_repo, gh_token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{dst_repo}", dst_repo, gh_token + ) ) except GitError as e: perror(f"unable to prepare remote repository '{dst_repo}': {e}") @@ -482,8 +490,8 @@ def cmd_release_list( remote_base_releases: list[str] = [] releases_meta: dict[str, Release | None] = {} - for r in remote.refs: - ref_name = r.name[len(dst_repo) + 1 :] + for r in asyncio.run(git_remote_ref_names(ceph_repo_path, dst_repo)): + ref_name = r[len(dst_repo) + 1 :] m = re.match(r"(release|release-base)/((?:ces|ccs)-.+)", ref_name) if not m: continue @@ -693,11 +701,13 @@ def cmd_release_finish( f"fetch manifest branch '{manifest.dst_branch}' to '{release.release_branch}'" ) try: - _ = git_fetch_ref( - ceph_repo_path, - manifest.dst_branch, - release.release_branch, - manifest.dst_repo, + _ = asyncio.run( + git_fetch_ref( + ceph_repo_path, + manifest.dst_branch, + release.release_branch, + manifest.dst_repo, + ) ) except GitError as e: perror(f"failed to fetch manifest branch '{manifest.dst_branch}': {e}") @@ -708,7 +718,9 @@ def cmd_release_finish( f"push release branch '{release.release_branch}' to '{release.release_repo}'" ) try: - _ = git_push(ceph_repo_path, release.release_branch, release.release_repo) + _ = asyncio.run( + git_push(ceph_repo_path, release.release_branch, release.release_repo) + ) except GitError as e: perror(f"failed to push release branch '{release.release_branch}': {e}") progress.stop_error() @@ -724,12 +736,14 @@ def cmd_release_finish( f"tagging release branch '{release.release_branch}' with '{release.name}'" ) try: - git_tag( - ceph_repo_path, - release.name, - release.release_branch, - msg=f"Release '{release.name}'", - push_to=release.release_repo, + asyncio.run( + git_tag( + ceph_repo_path, + release.name, + release.release_branch, + msg=f"Release '{release.name}'", + push_to=release.release_repo, + ) ) except GitError as e: perror(f"failed to create and push tag '{release.name}': {e}") diff --git a/crt/src/crt/crtlib/apply.py b/crt/src/crt/crtlib/apply.py index 65a21d34..71c1d8b5 100644 --- a/crt/src/crt/crtlib/apply.py +++ b/crt/src/crt/crtlib/apply.py @@ -12,22 +12,25 @@ # GNU General Public License for more details. +import asyncio import datetime from datetime import datetime as dt from pathlib import Path from typing import override -import git - -from crt.crtlib.git_utils import ( - SHA, - GitAMApplyError, +from cbscommon.git.cmds import ( + get_git_user, git_am_abort, git_am_apply, - git_cleanup_repo, - git_get_local_head, + git_branch_delete, + git_checkout, git_prepare_remote, + git_reset_state, + git_update_submodules, ) +from cbscommon.git.exceptions import GitAMApplyError, GitError +from cbscommon.git.types import SHA + from crt.crtlib.logger import logger as parent_logger from crt.crtlib.models.common import ManifestPatchEntry from crt.crtlib.models.manifest import ReleaseManifest @@ -58,84 +61,25 @@ def __init__(self, sha: SHA, files: list[str]) -> None: self.conflict_files = files -def _update_submodules(repo_path: Path) -> None: - logger.debug("update submodules") - repo = git.Repo(repo_path) - try: - repo.git.execute( # pyright: ignore[reportCallIssue] - ["git", "submodule", "update", "--init", "--recursive"], - as_process=False, - with_stdout=True, - ) - except Exception as e: - msg = f"unable to update repository's submodules: {e}" - logger.error(msg) - raise ApplyError(msg=msg) from None - - -def _checkout_ref(repo_path: Path, from_ref: str, branch_name: str) -> git.Head: - logger.debug(f"checkout ref '{from_ref}' to '{branch_name}'") - repo = git.Repo(repo_path) - if head := git_get_local_head(repo_path, branch_name): - logger.debug(f"branch '{branch_name}' already exists, simply checkout") - repo.head.reference = head - _ = repo.head.reset(index=True, working_tree=True) - return head - - assert branch_name not in repo.heads - try: - new_head = repo.create_head(branch_name, from_ref) - except Exception: - msg = f"unable to create new head '{branch_name}' " + f"from '{from_ref}'" - logger.exception(msg) - raise ApplyError(msg=msg) from None - - repo.head.reference = new_head - _ = repo.head.reset(index=True, working_tree=True) - +def _prepare_repo(repo_path: Path): try: - git_cleanup_repo(repo_path) - _update_submodules(repo_path) - except Exception as e: - msg = f"unable to clean up repo state after checkout: {e}" + name, email = asyncio.run(get_git_user(repo_path)) + if not name: + msg = "user's name not set for repository" + logger.error(msg) + raise ApplyError(msg=msg) + if not email: + msg = "user's email not set for repository" + logger.error(msg) + raise ApplyError(msg=msg) + except GitError as e: + msg = f"error obtaining repository's user's data: {e}" logger.error(msg) raise ApplyError(msg=msg) from None - return new_head - - -def _prepare_repo(repo_path: Path): - repo = git.Repo(repo_path) - - def _check_repo() -> None: - logger.debug("check repo's config user and email") - for what in ["name", "email"]: - try: - res = repo.git.execute( - ["git", "config", f"user.{what}"], - with_extended_output=False, - as_process=False, - stdout_as_string=True, - ) - except Exception: - msg = f"error obtaining repository's user's {what}" - logger.error(msg) - raise ApplyError(msg=msg) from None - - if not res: - msg = f"user's {what} not set for repository" - logger.error(msg) - raise ApplyError(msg=msg) - - def _cleanup_repo() -> None: - git_cleanup_repo(repo_path) - repo.head.reference = repo.heads.main - _ = repo.index.reset(index=True, working_tree=True) - # propagate exceptions - _check_repo() - _cleanup_repo() - _update_submodules(repo_path) + asyncio.run(git_reset_state(repo_path)) + asyncio.run(git_update_submodules(repo_path)) def apply_manifest( @@ -148,18 +92,15 @@ def apply_manifest( no_cleanup: bool = False, run_locally: bool = False, ) -> tuple[bool, list[ManifestPatchEntry], list[ManifestPatchEntry]]: - ceph_repo = git.Repo(ceph_repo_path) - logger.info(f"apply manifest '{manifest.release_uuid}' to branch '{target_branch}'") def _cleanup(*, abort_apply: bool = False) -> None: logger.debug(f"cleanup state, branch '{target_branch}'") if abort_apply: - git_am_abort(ceph_repo_path) + asyncio.run(git_am_abort(ceph_repo_path)) - git_cleanup_repo(ceph_repo_path) - ceph_repo.head.reference = ceph_repo.heads.main - ceph_repo.git.branch(["-D", target_branch]) # pyright: ignore[reportAny] + asyncio.run(git_reset_state(ceph_repo_path)) + asyncio.run(git_branch_delete(ceph_repo_path, target_branch)) def _apply_patches( patches: list[ManifestPatchEntry], @@ -181,7 +122,7 @@ def _apply_patches( raise ApplyError(msg=f"missing patch uuid '{entry.entry_uuid}'") try: - git_am_apply(ceph_repo_path, patch_path) + asyncio.run(git_am_apply(ceph_repo_path, patch_path)) except Exception as e: raise e from None @@ -193,15 +134,25 @@ def _apply_patches( _prepare_repo(ceph_repo_path) repo_name = f"{manifest.base_ref_org}/{manifest.base_ref_repo}" if not run_locally: - _ = git_prepare_remote( - ceph_repo_path, f"github.com/{repo_name}", repo_name, token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{repo_name}", repo_name, token + ) ) except ApplyError as e: logger.error(e) raise e from None try: - _branch = _checkout_ref(ceph_repo_path, manifest.base_ref, target_branch) + asyncio.run( + git_checkout( + ceph_repo_path, + manifest.base_ref, + to_branch=target_branch, + clean=True, + update_submodules=True, + ) + ) except ApplyError as e: msg = f"unable to apply manifest patchsets: {e}" logger.error(msg) diff --git a/crt/src/crt/crtlib/errors/manifest.py b/crt/src/crt/crtlib/errors/manifest.py index 9276e3eb..b3413edd 100644 --- a/crt/src/crt/crtlib/errors/manifest.py +++ b/crt/src/crt/crtlib/errors/manifest.py @@ -14,8 +14,9 @@ import uuid from typing import override +from cbscommon.git.types import SHA + from crt.crtlib.errors import CRTError -from crt.crtlib.git_utils import SHA from crt.crtlib.models.common import AuthorData diff --git a/crt/src/crt/crtlib/git_utils.py b/crt/src/crt/crtlib/git_utils.py deleted file mode 100644 index cdccabc2..00000000 --- a/crt/src/crt/crtlib/git_utils.py +++ /dev/null @@ -1,869 +0,0 @@ -# crt - utils -# Copyright (C) 2025 Clyso GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -import logging -import re -import sys -import tempfile -from pathlib import Path -from typing import cast, override - -import git - -from crt.crtlib.errors import CRTError -from crt.crtlib.logger import logger as parent_logger - -logger = parent_logger.getChild("git") - - -SHA = str - - -class GitError(CRTError): - @override - def __str__(self) -> str: - return "git error" + (f": {self.msg}" if self.msg else "") - - -class GitIsTagError(GitError): - def __init__(self, tag: str) -> None: - super().__init__(msg=tag) - - @override - def __str__(self) -> str: - return self.with_maybe_msg("found unexpected tag") - - -class GitPatchDiffError(GitError): - @override - def __str__(self) -> str: - return "patches diff error" + (f": {self.msg}" if self.msg else "") - - -class GitEmptyPatchDiffError(GitPatchDiffError): - pass - - -class GitCherryPickError(GitError): - @override - def __str__(self) -> str: - return "cherry-pick error" + (f": {self.msg}" if self.msg else "") - - -class GitCherryPickConflictError(GitCherryPickError): - sha: SHA - conflicts: list[str] - - def __init__(self, sha: SHA, files: list[str] | None = None) -> None: - super().__init__(msg=f"conflict occurred on sha '{sha}'") - self.sha = sha - self.conflicts = files if files else [] - - -class GitAMApplyError(GitError): - pass - - -class GitMissingRemoteError(GitError): - remote_name: str - - def __init__(self, remote_name: str) -> None: - super().__init__() - self.remote_name = remote_name - - @override - def __str__(self) -> str: - return f"missing remote '{self.remote_name}'" - - -class GitMissingBranchError(GitError): - def __init__(self, branch_name: str) -> None: - super().__init__(msg=branch_name) - - @override - def __str__(self) -> str: - return self.with_maybe_msg("missing branch") - - -class GitCreateHeadExistsError(GitError): - def __init__(self, name: str) -> None: - super().__init__(msg=name) - - @override - def __str__(self) -> str: - return self.with_maybe_msg("head already exists") - - -class GitHeadNotFoundError(GitError): - def __init__(self, name: str) -> None: - super().__init__(msg=name) - - @override - def __str__(self) -> str: - return self.with_maybe_msg("head not found") - - -class GitFetchError(GitError): - remote_name: str - from_ref: str - to_ref: str - - def __init__(self, remote_name: str, from_ref: str, to_ref: str) -> None: - super().__init__() - self.remote_name = remote_name - self.from_ref = from_ref - self.to_ref = to_ref - - @override - def __str__(self) -> str: - return ( - f"failed fetching from remote '{self.remote_name}': " - + f"from '{self.from_ref}' to '{self.to_ref}'" - ) - - -class GitFetchHeadNotFoundError(GitFetchError): - def __init__(self, remote_name: str, head: str) -> None: - super().__init__(remote_name, head, "") - - @override - def __str__(self) -> str: - return f"head '{self.from_ref}' not found in remote '{self.remote_name}'" - - -class GitPushError(GitError): - def __init__(self, branch: str, dst_branch: str, remote_name: str) -> None: - super().__init__( - msg=f"from branch '{branch}' to '{dst_branch}' on remote '{remote_name}'" - ) - - @override - def __str__(self) -> str: - return self.with_maybe_msg("unable to push") - - -def git_check_patches_diff( - ceph_git_path: Path, - upstream_ref: str | SHA, - head_ref: str | SHA, - *, - limit: str | SHA | None = None, -) -> tuple[list[str], list[str]]: - logger.debug( - f"check ref '{head_ref}' against upstream '{upstream_ref}', limit '{limit}'" - ) - repo = git.Repo(ceph_git_path) - - cmd = ["git", "cherry", upstream_ref, head_ref] - if limit: - cmd.append(limit) - - try: - res = repo.git.execute( - cmd, - with_extended_output=False, - as_process=False, - stdout_as_string=True, - ) - except Exception as e: - msg = ( - f"unable to check patch diff between '{upstream_ref}' and '{head_ref}': {e}" - ) - logger.error(msg) - raise GitPatchDiffError(msg=msg) from None - - if not res: - logger.warning(f"empty diff between '{upstream_ref}' and '{head_ref}") - raise GitEmptyPatchDiffError() - - patches_res = res.splitlines() - patches_add: list[str] = [] - patches_drop: list[str] = [] - - entry_re = re.compile(r"^([-+])\s+(.*)$") - for entry in patches_res: - m = re.match(entry_re, entry) - if not m: - logger.error(f"unexpected entry format: {entry}") - continue - - action = cast(str, m.group(1)) - sha = cast(str, m.group(2)) - - match action: - case "+": - patches_add.append(sha) - case "-": - patches_drop.append(sha) - case _: - logger.error(f"unexpected patch action '{action}' for sha '{sha}'") - - logger.debug(f"ref '{head_ref}' add {patches_add}") - logger.debug(f"ref '{head_ref}' drop {patches_drop}") - - return (patches_add, patches_drop) - - -def git_patches_in_interval( - repo_path: Path, from_ref: SHA, to_ref: SHA -) -> list[tuple[SHA, str]]: - logger.debug(f"get patch interval from '{from_ref}' to '{to_ref}'") - repo = git.Repo(repo_path) - - cmd = [ - "git", - "rev-list", - "--ancestry-path", - "--pretty=oneline", - f"{from_ref}~1..{to_ref}", - ] - try: - res = repo.git.execute( - cmd, - with_extended_output=False, - as_process=False, - stdout_as_string=True, - ) - except Exception as e: - msg = f"unable to obtain patch interval: {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - def _split(ln: str) -> tuple[str, str]: - sha, title = ln.split(maxsplit=1) - return (sha, title) - - return list( - map(_split, [line.strip() for line in res.splitlines() if line.strip()]) - ) - - -def git_get_patch_sha_title(repo_path: Path, sha: SHA) -> tuple[str, str]: - logger.debug(f"get patch sha and title for '{sha}'") - repo = git.Repo(repo_path) - - cmd = ["git", "show", "--format=%H %s", "--no-patch", sha] - try: - res = repo.git.execute( - cmd, with_extended_output=False, as_process=False, stdout_as_string=True - ) - except Exception as e: - msg = f"unable to obtain patch sha and title for '{sha}': {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - logger.debug(res) - lst = [line.strip() for line in res.splitlines() if line.strip()] - if len(lst) > 1: - raise GitError(msg=f"unexpected multiple lines for patch '{sha}'") - logger.debug(lst) - patch_sha, patch_title = next(iter(lst)).split(maxsplit=1) - return (patch_sha, patch_title) - - -def git_status(repo_path: Path) -> list[tuple[str, str]]: - repo = git.Repo(repo_path) - - try: - res = cast(str, repo.git.status(["--porcelain"])) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to run git status on '{repo_path}'" - logger.error(msg) - logger.error(e.stderr) - raise GitError(msg=msg) from None - - status_lst: list[tuple[str, str]] = [] - for entry in res.splitlines(): - status, file = entry.split() - status_lst.append((status, file)) - - return status_lst - - -def git_cherry_pick(repo_path: Path, sha: SHA) -> None: - repo = git.Repo(repo_path) - - try: - repo.git.cherry_pick(["-x", "-s", sha]) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to cherry-pick patch sha '{sha}'" - logger.error(msg) - - status_files = git_status(repo_path) - conflicts: list[str] = [f for s, f in status_files if s == "UU"] - - if conflicts: - raise GitCherryPickConflictError(sha, conflicts) from None - - logger.error(e.stderr) - raise GitCherryPickError(msg=msg) from None - - -def git_abort_cherry_pick(repo_path: Path) -> None: - repo = git.Repo(repo_path) - - try: - _ = repo.git.cherry_pick("--abort") # pyright: ignore[reportAny] - except git.CommandError as e: - logger.error(f"found error aborting cherry-pick: {e.stderr}") - - -def git_am_apply(repo_path: Path, patch_path: Path) -> None: - repo = git.Repo(repo_path) - - try: - _ = repo.git.am(str(patch_path)) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to apply patch '{patch_path}'" - logger.error(msg) - logger.error(e.stderr) - raise GitAMApplyError(msg=msg) from None - - -def git_am_abort(repo_path: Path) -> None: - repo = git.Repo(repo_path) - - try: - _ = repo.git.am(["--abort"]) # pyright: ignore[reportAny] - except git.CommandError as e: - logger.error(f"found error aborting git-am:\n{e.stderr}") - - -def git_cleanup_repo(repo_path: Path) -> None: - repo = git.Repo(repo_path) - try: - repo.git.submodule( # pyright: ignore[reportAny] - [ - "deinit", - "--all", - "-f", - ] - ) - repo.git.clean(["-ffdx"]) # pyright: ignore[reportAny] - repo.git.reset(["--hard"]) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to clean up repository '{repo_path}': {e}" - logger.error(msg) - logger.error(e.stderr) - raise GitError(msg=msg) from None - - -def git_prepare_remote( - repo_path: Path, remote_uri: str, remote_name: str, token: str -) -> git.Remote: - logger.info(f"prepare remote '{remote_name}' uri '{remote_uri}'") - - repo = git.Repo(repo_path) - try: - remote = repo.remote(remote_name) - except ValueError: - remote_url = f"https://crt:{token}@{remote_uri}" - remote = repo.create_remote(remote_name, remote_url) - logger.debug(f"created remote '{remote_name}' url '{remote_url}'") - - logger.info(f"update remote '{remote_name}'") - try: - _ = remote.update() - except git.CommandError as e: - logger.error(f"unable to update remote '{remote_name}'") - logger.error(e.stderr) - raise GitError(msg=f"unable to update remote '{remote_name}'") from None - - return remote - - -def git_remote(repo_path: Path, remote_name: str) -> git.Remote | None: - logger.info(f"get remote '{remote_name}'") - - repo = git.Repo(repo_path) - try: - return repo.remote(remote_name) - except ValueError: - logger.debug(f"remote '{remote_name}' doesn't exist") - - return None - - -def _get_remote_ref_name( - remote_name: str, remote_ref: str, *, ref_name: str | None = None -) -> tuple[str, str] | None: - ref_re = re.compile(rf"^{remote_name}/(.*)$") - if m := re.match(ref_re, remote_ref): - name = cast(str, m.group(1)) - if ref_name and ref_name != name: - return None - - return (remote_name, m.group(1)) - return None - - -def git_get_remote_ref( - repo_path: Path, ref_name: str, remote_name: str -) -> git.RemoteReference | None: - repo = git.Repo(repo_path) - - try: - remote = repo.remote(remote_name) - except ValueError: - logger.error(f"remote '{remote_name}' not found") - raise GitMissingRemoteError(remote_name) from None - - for ref in remote.refs: - if _get_remote_ref_name(remote_name, ref.name, ref_name=ref_name): - return ref - - return None - - -def git_pull_ref(repo_path: Path, from_ref: str, to_ref: str, remote_name: str) -> bool: - repo = git.Repo(repo_path) - if repo.active_branch.name != to_ref: - return False - - if not git_get_remote_ref(repo_path, from_ref, remote_name): - logger.warning(f"ref '{from_ref}' not found in remote '{remote_name}'") - return False - - try: - _ = repo.git.pull([remote_name, f"{from_ref}:{to_ref}"]) # pyright: ignore[reportAny] - except git.CommandError as e: - logger.error( - f"unable to pull from '{remote_name}' ref '{from_ref}' to '{to_ref}'" - ) - logger.error(e.stderr) - raise GitFetchError(remote_name, from_ref, to_ref) from None - - return True - - -def _get_tag(repo_path: Path, tag_name: str) -> git.TagReference | None: - repo = git.Repo(repo_path) - for tag in repo.tags: - if tag.name == tag_name: - return tag - return None - - -def git_get_local_head(repo_path: Path, name: str) -> git.Head | None: - repo = git.Repo(repo_path) - for head in repo.heads: - if head.name == name: - return head - return None - - -def git_reset_head(repo_path: Path, new_head: str) -> None: - """Reset current checked out head to `new_head`.""" - repo = git.Repo(repo_path) - - head = git_get_local_head(repo_path, new_head) - if not head: - msg = f"unexpected missing local head '{new_head}'" - logger.error(msg) - raise GitError(msg) - - repo.head.reference = head - _ = repo.head.reset(index=True, working_tree=True) - - -def git_branch_from(repo_path: Path, src_ref: str, dst_branch: str) -> None: - """Create a new branch `dst_branch` from `src_ref`.""" - logger.debug(f"create branch '{dst_branch}' from '{src_ref}'") - - repo = git.Repo(repo_path) - logger.debug(f"repo active branch: {repo.active_branch}") - - if git_get_local_head(repo_path, dst_branch): - msg = f"unable to create branch '{dst_branch}', already exists" - logger.error(msg) - raise GitCreateHeadExistsError(dst_branch) - - if _get_tag(repo_path, src_ref): - logger.debug(f"source ref '{src_ref}' is a tag") - src_ref = f"refs/tags/{src_ref}" - - try: - _ = repo.git.branch([dst_branch, src_ref]) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to create branch '{dst_branch}' from '{src_ref}': {e}" - logger.error(msg) - logger.error(e.stderr) - raise GitError(msg) from None - - -def git_fetch_ref( - repo_path: Path, from_ref: str, to_ref: str, remote_name: str -) -> bool: - """ - Fetch a reference from a remote into a given branch. - - If the target branch is already checked out, perform a `git pull` instead. - If the source ref is a tag, do not fetch. - - Will raise if `from_ref` is a tag, or if it doesn't exist in the specified remote. - Might raise in other `git fetch` error conditions. - """ - logger.debug(f"fetch from '{remote_name}' ref '{from_ref}' to '{to_ref}'") - - repo = git.Repo(repo_path) - logger.debug(f"repo active branch: {repo.active_branch}") - - if repo.active_branch.name == to_ref: - logger.warning(f"checked out branch is '{to_ref}', pull instead.") - return git_pull_ref(repo_path, from_ref, to_ref, remote_name) - - # check whether 'from_ref' is a tag - if _get_tag(repo_path, from_ref): - logger.warning(f"can't fetch tag '{from_ref}' from remote '{remote_name}'") - raise GitIsTagError(from_ref) - - # check whether 'from_ref' is a remote head - if not git_get_remote_ref(repo_path, from_ref, remote_name): - logger.warning(f"unable to find ref '{from_ref}' in remote '{remote_name}'") - raise GitFetchHeadNotFoundError(remote_name, from_ref) - - try: - remote = repo.remote(remote_name) - except ValueError: - msg = f"unexpected error obtaining remote '{remote_name}'" - logger.error(msg) - raise GitError(msg) from None - - try: - _ = remote.fetch(f"{from_ref}:{to_ref}") - except git.CommandError as e: - logger.error( - f"unable to fetch from remote '{remote_name}' " - + f"ref '{from_ref}' to '{to_ref}'" - ) - logger.error(e.stderr) - raise GitFetchError(remote_name, from_ref, to_ref) from None - - return True - - -def git_checkout_ref( - repo_path: Path, - ref: str, - *, - to_branch: str | None = None, - remote_name: str | None = None, - update_from_remote: bool = False, - fetch_if_not_exists: bool = False, -) -> None: - """ - Check out a reference, possibly to a new branch. - - If `ref` exists in the repository, checks out said head. Otherwise, either raise - `GitMissingBranchError`, or attempt to fetch the branch from `remote_name` if - `remote_name` is `True` and `fetch_if_not_exists` is defined. - - If `to_branch` is defined, either checks out the provided `ref` to the specified - branch, or attempts to fetch it from remote `remote_name` (if defined). - - If `update_from_remote` is `True`, always attempt to fetch the latest updates in - the remote branch to the target branch. The target branch can be `ref` or - `to_branch` depending on whether the latter is defined. If `remote_name` is not - specified, `update_from_remote` has no effect. - """ - repo = git.Repo(repo_path) - - def _update_from_remote(head: git.Head, remote: str) -> None: - logger.debug(f"update '{head}' from remote if it exists") - try: - res = git_fetch_ref(repo_path, head.name, head.name, remote) - except Exception as e: - logger.error(f"unable to update '{head.name}' from remote '{remote}: {e}") - return - - if not res: - logger.info(f"whatever to update for '{head.name}' from remote '{remote}'") - pass - - def _checkout_head(head: git.Head, *, target_branch: str | None = None) -> None: - """ - Checkout a given head. - - If `target_branch` is specified, checkout the provided head to a new branch. - """ - logger.debug(f"checkout head '{head}' to '{target_branch}'") - - if target_branch and head.name != target_branch: - # checkout 'head' to a new branch - if git_get_local_head(repo_path, target_branch): - raise GitCreateHeadExistsError(target_branch) - head = repo.create_head(target_branch, head) - - # should we update from remote first? - if update_from_remote and remote_name: - _update_from_remote(head, remote_name) - - repo.head.reference = head - _ = repo.head.reset(index=True, working_tree=True) - pass - - target_branch = to_branch if to_branch else ref - - # check if 'ref' exists as a branch locally - if head := git_get_local_head(repo_path, target_branch): - _checkout_head(head, target_branch=target_branch) - return - - if not fetch_if_not_exists: - logger.debug(f"not fetching '{ref}' as specified") - raise GitMissingBranchError(ref) - - if not remote_name: - msg = f"unable to fetch ref '{ref}', no remote given" - logger.error(msg) - raise GitError(msg) from None - - # local head does not exist, fetch it. - is_tag = False - try: - _ = git_fetch_ref(repo_path, ref, target_branch, remote_name) - except GitIsTagError: - logger.debug(f"ref '{ref}' is a tag, must checkout instead.") - is_tag = True - except GitFetchHeadNotFoundError as e: - logger.error(f"ref '{ref}' not found in remote.") - raise e from None - except GitError as e: - logger.error(f"error occurred fetching ref '{ref}': {e}") - raise e from None - - if is_tag: - try: - _ = repo.git.checkout( # pyright: ignore[reportAny] - [ref, "-b", target_branch] - ) - except git.CommandError as e: - msg = f"unable to checkout ref '{ref}' to '{target_branch}': {e}" - logger.error(msg) - logger.error(e.stderr) - raise GitError(msg) from None - return - - # propagate exceptions - git_reset_head(repo_path, target_branch) - - -def git_branch_delete(repo_path: Path, branch: str) -> None: - """Delete a local branch.""" - repo = git.Repo(repo_path) - if repo.active_branch.name == branch: - git_cleanup_repo(repo_path) - repo.head.reference = repo.heads["main"] - - repo.git.branch(["-D", branch]) # pyright: ignore[reportAny] - - -def git_push( - repo_path: Path, - ref: str, - remote_name: str, - *, - ref_to: str | None = None, -) -> tuple[bool, list[str], list[str]]: - """Pushes either a local head of branch or a local tag to the remote.""" - dst_ref = ref_to if ref_to else ref - - if _get_tag(repo_path, ref): - ref = f"refs/tags/{ref}" - dst_ref = f"refs/tags/{dst_ref}" - elif not git_get_local_head(repo_path, ref): - # ref is neither a local branch nor tag - logger.error(f"unable to find ref '{ref}' to push") - raise GitHeadNotFoundError(ref) - - repo = git.Repo(repo_path) - try: - remote = repo.remote(remote_name) - except ValueError: - logger.error(f"unable to find remote '{remote_name}'") - raise GitMissingRemoteError(remote_name) from None - - try: - info = remote.push(f"{ref}:{dst_ref}") - except git.CommandError as e: - msg = f"unable to push '{ref}' to '{dst_ref}': {e}" - logger.error(msg) - logger.error(e.stderr) - raise GitPushError(ref, dst_ref, remote_name) from None - - updated: list[str] = [] - rejected: list[str] = [] - failed = len(info) == 0 - - for entry in info: - entry_names = _get_remote_ref_name(remote_name, entry.remote_ref.name) - if not entry_names: - logger.warning(f"mismatched remote ref on push: '{entry.remote_ref.name}'") - continue - - remote_ref_name = entry_names[1] - logger.debug(f"entry '{remote_ref_name}' flags '{entry.flags}'") - if entry.flags & entry.ERROR: - logger.debug(f"rejected head: {remote_ref_name}") - rejected.append(remote_ref_name) - elif entry.flags & (entry.NEW_HEAD | entry.FAST_FORWARD): - logger.debug(f"updated head: {remote_ref_name}") - updated.append(remote_ref_name) - - return (failed, updated, rejected) - - -def git_tag( - repo_path: Path, - tag_name: str, - ref: str, - *, - msg: str | None = None, - push_to: str | None = None, -) -> None: - repo = git.Repo(repo_path) - - logger.debug(f"create tag '{tag_name}' at ref '{ref}'") - try: - _ = repo.create_tag(tag_name, ref, msg) - except Exception as e: - msg = f"unable to create tag '{tag_name}' at ref '{ref}': {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - if push_to: - logger.debug(f"push tag '{tag_name}' to remote '{push_to}'") - try: - repo.git.push([push_to, "tag", tag_name]) # pyright: ignore[reportAny] - except Exception as e: - msg = f"unable to push tag '{tag_name}' to remote '{push_to}': {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - -def git_patch_id(repo_path: Path, sha: SHA) -> str: - repo = git.Repo(repo_path) - - with tempfile.TemporaryFile() as tmp: - try: - repo.git.show(sha, output_stream=tmp) # pyright: ignore[reportAny] - except git.CommandError: - msg = f"unable to find patch sha '{sha}'" - logger.error(msg) - raise GitError(msg=msg) from None - - _ = tmp.seek(0) - res = cast(str, repo.git.patch_id(["--stable"], istream=tmp)) # pyright: ignore[reportAny] - - if not res: - raise GitError(msg="unable to obtain git patch id") - return res.split()[0] - - -def git_revparse(repo_path: Path, commitish: SHA | str) -> str: - repo = git.Repo(repo_path) - - try: - res = repo.rev_parse(commitish) - except git.BadObject: - msg = f"rev '{commitish}' not found" - logger.error(msg) - raise GitError(msg=msg) from None - except ValueError: - msg = f"malformed rev '{commitish}'" - logger.error(msg) - raise GitError(msg=msg) from None - except Exception as e: - msg = f"unable to obtain revision for '{commitish}': {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - return res.hexsha - - -def git_format_patch(repo_path: Path, rev: SHA, *, base_rev: SHA | None = None) -> str: - repo = git.Repo(repo_path) - - args = ["--stdout"] - if not base_rev: - args.append("-1") - - rev_str = f"{base_rev}..{rev}" if base_rev else rev - args.append(rev_str) - - try: - res = cast(str, repo.git.format_patch(args)) # pyright: ignore[reportAny] - except git.CommandError as e: - msg = f"unable to obtain format patch for '{rev_str}': {e}" - logger.error(msg) - raise GitError(msg=msg) from None - - return res - - -def git_tag_exists_in_remote(repo_path: Path, remote_name: str, tag_name: str) -> bool: - try: - repo = git.Repo(repo_path) - raw_tag: str = repo.git.ls_remote( - "--tags", remote_name, f"refs/tags/{tag_name}" - ) - return bool(raw_tag.strip()) - except git.CommandError as e: - msg = f"unable to execute git ls-remote --tags {remote_name} refs/tags/{tag_name}: {e}" - logger.error(msg) - raise GitError(msg) from None - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("error: missing repo path argument") - sys.exit(1) - - logger.setLevel(logging.DEBUG) - - repo_path = Path(sys.argv[1]) - - print("checkout refs") - try: - git_checkout_ref(repo_path, "foobar") - except Exception as e: - print(f"error getting 'foobar': {e}") - - try: - git_checkout_ref( - repo_path, "main", update_from_remote=True, remote_name="clyso/ceph" - ) - except Exception as e: - print(f"error getting 'foobar': {e}") - - try: - git_checkout_ref( - repo_path, - "tentacle", - update_from_remote=True, - remote_name="clyso/ceph", - ) - except Exception as e: - print(f"error getting 'tentacle': {e}") - - try: - git_checkout_ref( - repo_path, - "v18.2.7", - to_branch="test-v18.2.7", - remote_name="ceph/ceph", - update_from_remote=True, - fetch_if_not_exists=True, - ) - except Exception as e: - print(f"error checking out 'v18.2.7' to 'test-v18.2.7': {e}") diff --git a/crt/src/crt/crtlib/logger.py b/crt/src/crt/crtlib/logger.py index f6a32b23..53bb083c 100644 --- a/crt/src/crt/crtlib/logger.py +++ b/crt/src/crt/crtlib/logger.py @@ -13,6 +13,8 @@ import logging +from cbscommon import logger_set_handler as cbscommon_set_handler + logger = logging.getLogger("crt") logger.setLevel(logging.ERROR) @@ -20,6 +22,7 @@ def logger_set_handler(handler: logging.Handler) -> None: logger.propagate = False logger.addHandler(handler) + cbscommon_set_handler(handler) def logger_unset_handler(handler: logging.Handler) -> None: diff --git a/crt/src/crt/crtlib/manifest.py b/crt/src/crt/crtlib/manifest.py index 32b837e4..1291632a 100644 --- a/crt/src/crt/crtlib/manifest.py +++ b/crt/src/crt/crtlib/manifest.py @@ -12,6 +12,7 @@ # GNU General Public License for more details. +import asyncio import datetime import re import uuid @@ -20,6 +21,23 @@ from typing import cast import pydantic +from cbscommon.git.cmds import ( + git_branch_from, + git_checkout, + git_cleanup_repo, + git_fetch_ref, + git_prepare_remote, + git_push, + git_status, +) +from cbscommon.git.exceptions import ( + GitCreateHeadExistsError, + GitError, + GitFetchError, + GitFetchHeadNotFoundError, + GitIsTagError, + GitPushError, +) from cbscore.versions.utils import parse_version from crt.crtlib.apply import ApplyError, apply_manifest @@ -31,21 +49,6 @@ NoSuchManifestError, ) from crt.crtlib.errors.stages import MissingStagePatchError -from crt.crtlib.git_utils import ( - GitCreateHeadExistsError, - GitError, - GitFetchError, - GitFetchHeadNotFoundError, - GitIsTagError, - GitPushError, - git_branch_from, - git_checkout_ref, - git_cleanup_repo, - git_fetch_ref, - git_prepare_remote, - git_push, - git_status, -) from crt.crtlib.logger import logger as parent_logger from crt.crtlib.models.common import ManifestPatchEntry from crt.crtlib.models.manifest import ReleaseManifest @@ -81,7 +84,7 @@ def _prepare_repo( run_locally: bool = False, ) -> None: try: - git_cleanup_repo(repo_path) + asyncio.run(git_cleanup_repo(repo_path)) except GitError as e: msg = f"unable to clean up repository: {e}" logger.error(msg) @@ -90,15 +93,21 @@ def _prepare_repo( if not run_locally: try: base_remote_uri = f"github.com/{base_remote_name}" - _ = git_prepare_remote(repo_path, base_remote_uri, base_remote_name, token) + asyncio.run( + git_prepare_remote(repo_path, base_remote_uri, base_remote_name, token) + ) push_remote_uri = f"github.com/{push_remote_name}" - _ = git_prepare_remote(repo_path, push_remote_uri, push_remote_name, token) + asyncio.run( + git_prepare_remote(repo_path, push_remote_uri, push_remote_name, token) + ) except GitError as e: raise ManifestError(uuid=manifest_uuid, msg=str(e)) from None # fetch from base repository, if it exists. try: - _ = git_fetch_ref(repo_path, target_branch, target_branch, push_remote_name) + _ = asyncio.run( + git_fetch_ref(repo_path, target_branch, target_branch, push_remote_name) + ) except GitIsTagError as e: msg = f"unexpected tag for branch '{target_branch}': {e}" logger.error(msg) @@ -127,21 +136,23 @@ def _prepare_repo( try: if run_locally: try: - git_branch_from(repo_path, base_ref, target_branch) + asyncio.run(git_branch_from(repo_path, base_ref, target_branch)) except GitCreateHeadExistsError: msg = f"branch {target_branch} exists" logger.info(msg) - _ = git_checkout_ref( - repo_path, - base_ref, - to_branch=target_branch, - remote_name=base_remote_name, - update_from_remote=False, - fetch_if_not_exists=not run_locally, - ) - git_cleanup_repo(repo_path) + asyncio.run( + git_checkout( + repo_path, + base_ref, + to_branch=target_branch, + remote_name=base_remote_name, + update_from_remote=False, + fetch_if_not_exists=run_locally, + clean=True, + ) + ) - logger.debug(f"git status:\n{git_status(repo_path)}") + logger.debug(f"git status:\n{asyncio.run(git_status(repo_path))}") except GitError as e: msg = f"unable to checkout ref '{base_ref}' to '{target_branch}': {e}" logger.error(msg) @@ -339,11 +350,13 @@ def manifest_publish_branch( heads_rejected: list[str] = [] logger.info(f"push '{our_branch}' to '{dst_repo}'") try: - push_res, heads_updated, heads_rejected = git_push( - repo_path, - our_branch, - dst_repo, - ref_to=dst_branch, + push_res, heads_updated, heads_rejected = asyncio.run( + git_push( + repo_path, + our_branch, + dst_repo, + ref_to=dst_branch, + ) ) except GitPushError as e: msg = f"unable to push '{our_branch}': {e}" diff --git a/crt/src/crt/crtlib/models/patch.py b/crt/src/crt/crtlib/models/patch.py index 6c34b992..286ec911 100644 --- a/crt/src/crt/crtlib/models/patch.py +++ b/crt/src/crt/crtlib/models/patch.py @@ -16,8 +16,8 @@ from typing import override import pydantic +from cbscommon.git.types import SHA -from crt.crtlib.git_utils import SHA from crt.crtlib.models.common import ( AuthorData, ManifestPatchEntry, diff --git a/crt/src/crt/crtlib/models/patchset.py b/crt/src/crt/crtlib/models/patchset.py index 0baa1958..80b165db 100644 --- a/crt/src/crt/crtlib/models/patchset.py +++ b/crt/src/crt/crtlib/models/patchset.py @@ -17,9 +17,9 @@ from typing import Annotated, Any, override import pydantic +from cbscommon.git.types import SHA from crt.crtlib.errors.patchset import EmptyPatchSetError -from crt.crtlib.git_utils import SHA from crt.crtlib.models.common import ( AuthorData, ManifestPatchEntry, diff --git a/crt/src/crt/crtlib/patch.py b/crt/src/crt/crtlib/patch.py index bdfefd5d..5efbef52 100644 --- a/crt/src/crt/crtlib/patch.py +++ b/crt/src/crt/crtlib/patch.py @@ -12,13 +12,17 @@ # GNU General Public License for more details. +import asyncio import re from datetime import datetime as dt from pathlib import Path from typing import cast +from cbscommon.git.cmds import git_format_patch, git_patch_id +from cbscommon.git.exceptions import GitError +from cbscommon.git.types import SHA + from crt.crtlib.errors import CRTError -from crt.crtlib.git_utils import SHA, GitError, git_format_patch, git_patch_id from crt.crtlib.logger import logger as parent_logger from crt.crtlib.models.common import AuthorData from crt.crtlib.models.patch import PatchInfo, PatchMeta @@ -141,14 +145,14 @@ def patch_import( target_version: str | None = None, ) -> None: try: - patch_id = git_patch_id(repo_path, sha) + patch_id = asyncio.run(git_patch_id(repo_path, sha)) except GitError as e: msg = f"unable to obtain patch id for sha '{sha}': {e}" logger.error(msg) raise PatchError(msg=msg) from None try: - formatted_patch = git_format_patch(repo_path, sha) + formatted_patch = asyncio.run(git_format_patch(repo_path, sha)) except GitError as e: msg = f"unable to obtain formatted patch for sha '{sha}': {e}" logger.error(msg) @@ -239,14 +243,14 @@ def patch_add( src_version: str | None, ) -> PatchMeta: try: - patch_id = git_patch_id(src_repo_path, sha) + patch_id = asyncio.run(git_patch_id(src_repo_path, sha)) except GitError as e: msg = f"unable to obtain patch id for sha '{sha}': {e}" logger.error(msg) raise PatchError(msg=msg) from None try: - formatted_patch = git_format_patch(src_repo_path, sha) + formatted_patch = asyncio.run(git_format_patch(src_repo_path, sha)) except GitError as e: msg = f"unable to obtain formatted patch for sha '{sha}': {e}" logger.error(msg) diff --git a/crt/src/crt/crtlib/patchset.py b/crt/src/crt/crtlib/patchset.py index e37ae9a6..693a66d6 100644 --- a/crt/src/crt/crtlib/patchset.py +++ b/crt/src/crt/crtlib/patchset.py @@ -12,12 +12,22 @@ # GNU General Public License for more details. +import asyncio import datetime import uuid from datetime import datetime as dt from pathlib import Path import pydantic +from cbscommon.git.cmds import ( + git_branch_delete, + git_branch_from, + git_check_patches_diff, + git_fetch_ref, + git_format_patch, + git_prepare_remote, +) +from cbscommon.git.exceptions import GitEmptyPatchDiffError, GitError, GitPatchDiffError from crt.crtlib.errors.patchset import ( MalformedPatchSetError, @@ -25,16 +35,6 @@ PatchSetCheckError, PatchSetError, ) -from crt.crtlib.git_utils import ( - GitEmptyPatchDiffError, - GitError, - GitPatchDiffError, - git_branch_delete, - git_branch_from, - git_check_patches_diff, - git_format_patch, - git_prepare_remote, -) from crt.crtlib.logger import logger as parent_logger from crt.crtlib.models.common import ManifestPatchEntry from crt.crtlib.models.discriminator import ( @@ -57,8 +57,10 @@ def patchset_check_patches_diff( logger.debug(f"check patchset branch '{patchset_branch}' against '{base_ref}'") try: - added, skipped = git_check_patches_diff( - ceph_git_path, base_ref, patchset_branch, limit=patchset.base_sha + added, skipped = asyncio.run( + git_check_patches_diff( + ceph_git_path, base_ref, patchset_branch, limit=patchset.base_sha + ) ) except GitEmptyPatchDiffError: logger.warning( @@ -263,23 +265,25 @@ def _set_latest(pr_dir_path: Path, sha: str) -> None: return # obtain patches - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{repo_path}", repo_path, token + asyncio.run( + git_prepare_remote(ceph_repo_path, f"github.com/{repo_path}", repo_path, token) ) src_ref = f"pull/{pr_id}/head" dst_ref = f"patchset/gh/{repo_path}/{pr_id}" try: - _ = remote.fetch(f"{src_ref}:{dst_ref}") + _ = asyncio.run(git_fetch_ref(ceph_repo_path, src_ref, dst_ref, repo_path)) except Exception as e: msg = f"error fetching patchset '{pr_id}' from '{repo_path}': {e}" logger.error(msg) raise PatchSetError(msg=msg) from None try: - formatted_patchset = git_format_patch( - ceph_repo_path, - patchset.head_sha, - base_rev=patchset.base_sha, + formatted_patchset = asyncio.run( + git_format_patch( + ceph_repo_path, + patchset.head_sha, + base_rev=patchset.base_sha, + ) ) except GitError as e: msg = f"error formatting patch set: {e}" @@ -383,7 +387,7 @@ def fetch_custom_patchset_patches( if run_locally: try: - git_branch_from(ceph_repo_path, meta.branch, dst_branch) + asyncio.run(git_branch_from(ceph_repo_path, meta.branch, dst_branch)) except GitError as e: msg = ( f"error creating patchset branch '{dst_branch}' " @@ -393,10 +397,14 @@ def fetch_custom_patchset_patches( raise PatchSetError(msg=msg) from None else: try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{meta.repo}", meta.repo, token + asyncio.run( + git_prepare_remote( + ceph_repo_path, f"github.com/{meta.repo}", meta.repo, token + ) + ) + _ = asyncio.run( + git_fetch_ref(ceph_repo_path, meta.branch, dst_branch, meta.repo) ) - _ = remote.fetch(refspec=f"{meta.branch}:{dst_branch}") except Exception as e: msg = ( f"error fetching patchset branch '{meta.branch}' " @@ -410,7 +418,7 @@ def fetch_custom_patchset_patches( def _cleanup() -> None: for branch in fetched_branches: try: - git_branch_delete(ceph_repo_path, branch) + asyncio.run(git_branch_delete(ceph_repo_path, branch)) except Exception as e: msg = f"unable to delete temporary branch '{branch}': {e}" logger.error(msg) @@ -425,7 +433,7 @@ def _cleanup() -> None: title = patch[1] logger.debug(f"format patch sha '{sha}' title '{title}'") try: - formatted_patch = git_format_patch(ceph_repo_path, sha) + formatted_patch = asyncio.run(git_format_patch(ceph_repo_path, sha)) except GitError as e: _cleanup() msg = f"unable to obtain formatted patch for sha '{sha}': {e}"