From c260ab554ae39b43703f821a447b087e7def4970 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:10:55 -0400 Subject: [PATCH 01/10] feat: add pip as a runtime dependency Plugins are installed at runtime by shelling out to pip. A uv managed .venv ships no pip at all, and a PyInstaller bundle has no interpreter to bootstrap one with ensurepip, so pip has to be declared explicitly. The floor is 22.2, the first release with "pip install --report". --- pyproject.toml | 5 +++++ uv.lock | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6937816ee..cdb20efda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,11 @@ classifiers = [ ] dependencies = [ "setuptools>=65.0.0,<82", + # Plugins are installed at runtime through "sys.executable -m pip". A uv + # managed .venv ships no pip, and a bundled build has no interpreter to + # bootstrap one, so pip has to be a real dependency. 22.2 is the first + # release with "pip install --report". + "pip>=22.2", "fastapi[all]", "SQLAlchemy", "streaming_form_data", diff --git a/uv.lock b/uv.lock index e22e7bdeb..b1657807c 100644 --- a/uv.lock +++ b/uv.lock @@ -1511,6 +1511,7 @@ dependencies = [ { name = "oslo-concurrency", version = "7.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-6-dashai-cpu' and extra == 'extra-6-dashai-cuda')" }, { name = "pandas" }, { name = "pillow" }, + { name = "pip" }, { name = "plotly" }, { name = "protobuf" }, { name = "pydantic" }, @@ -1611,6 +1612,7 @@ requires-dist = [ { name = "oslo-concurrency" }, { name = "pandas", specifier = "<3.0.0" }, { name = "pillow" }, + { name = "pip", specifier = ">=22.2" }, { name = "plotly" }, { name = "protobuf" }, { name = "pydantic" }, @@ -5283,6 +5285,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pip" +version = "26.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/15/4500e320e6b101ec3b719ae85b697d9940b6cda672bc555bd6016fc60c6f/pip-26.2.1.tar.gz", hash = "sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f", size = 1848877, upload-time = "2026-08-04T22:51:14.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", hash = "sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e", size = 1816632, upload-time = "2026-08-04T22:51:12.472Z" }, +] + [[package]] name = "platformdirs" version = "4.11.0" From 46e2ca70d5d1f452fe269ad2d0f255afea49a537 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:11:14 -0400 Subject: [PATCH 02/10] feat: add a writable plugins directory outside the environment Plugins cannot live in the interpreter's own environment. The packaged distributions are read only (a PyInstaller bundle, the squashfs image inside an AppImage), and in a uv managed checkout every uv run re-syncs .venv against uv.lock and uninstalls whatever is not locked, plugins included. Add /plugins/py. instead, activated on sys.path and exported through PYTHONPATH so child processes such as the Huey consumer see it too. The directory is scoped by interpreter version because plugins may ship compiled extension modules. --- DashAI/back/plugins/environment.py | 92 +++++++++++++++++++ tests/back/plugins/test_plugin_environment.py | 84 +++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 DashAI/back/plugins/environment.py create mode 100644 tests/back/plugins/test_plugin_environment.py diff --git a/DashAI/back/plugins/environment.py b/DashAI/back/plugins/environment.py new file mode 100644 index 000000000..d50d33a4f --- /dev/null +++ b/DashAI/back/plugins/environment.py @@ -0,0 +1,92 @@ +"""Writable location where DashAI plugin distributions live. + +Plugins are never installed into the interpreter's own environment. The +packaged distributions ship a read only environment (a PyInstaller bundle, or +the squashfs image inside an AppImage), and in a uv managed checkout any +``uv run`` re-syncs ``.venv`` against ``uv.lock`` and uninstalls everything +that is not locked, plugins included. Instead every plugin is installed into a +per user directory that is added to ``sys.path`` on startup. + +The directory is scoped by interpreter version because plugins may ship +compiled extension modules, which are only importable by the CPython version +they were built for. +""" + +import importlib +import logging +import os +import pathlib +import site +import sys +from typing import Optional + +logger = logging.getLogger(__name__) + +DEFAULT_LOCAL_PATH = "~/.DashAI" +PLUGINS_DIR_NAME = "plugins" + + +def get_plugins_directory(local_path: Optional[os.PathLike] = None) -> pathlib.Path: + """Resolve the directory that holds the installed plugin distributions. + + Parameters + ---------- + local_path : Optional[os.PathLike] + Base dashAI data directory. Defaults to the ``DASHAI_LOCAL_PATH`` + environment variable, and to ``~/.DashAI`` when that is unset. + + Returns + ------- + pathlib.Path + Absolute path of the version scoped plugins directory. The directory is + not created by this function. + """ + if local_path is None: + local_path = os.environ.get("DASHAI_LOCAL_PATH") or DEFAULT_LOCAL_PATH + + base = pathlib.Path(local_path).expanduser().absolute() + interpreter = f"py{sys.version_info.major}.{sys.version_info.minor}" + return base / PLUGINS_DIR_NAME / interpreter + + +def activate_plugins_directory( + local_path: Optional[os.PathLike] = None, +) -> pathlib.Path: + """Create the plugins directory and make it importable. + + The directory is appended to ``sys.path`` (so distributions shipped with + dashAI always win over a plugin's copy of the same package) and exported + through ``PYTHONPATH`` so that child processes, such as the Huey consumer, + see the plugins too. Import caches are invalidated so that plugins + installed while the app is running are discoverable without a restart. + + Parameters + ---------- + local_path : Optional[os.PathLike] + Base dashAI data directory, forwarded to + :func:`get_plugins_directory`. + + Returns + ------- + pathlib.Path + Absolute path of the activated plugins directory. + """ + directory = get_plugins_directory(local_path) + try: + directory.mkdir(parents=True, exist_ok=True) + except OSError: + logger.exception("Could not create the plugins directory %s", directory) + return directory + + path = str(directory) + if path not in sys.path: + site.addsitedir(path) + + entries = [ + entry for entry in os.environ.get("PYTHONPATH", "").split(os.pathsep) if entry + ] + if path not in entries: + os.environ["PYTHONPATH"] = os.pathsep.join([path, *entries]) + + importlib.invalidate_caches() + return directory diff --git a/tests/back/plugins/test_plugin_environment.py b/tests/back/plugins/test_plugin_environment.py new file mode 100644 index 000000000..3b3cf5210 --- /dev/null +++ b/tests/back/plugins/test_plugin_environment.py @@ -0,0 +1,84 @@ +import os +import pathlib +import sys + +import pytest + +from DashAI.back.plugins.environment import ( + activate_plugins_directory, + get_plugins_directory, +) + +INTERPRETER_DIR = f"py{sys.version_info.major}.{sys.version_info.minor}" + + +@pytest.fixture(autouse=True) +def _restore_import_state(): + original_path = list(sys.path) + original_pythonpath = os.environ.get("PYTHONPATH") + yield + sys.path[:] = original_path + if original_pythonpath is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = original_pythonpath + + +def test_get_plugins_directory_uses_the_local_path_env_var(tmp_path, monkeypatch): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + + assert get_plugins_directory() == tmp_path / "plugins" / INTERPRETER_DIR + + +def test_get_plugins_directory_prefers_the_explicit_local_path(tmp_path, monkeypatch): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path / "ignored")) + + directory = get_plugins_directory(tmp_path / "explicit") + + assert directory == tmp_path / "explicit" / "plugins" / INTERPRETER_DIR + + +def test_get_plugins_directory_falls_back_to_the_home_directory(monkeypatch): + monkeypatch.delenv("DASHAI_LOCAL_PATH", raising=False) + + directory = get_plugins_directory() + + expected = pathlib.Path("~/.DashAI").expanduser().absolute() + assert directory == expected / "plugins" / INTERPRETER_DIR + + +def test_activate_plugins_directory_makes_the_directory_importable( + tmp_path, monkeypatch +): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + monkeypatch.delenv("PYTHONPATH", raising=False) + + directory = activate_plugins_directory() + + assert directory.is_dir() + assert str(directory) in sys.path + assert os.environ["PYTHONPATH"].split(os.pathsep)[0] == str(directory) + + +def test_activate_plugins_directory_keeps_existing_pythonpath_entries( + tmp_path, monkeypatch +): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + monkeypatch.setenv("PYTHONPATH", "/somewhere/else") + + directory = activate_plugins_directory() + + assert os.environ["PYTHONPATH"] == os.pathsep.join( + [str(directory), "/somewhere/else"] + ) + + +def test_activate_plugins_directory_is_idempotent(tmp_path, monkeypatch): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + monkeypatch.delenv("PYTHONPATH", raising=False) + + directory = activate_plugins_directory() + activate_plugins_directory() + + assert sys.path.count(str(directory)) == 1 + assert os.environ["PYTHONPATH"].count(str(directory)) == 1 From a1181269ee35463f0281a77c018e9d1191a65e43 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:11:33 -0400 Subject: [PATCH 03/10] feat: install plugin distributions with a resolve then target pip run pip's --target mode hardcodes ignore_installed, so installing a plugin straight into the plugins directory re-downloads its whole dependency tree, torch included. Split the installation in two phases instead: 1. resolve the requirement against the running environment with pip install --dry-run --report, which lists only what is missing; 2. install exactly those, pinned to the resolved artifact URLs, with --target --no-deps. pip is invoked as "sys.executable -m pip" rather than as a bare pip executable, since a bare pip found on PATH belongs to some unrelated interpreter and would put the plugin where dashAI can never import it. pip uninstall refuses to touch a --target directory, so removal walks the RECORD file of every distribution the plugin owns and that no other installed plugin still needs, tracked in a ledger next to them. --- DashAI/back/plugins/installer.py | 563 ++++++++++++++++++++ tests/back/plugins/test_plugin_installer.py | 416 +++++++++++++++ 2 files changed, 979 insertions(+) create mode 100644 DashAI/back/plugins/installer.py create mode 100644 tests/back/plugins/test_plugin_installer.py diff --git a/DashAI/back/plugins/installer.py b/DashAI/back/plugins/installer.py new file mode 100644 index 000000000..3e91c1714 --- /dev/null +++ b/DashAI/back/plugins/installer.py @@ -0,0 +1,563 @@ +"""Install and remove plugin distributions in the dashAI plugins directory. + +Everything here drives pip through ``sys.executable -m pip`` instead of a bare +``pip`` executable: a uv managed ``.venv`` has no ``pip`` script at all, and in +a frozen build a ``pip`` found on ``PATH`` belongs to some unrelated +interpreter, so the plugin would be installed where dashAI can never import it. + +pip's ``--target`` mode always forces ``--ignore-installed``, so installing a +plugin directly into the plugins directory would re-download its whole +dependency tree, torch included. Installation is therefore split in two phases: + +1. resolve the requirement against the running environment with + ``pip install --dry-run --report``, which reports only the distributions + that are actually missing; +2. install exactly those, pinned to the artifact URLs the resolver picked, with + ``--target --no-deps``. + +``pip uninstall`` refuses to work on a ``--target`` directory, so removal walks +the ``RECORD`` file of every distribution the plugin brought in and that no +other installed plugin still needs. +""" + +import importlib +import importlib.util +import json +import logging +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Dict, List, Optional + +from DashAI.back.plugins.environment import ( + activate_plugins_directory, + get_plugins_directory, +) + +logger = logging.getLogger(__name__) + +LEDGER_FILENAME = ".dashai-plugins.json" +LEDGER_VERSION = 1 +_PIP_TIMEOUT_SECONDS = 60 * 60 + + +class PluginInstallError(RuntimeError): + """Raised when a plugin distribution cannot be installed or removed.""" + + +def canonical_name(name: str) -> str: + """Normalize a distribution name as defined by the packaging specs. + + Parameters + ---------- + name : str + Raw distribution or requirement name. + + Returns + ------- + str + The lowercase, dash separated canonical form. + """ + return re.sub(r"[-_.]+", "-", name).strip().lower() + + +def pip_command() -> List[str]: + """Build the argv prefix that runs pip for the interpreter hosting dashAI. + + Returns + ------- + List[str] + The ``[sys.executable, "-m", "pip"]`` prefix. In a frozen build + ``sys.executable`` is the dashAI launcher, which the bundled runtime + hook makes behave like ``python -m ...``. + + Raises + ------ + PluginInstallError + If pip is not importable from the current interpreter, in which case + there is no way to reach the environment dashAI itself runs from. + """ + if importlib.util.find_spec("pip") is None: + raise PluginInstallError( + "pip is not available in the environment running dashAI, so plugins " + "cannot be installed. Install it with 'uv pip install pip' (or " + "'python -m ensurepip') and try again." + ) + return [sys.executable, "-m", "pip"] + + +def _pip_environment(directory: pathlib.Path) -> Dict[str, str]: + """Build the environment used for pip subprocesses. + + The plugins directory is exported through ``PYTHONPATH`` so pip counts + distributions installed by earlier plugins as already satisfied, and the + user level pip switches that are incompatible with ``--target`` are + neutralized. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + + Returns + ------- + Dict[str, str] + The environment to hand to :func:`subprocess.run`. + """ + environment = os.environ.copy() + entries = [ + entry for entry in environment.get("PYTHONPATH", "").split(os.pathsep) if entry + ] + path = str(directory) + if path not in entries: + entries.insert(0, path) + environment["PYTHONPATH"] = os.pathsep.join(entries) + # --target is rejected together with --user, and pip refuses to install at + # all when the user has PIP_REQUIRE_VIRTUALENV set while dashAI runs from a + # bundle, where sys.prefix is not a virtual environment. + environment["PIP_USER"] = "0" + environment["PIP_REQUIRE_VIRTUALENV"] = "0" + environment.pop("PIP_TARGET", None) + return environment + + +def _format_pip_error(result: subprocess.CompletedProcess) -> str: + """Extract a readable error message from a failed pip run. + + Parameters + ---------- + result : subprocess.CompletedProcess + The finished pip process. + + Returns + ------- + str + The ``ERROR`` lines pip printed, or the tail of its output when pip + failed without printing any. + """ + output = f"{result.stdout or ''}\n{result.stderr or ''}" + errors = [line for line in output.splitlines() if "ERROR" in line] + if errors: + return "\n".join(errors) + tail = "\n".join(output.splitlines()[-10:]).strip() + return tail or f"pip exited with code {result.returncode}" + + +def _run_pip( + arguments: List[str], directory: pathlib.Path +) -> subprocess.CompletedProcess: + """Run a pip subcommand and raise on failure. + + Parameters + ---------- + arguments : List[str] + Arguments appended to the pip argv prefix. + directory : pathlib.Path + The plugins directory, used to build the subprocess environment. + + Returns + ------- + subprocess.CompletedProcess + The finished process. + + Raises + ------ + PluginInstallError + If pip exits with a non zero status. + """ + command = [*pip_command(), *arguments] + logger.debug("Running pip: %s", " ".join(command)) + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=_pip_environment(directory), + timeout=_PIP_TIMEOUT_SECONDS, + check=False, + ) + if result.returncode != 0: + raise PluginInstallError(_format_pip_error(result)) + return result + + +def resolve_missing_distributions( + requirement: str, directory: pathlib.Path +) -> List[Dict[str, str]]: + """Resolve a requirement against the environment dashAI runs in. + + Parameters + ---------- + requirement : str + The plugin requirement, usually a plain PyPI project name. + directory : pathlib.Path + The plugins directory. + + Returns + ------- + List[Dict[str, str]] + One ``{"name", "version", "url"}`` entry per distribution that is not + importable yet. Empty when the requirement is already satisfied. + + Raises + ------ + PluginInstallError + If pip fails, or writes a report that cannot be parsed. + """ + with tempfile.TemporaryDirectory() as workdir: + report_path = pathlib.Path(workdir) / "report.json" + _run_pip( + [ + "install", + "--dry-run", + "--quiet", + "--disable-pip-version-check", + "--report", + str(report_path), + requirement, + ], + directory, + ) + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise PluginInstallError( + f"Could not read the pip resolution report for '{requirement}'." + ) from error + + distributions = [] + for entry in report.get("install", []): + metadata = entry.get("metadata") or {} + url = (entry.get("download_info") or {}).get("url") + if not metadata.get("name") or not url: + continue + distributions.append( + { + "name": canonical_name(metadata["name"]), + "version": metadata.get("version", ""), + "url": url, + } + ) + return distributions + + +def read_ledger(directory: pathlib.Path) -> Dict[str, List[str]]: + """Read the record of which distributions each plugin brought in. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + + Returns + ------- + Dict[str, List[str]] + Canonical plugin name to the canonical names of the distributions it + owns. Empty when the ledger is missing or unreadable. + """ + path = directory / LEDGER_FILENAME + try: + content = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + plugins = content.get("plugins") + if not isinstance(plugins, dict): + return {} + return { + canonical_name(name): list(distributions) + for name, distributions in plugins.items() + } + + +def _write_ledger(directory: pathlib.Path, ledger: Dict[str, List[str]]) -> None: + """Persist the plugin ownership record. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + ledger : Dict[str, List[str]] + Canonical plugin name to the canonical names of the distributions it + owns. + """ + path = directory / LEDGER_FILENAME + payload = {"version": LEDGER_VERSION, "plugins": ledger} + try: + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + except OSError: + logger.exception("Could not write the plugins ledger at %s", path) + + +def install_requirement( + requirement: str, local_path: Optional[os.PathLike] = None +) -> List[str]: + """Install a plugin and its missing dependencies in the plugins directory. + + Parameters + ---------- + requirement : str + The plugin requirement, usually a plain PyPI project name. + local_path : Optional[os.PathLike] + Base dashAI data directory, forwarded to the plugins directory + resolution. + + Returns + ------- + List[str] + Canonical names of the distributions that were installed. Empty when + the requirement was already satisfied. + + Raises + ------ + PluginInstallError + If pip fails in either phase. + """ + directory = activate_plugins_directory(local_path) + distributions = resolve_missing_distributions(requirement, directory) + if not distributions: + logger.info("Plugin '%s' is already satisfied, nothing to install", requirement) + return [] + + _run_pip( + [ + "install", + "--no-cache-dir", + "--disable-pip-version-check", + "--no-deps", + "--upgrade", + "--target", + str(directory), + *[distribution["url"] for distribution in distributions], + ], + directory, + ) + + names = [distribution["name"] for distribution in distributions] + ledger = read_ledger(directory) + ledger[canonical_name(requirement)] = names + _write_ledger(directory, ledger) + + importlib.invalidate_caches() + logger.info("Installed plugin '%s' with distributions %s", requirement, names) + return names + + +def _find_distribution_directory( + directory: pathlib.Path, distribution: str +) -> Optional[pathlib.Path]: + """Locate the ``.dist-info`` directory of an installed distribution. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + distribution : str + Canonical distribution name. + + Returns + ------- + Optional[pathlib.Path] + The ``.dist-info`` path, or None when the distribution is not installed + in the plugins directory. + """ + for candidate in directory.glob("*.dist-info"): + name = candidate.name[: -len(".dist-info")].rsplit("-", 1)[0] + if canonical_name(name) == distribution: + return candidate + return None + + +def _prune_empty_directories(root: pathlib.Path, leaf: pathlib.Path) -> None: + """Delete leftover empty directories under the plugins directory. + + Parameters + ---------- + root : pathlib.Path + Resolved plugins directory, which is never removed. + leaf : pathlib.Path + Directory a removed file used to live in. + """ + cache = leaf / "__pycache__" + if cache.is_dir(): + shutil.rmtree(cache, ignore_errors=True) + + candidate = leaf + while candidate != root and root in candidate.parents: + if not candidate.is_dir() or any(candidate.iterdir()): + return + candidate.rmdir() + candidate = candidate.parent + + +def _resolve_record_entry( + directory: pathlib.Path, relative: str +) -> Optional[pathlib.Path]: + """Map a ``RECORD`` entry onto its location in the plugins directory. + + Wheels record scripts and data files relative to the environment root, as + in ``../../bin/plugin-cli.exe``. A ``--target`` install puts those inside + the target directory instead, so the parent references are dropped before + resolving. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + relative : str + The path as written in ``RECORD``. + + Returns + ------- + Optional[pathlib.Path] + The absolute path to delete, or None when the entry cannot be placed + inside the plugins directory. + """ + parts = [ + part + for part in pathlib.PurePosixPath(relative.replace("\\", "/")).parts + if part not in ("..", ".", "") + ] + if not parts: + return None + target = directory.joinpath(*parts).resolve() + if directory.resolve() not in target.parents: + return None + return target + + +def _remove_distribution(directory: pathlib.Path, distribution: str) -> bool: + """Delete every file a distribution installed in the plugins directory. + + Parameters + ---------- + directory : pathlib.Path + The plugins directory. + distribution : str + Canonical distribution name. + + Returns + ------- + bool + True when the distribution was found in the plugins directory and + removed, False when it is not installed there. + """ + dist_info = _find_distribution_directory(directory, distribution) + if dist_info is None: + return False + + root = directory.resolve() + touched_directories = set() + record = dist_info / "RECORD" + if record.exists(): + for line in record.read_text(encoding="utf-8").splitlines(): + relative = line.split(",", 1)[0].strip() + if not relative: + continue + target = _resolve_record_entry(directory, relative) + if target is None: + continue + touched_directories.add(target.parent) + try: + if target.is_dir(): + shutil.rmtree(target, ignore_errors=True) + else: + target.unlink(missing_ok=True) + except OSError: + logger.exception("Could not delete %s", target) + + shutil.rmtree(dist_info, ignore_errors=True) + + for leaf in sorted(touched_directories, key=lambda path: len(path.parts))[::-1]: + _prune_empty_directories(root, leaf) + return True + + +def _uninstall_from_environment(requirement: str, directory: pathlib.Path) -> None: + """Remove a legacy plugin installed in the interpreter's environment. + + Parameters + ---------- + requirement : str + The plugin requirement to remove. + directory : pathlib.Path + The plugins directory, used to build the subprocess environment. + """ + try: + _run_pip( + ["uninstall", "-y", "--disable-pip-version-check", requirement], + directory, + ) + except PluginInstallError: + logger.warning( + "Plugin '%s' was not found in the dashAI plugins directory and could " + "not be removed from the environment either.", + requirement, + ) + + +def uninstall_requirement( + requirement: str, local_path: Optional[os.PathLike] = None +) -> List[str]: + """Remove a plugin and the dependencies no other plugin needs. + + Parameters + ---------- + requirement : str + The plugin requirement, usually a plain PyPI project name. + local_path : Optional[os.PathLike] + Base dashAI data directory, forwarded to the plugins directory + resolution. + + Returns + ------- + List[str] + Canonical names of the distributions that were removed. + """ + directory = activate_plugins_directory(local_path) + plugin = canonical_name(requirement) + ledger = read_ledger(directory) + owned = ledger.pop(plugin, [plugin]) + still_needed = { + distribution + for distributions in ledger.values() + for distribution in distributions + } + + removed = [] + for distribution in owned: + if distribution in still_needed: + continue + if _remove_distribution(directory, distribution): + removed.append(distribution) + + if not removed: + # Plugins installed before dashAI moved to a per user plugins directory + # still live in the environment's site-packages. + _uninstall_from_environment(requirement, directory) + + _write_ledger(directory, ledger) + importlib.invalidate_caches() + logger.info("Uninstalled plugin '%s', removed distributions %s", plugin, removed) + return removed + + +def get_installed_plugins_directory( + local_path: Optional[os.PathLike] = None, +) -> pathlib.Path: + """Return the plugins directory without creating or activating it. + + Parameters + ---------- + local_path : Optional[os.PathLike] + Base dashAI data directory. + + Returns + ------- + pathlib.Path + The version scoped plugins directory. + """ + return get_plugins_directory(local_path) diff --git a/tests/back/plugins/test_plugin_installer.py b/tests/back/plugins/test_plugin_installer.py new file mode 100644 index 000000000..a2541653a --- /dev/null +++ b/tests/back/plugins/test_plugin_installer.py @@ -0,0 +1,416 @@ +import json +import os +import pathlib +import subprocess +import sys +from typing import Iterable, Tuple +from unittest.mock import Mock, patch + +import pytest + +from DashAI.back.plugins import installer +from DashAI.back.plugins.installer import ( + PluginInstallError, + canonical_name, + install_requirement, + pip_command, + read_ledger, + resolve_missing_distributions, + uninstall_requirement, +) + + +@pytest.fixture(autouse=True) +def _restore_import_state(): + original_path = list(sys.path) + original_pythonpath = os.environ.get("PYTHONPATH") + yield + sys.path[:] = original_path + if original_pythonpath is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = original_pythonpath + + +INTERPRETER_DIR = f"py{sys.version_info.major}.{sys.version_info.minor}" + + +@pytest.fixture +def plugins_dir(tmp_path, monkeypatch) -> pathlib.Path: + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + directory = tmp_path / "plugins" / INTERPRETER_DIR + directory.mkdir(parents=True) + return directory + + +def _completed(returncode: int = 0, stdout: str = "", stderr: str = ""): + return subprocess.CompletedProcess( + args=["pip"], returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def _report(*distributions: Tuple[str, str, str]) -> dict: + return { + "version": "1", + "install": [ + { + "metadata": {"name": name, "version": version}, + "download_info": {"url": url}, + } + for name, version, url in distributions + ], + } + + +def _write_fake_distribution( + directory: pathlib.Path, + name: str, + version: str = "1.0.0", + files: Iterable[str] = ("dummy_pkg/__init__.py",), + extra_record_lines: Iterable[str] = (), +) -> pathlib.Path: + dist_info = directory / f"{name.replace('-', '_')}-{version}.dist-info" + dist_info.mkdir(parents=True) + records = [] + for relative in files: + target = directory / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("", encoding="utf-8") + records.append(f"{relative},sha256=x,0") + records.extend(extra_record_lines) + records.append(f"{dist_info.name}/METADATA,sha256=x,0") + records.append(f"{dist_info.name}/RECORD,,") + (dist_info / "RECORD").write_text("\n".join(records), encoding="utf-8") + (dist_info / "METADATA").write_text( + f"Name: {name}\nVersion: {version}\n", encoding="utf-8" + ) + return dist_info + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("DashAI-Frankenstein", "dashai-frankenstein"), + ("dashai_frankenstein", "dashai-frankenstein"), + ("dashai.frankenstein", "dashai-frankenstein"), + (" DashAI__Plugin ", "dashai-plugin"), + ], +) +def test_canonical_name(raw, expected): + assert canonical_name(raw) == expected + + +def test_pip_command_targets_the_interpreter_running_dashai(): + assert pip_command() == [sys.executable, "-m", "pip"] + + +def test_pip_command_fails_when_pip_is_missing(): + with patch("importlib.util.find_spec", return_value=None): # noqa: SIM117 + with pytest.raises(PluginInstallError, match="pip is not available"): + pip_command() + + +def test_pip_environment_exposes_the_plugins_directory(plugins_dir, monkeypatch): + monkeypatch.setenv("PYTHONPATH", "/elsewhere") + monkeypatch.setenv("PIP_USER", "1") + monkeypatch.setenv("PIP_TARGET", "/somewhere") + + environment = installer._pip_environment(plugins_dir) + + assert environment["PYTHONPATH"].split(os.pathsep) == [ + str(plugins_dir), + "/elsewhere", + ] + assert environment["PIP_USER"] == "0" + assert environment["PIP_REQUIRE_VIRTUALENV"] == "0" + assert "PIP_TARGET" not in environment + + +def test_run_pip_raises_with_the_error_lines(plugins_dir): + failure = _completed(returncode=1, stderr="noise\nERROR: no such package\n") + + with patch("subprocess.run", return_value=failure): # noqa: SIM117 + with pytest.raises(PluginInstallError, match="ERROR: no such package"): + installer._run_pip(["install", "nope"], plugins_dir) + + +def test_run_pip_falls_back_to_the_output_tail(plugins_dir): + failure = _completed(returncode=2, stderr="something went wrong") + + with patch("subprocess.run", return_value=failure): # noqa: SIM117 + with pytest.raises(PluginInstallError, match="something went wrong"): + installer._run_pip(["install", "nope"], plugins_dir) + + +def test_resolve_missing_distributions_reads_the_pip_report(plugins_dir): + report = _report( + ("DashAI_Frankenstein", "1.2.0", "https://files/dashai_frankenstein.whl"), + ("frankenstein-transformer", "1.1.0", "https://files/transformer.whl"), + ) + + def fake_run_pip(arguments, directory): + pathlib.Path(arguments[arguments.index("--report") + 1]).write_text( + json.dumps(report), encoding="utf-8" + ) + return _completed() + + with patch.object(installer, "_run_pip", side_effect=fake_run_pip) as run_pip: + distributions = resolve_missing_distributions( + "dashai-frankenstein", plugins_dir + ) + + arguments = run_pip.call_args.args[0] + assert arguments[:2] == ["install", "--dry-run"] + assert "--target" not in arguments + assert arguments[-1] == "dashai-frankenstein" + assert distributions == [ + { + "name": "dashai-frankenstein", + "version": "1.2.0", + "url": "https://files/dashai_frankenstein.whl", + }, + { + "name": "frankenstein-transformer", + "version": "1.1.0", + "url": "https://files/transformer.whl", + }, + ] + + +def test_resolve_missing_distributions_ignores_incomplete_entries(plugins_dir): + report = {"install": [{"metadata": {"name": "broken"}}, {"download_info": {}}]} + + def fake_run_pip(arguments, directory): + pathlib.Path(arguments[arguments.index("--report") + 1]).write_text( + json.dumps(report), encoding="utf-8" + ) + return _completed() + + with patch.object(installer, "_run_pip", side_effect=fake_run_pip): + assert resolve_missing_distributions("broken", plugins_dir) == [] + + +def test_install_requirement_only_installs_the_missing_distributions(plugins_dir): + distributions = [ + {"name": "dashai-plugin", "version": "1.0", "url": "https://files/plugin.whl"}, + {"name": "extra-dep", "version": "2.0", "url": "https://files/extra.whl"}, + ] + + with ( + patch.object( + installer, "resolve_missing_distributions", return_value=distributions + ), + patch.object(installer, "_run_pip", return_value=_completed()) as run_pip, + ): + installed = install_requirement("DashAI-Plugin") + + assert installed == ["dashai-plugin", "extra-dep"] + arguments = run_pip.call_args.args[0] + assert "--no-deps" in arguments + assert arguments[arguments.index("--target") + 1] == str(plugins_dir) + assert arguments[-2:] == ["https://files/plugin.whl", "https://files/extra.whl"] + assert read_ledger(plugins_dir) == {"dashai-plugin": ["dashai-plugin", "extra-dep"]} + + +def test_install_requirement_is_a_no_op_when_already_satisfied(plugins_dir): + with ( + patch.object(installer, "resolve_missing_distributions", return_value=[]), + patch.object(installer, "_run_pip") as run_pip, + ): + assert install_requirement("dashai-plugin") == [] + + run_pip.assert_not_called() + + +def test_uninstall_requirement_removes_the_recorded_files(plugins_dir): + _write_fake_distribution( + plugins_dir, "dashai-plugin", files=("dashai_plugin/__init__.py",) + ) + installer._write_ledger(plugins_dir, {"dashai-plugin": ["dashai-plugin"]}) + + removed = uninstall_requirement("DashAI-Plugin") + + assert removed == ["dashai-plugin"] + assert not (plugins_dir / "dashai_plugin").exists() + assert list(plugins_dir.glob("*.dist-info")) == [] + assert read_ledger(plugins_dir) == {} + + +def test_uninstall_requirement_keeps_dependencies_other_plugins_need(plugins_dir): + _write_fake_distribution( + plugins_dir, "dashai-plugin", files=("dashai_plugin/__init__.py",) + ) + _write_fake_distribution(plugins_dir, "shared-dep", files=("shared_dep/core.py",)) + installer._write_ledger( + plugins_dir, + { + "dashai-plugin": ["dashai-plugin", "shared-dep"], + "dashai-other": ["shared-dep"], + }, + ) + + removed = uninstall_requirement("dashai-plugin") + + assert removed == ["dashai-plugin"] + assert (plugins_dir / "shared_dep" / "core.py").exists() + assert read_ledger(plugins_dir) == {"dashai-other": ["shared-dep"]} + + +def test_uninstall_requirement_ignores_records_outside_the_plugins_directory( + plugins_dir, +): + escapee = plugins_dir.parent / "escapee.txt" + escapee.write_text("keep me", encoding="utf-8") + _write_fake_distribution( + plugins_dir, + "dashai-plugin", + files=("dashai_plugin/__init__.py",), + extra_record_lines=("../escapee.txt,sha256=x,0",), + ) + installer._write_ledger(plugins_dir, {"dashai-plugin": ["dashai-plugin"]}) + + uninstall_requirement("dashai-plugin") + + assert escapee.exists() + + +def test_uninstall_requirement_removes_console_scripts(plugins_dir): + """Wheels record scripts as ../../bin/name, which --target puts in bin/.""" + script = plugins_dir / "bin" / "plugin-cli.exe" + script.parent.mkdir(parents=True) + script.write_text("", encoding="utf-8") + _write_fake_distribution( + plugins_dir, + "dashai-plugin", + files=("dashai_plugin/__init__.py",), + extra_record_lines=("../../bin/plugin-cli.exe,sha256=x,0",), + ) + installer._write_ledger(plugins_dir, {"dashai-plugin": ["dashai-plugin"]}) + + uninstall_requirement("dashai-plugin") + + assert not script.exists() + + +@pytest.mark.parametrize( + ("relative", "expected"), + [ + ("dashai_plugin/__init__.py", "dashai_plugin/__init__.py"), + ("../../bin/plugin-cli.exe", "bin/plugin-cli.exe"), + (r"..\..\Scripts\plugin.exe", "Scripts/plugin.exe"), + ], +) +def test_resolve_record_entry_stays_inside_the_plugins_directory( + plugins_dir, relative, expected +): + resolved = installer._resolve_record_entry(plugins_dir, relative) + + assert resolved == (plugins_dir / expected).resolve() + + +def test_resolve_record_entry_rejects_entries_it_cannot_place(plugins_dir): + assert installer._resolve_record_entry(plugins_dir, "../..") is None + + +def test_uninstall_requirement_falls_back_to_the_environment(plugins_dir): + with patch.object(installer, "_run_pip", return_value=_completed()) as run_pip: + assert uninstall_requirement("dashai-legacy") == [] + + assert run_pip.call_args.args[0] == [ + "uninstall", + "-y", + "--disable-pip-version-check", + "dashai-legacy", + ] + + +def test_uninstall_requirement_survives_a_failed_environment_removal(plugins_dir): + with patch.object( + installer, "_run_pip", side_effect=PluginInstallError("not installed") + ): + assert uninstall_requirement("dashai-legacy") == [] + + +def test_read_ledger_tolerates_a_corrupted_file(plugins_dir): + (plugins_dir / installer.LEDGER_FILENAME).write_text("{oops", encoding="utf-8") + + assert read_ledger(plugins_dir) == {} + + +def test_read_ledger_normalizes_plugin_names(plugins_dir): + (plugins_dir / installer.LEDGER_FILENAME).write_text( + json.dumps({"version": 1, "plugins": {"DashAI_Plugin": ["Some_Dep"]}}), + encoding="utf-8", + ) + + assert read_ledger(plugins_dir) == {"dashai-plugin": ["Some_Dep"]} + + +def test_install_requirement_reports_pip_failures(plugins_dir): + with ( + patch.object( + installer, + "resolve_missing_distributions", + side_effect=PluginInstallError("ERROR: no matching distribution"), + ), + pytest.raises(PluginInstallError, match="no matching distribution"), + ): + install_requirement("dashai-missing") + + +def test_find_distribution_directory_matches_canonical_names(plugins_dir): + dist_info = _write_fake_distribution(plugins_dir, "DashAI.Weird_Name") + + found = installer._find_distribution_directory(plugins_dir, "dashai-weird-name") + + assert found == dist_info + + +def test_remove_distribution_returns_false_when_not_installed(plugins_dir): + assert installer._remove_distribution(plugins_dir, "absent") is False + + +def test_get_installed_plugins_directory_does_not_create_it(tmp_path, monkeypatch): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path / "fresh")) + + directory = installer.get_installed_plugins_directory() + + assert not directory.exists() + + +def test_run_pip_uses_the_plugins_environment(plugins_dir): + with patch("subprocess.run", return_value=_completed()) as run: + installer._run_pip(["install", "anything"], plugins_dir) + + assert run.call_args.args[0][:3] == [sys.executable, "-m", "pip"] + assert str(plugins_dir) in run.call_args.kwargs["env"]["PYTHONPATH"] + + +def test_install_requirement_activates_the_plugins_directory(tmp_path, monkeypatch): + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + monkeypatch.delenv("PYTHONPATH", raising=False) + + with patch.object(installer, "resolve_missing_distributions", return_value=[]): + install_requirement("dashai-plugin") + + directory = installer.get_installed_plugins_directory() + assert directory.is_dir() + assert str(directory) in sys.path + + +def test_ledger_round_trip(plugins_dir): + installer._write_ledger(plugins_dir, {"a": ["b"]}) + + assert read_ledger(plugins_dir) == {"a": ["b"]} + + +def test_uninstall_requirement_defaults_to_the_plugin_name(plugins_dir): + _write_fake_distribution(plugins_dir, "dashai-plugin") + + assert uninstall_requirement("dashai-plugin") == ["dashai-plugin"] + + +def test_resolve_missing_distributions_reports_unreadable_reports(plugins_dir): + with patch.object(installer, "_run_pip", return_value=Mock()): # noqa: SIM117 + with pytest.raises(PluginInstallError, match="resolution report"): + resolve_missing_distributions("dashai-plugin", plugins_dir) From cb8930fdaf8414f945fced4750bf5e44e831dc8c Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:11:53 -0400 Subject: [PATCH 04/10] fix: stop installing plugins with a bare pip command execute_pip_command ran "pip install " through a PATH lookup and installed into the interpreter's own environment. Neither half survives the move to uv: a uv managed .venv has no pip script, so the call either fails outright or reaches an unrelated global pip whose entry points dashAI never sees, and anything that did land in .venv was deleted by the next uv run. Route install_plugin and uninstall_plugin through the plugins installer instead. get_available_plugins now activates the plugins directory and invalidates the import caches first, so a plugin installed while the app is running is discovered without a restart, and an entry point that fails to import is skipped rather than taking the whole registry down. --- DashAI/back/plugins/utils.py | 107 ++++++++++----------- tests/back/api/test_plugins.py | 10 +- tests/back/plugins/test_plugin_utils.py | 121 +++++++++++++----------- 3 files changed, 119 insertions(+), 119 deletions(-) diff --git a/DashAI/back/plugins/utils.py b/DashAI/back/plugins/utils.py index f972ab9ef..6aedcfd68 100644 --- a/DashAI/back/plugins/utils.py +++ b/DashAI/back/plugins/utils.py @@ -1,11 +1,18 @@ +import importlib import json -import subprocess +import logging import sys from typing import TYPE_CHECKING, List import requests from DashAI.back.core.enums.plugin_tags import PluginTag +from DashAI.back.plugins.environment import activate_plugins_directory +from DashAI.back.plugins.installer import ( + PluginInstallError, + install_requirement, + uninstall_requirement, +) if TYPE_CHECKING: from DashAI.back.dependencies.registry.component_registry import ComponentRegistry @@ -15,6 +22,19 @@ else: from importlib.metadata import entry_points +logger = logging.getLogger(__name__) + +__all__ = [ + "PluginInstallError", + "get_available_plugins", + "get_plugin_by_name_from_pypi", + "get_plugins_from_pypi", + "install_plugin", + "register_plugin_components", + "uninstall_plugin", + "unregister_plugin_components", +] + _PYPI_SIMPLE_JSON_ACCEPT = "application/vnd.pypi.simple.v1+json" _PYPI_SIMPLE_URL = "https://pypi.org/simple/" _REQUEST_TIMEOUT_SECONDS = 15 @@ -201,11 +221,18 @@ def get_available_plugins() -> List[type]: """ Get available DashAI plugins entrypoints + The plugins directory is activated first, so plugins installed while the + app is running are discovered without restarting it. A plugin that fails to + import is skipped instead of taking the whole registry down with it. + Returns ---------- List[type] A list of plugins' classes """ + activate_plugins_directory() + importlib.invalidate_caches() + # Retrieve plugins groups (DashAI components) plugins = entry_points(group="dashai.plugins") @@ -213,59 +240,18 @@ def get_available_plugins() -> List[type]: plugins_list = [] for plugin in plugins: # Retrieve plugin class - plugin_class = plugin.load() + try: + plugin_class = plugin.load() + except Exception: + logger.exception("Could not load the plugin entry point %s", plugin) + continue + if plugin_class in plugins_list: + continue plugins_list.append(plugin_class) return plugins_list -def execute_pip_command(pypi_plugin_name: str, pip_action: str) -> int: - """ - Execute a pip command to install or uninstall a plugin - - Parameters - ---------- - pypi_plugin_name : str - A string with the name of the plugin in pypi to install or uninstall - - pip_action : str - A string with the action to perform. It can be "install" or "uninstall" - - Returns - ---------- - int - The return code of the pip command - - Raises - ---------- - ValueError - If the pip action is not supported - RuntimeError - If the pip command returns an error - """ - if pip_action not in ["install", "uninstall"]: - raise ValueError(f"Pip action {pip_action} not supported") - - args = ["pip", pip_action] - if pip_action == "uninstall": - args.append("-y") - elif pip_action == "install": - args.append("--no-cache-dir") - args.append(pypi_plugin_name) - res = subprocess.run( - args, - stderr=subprocess.PIPE, - text=True, - ) - - if res.returncode != 0: - errors = [line for line in res.stderr.split("\n") if "ERROR" in line] - error_string = "\n".join(errors) - raise RuntimeError(error_string) - - return res.returncode - - def install_plugin(plugin_name: str) -> List[type]: """ Install and register new plugins in component registry @@ -275,12 +261,18 @@ def install_plugin(plugin_name: str) -> List[type]: plugin_name : str A string with the name of the plugin in pypi to install - component_registry : ComponentRegistry - The current app component registry + Returns + ---------- + List[type] + The plugin classes that became available after the installation. + Raises + ---------- + PluginInstallError + If the plugin distribution could not be installed. """ pre_installed_plugins: List[type] = get_available_plugins() - execute_pip_command(plugin_name, "install") + install_requirement(plugin_name) installed_plugins = set(get_available_plugins()) - set(pre_installed_plugins) return installed_plugins @@ -312,14 +304,15 @@ def uninstall_plugin( Parameters ---------- plugin_name : str - A string with the name of the plugin in pypi to install - - component_registry : ComponentRegistry - The current app component registry + A string with the name of the plugin in pypi to uninstall + Returns + ---------- + List[type] + The plugin classes that stopped being available after the removal. """ available_plugins: List[type] = get_available_plugins() - execute_pip_command(plugin_name, "uninstall") + uninstall_requirement(plugin_name) uninstalled_components: List[type] = set(available_plugins) - set( get_available_plugins() ) diff --git a/tests/back/api/test_plugins.py b/tests/back/api/test_plugins.py index 90d0aa1ad..2b978d6db 100644 --- a/tests/back/api/test_plugins.py +++ b/tests/back/api/test_plugins.py @@ -1,4 +1,3 @@ -import subprocess from unittest.mock import Mock, patch from fastapi.testclient import TestClient @@ -199,11 +198,12 @@ def test_get_unexistant_plugin(client: TestClient): def test_patch_plugin(client: TestClient): - with patch("subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess( - args=["pip", "install", "plugin_name"], returncode=0, stderr="" - ) + with patch( + "DashAI.back.plugins.utils.install_requirement", + return_value=["dashai-svc-plugin"], + ) as mock_install: response = client.patch("/api/v1/plugin/1", json={"new_status": 2}) + mock_install.assert_called_once_with("dashai-svc-plugin") assert response.status_code == 200, response.text response = client.get("/api/v1/plugin/1") diff --git a/tests/back/plugins/test_plugin_utils.py b/tests/back/plugins/test_plugin_utils.py index e92e18cb4..8f3eb9387 100644 --- a/tests/back/plugins/test_plugin_utils.py +++ b/tests/back/plugins/test_plugin_utils.py @@ -1,4 +1,3 @@ -import subprocess from abc import ABCMeta from typing import Final from unittest.mock import Mock, patch @@ -11,14 +10,21 @@ from DashAI.back.plugins.utils import ( _get_all_plugins, _is_verified_author, - execute_pip_command, + get_available_plugins, get_plugin_by_name_from_pypi, get_plugins_from_pypi, + install_plugin, uninstall_plugin, unregister_plugin_components, ) +@pytest.fixture(autouse=True) +def _isolated_plugins_directory(tmp_path, monkeypatch): + """Keep the plugins directory the tests activate out of the real home.""" + monkeypatch.setenv("DASHAI_LOCAL_PATH", str(tmp_path)) + + class DummyBaseComponent(ConfigObject, metaclass=ABCMeta): """Dummy base class representing a component""" @@ -201,56 +207,6 @@ def test_is_verified_author(author, author_email, expected): assert _is_verified_author(author, author_email) is expected -def test_execute_pip_install_command(): - subprocess_mock = Mock() - subprocess_mock.returncode = 0 - with patch("subprocess.run", return_value=subprocess_mock) as mock_run: - result = execute_pip_command("dashai-tabular-classification-package", "install") - - assert result == 0 - mock_run.assert_called_once_with( - ["pip", "install", "--no-cache-dir", "dashai-tabular-classification-package"], - stderr=subprocess.PIPE, - text=True, - ) - - -def test_execute_pip_uninstall_command(): - subprocess_mock = Mock() - subprocess_mock.returncode = 0 - with patch("subprocess.run", return_value=subprocess_mock) as mock_run: - result = execute_pip_command( - "dashai-tabular-classification-package", "uninstall" - ) - - assert result == 0 - mock_run.assert_called_once_with( - ["pip", "uninstall", "-y", "dashai-tabular-classification-package"], - stderr=subprocess.PIPE, - text=True, - ) - - -def test_error_execute_pip_command(): - subprocess_mock = Mock() - subprocess_mock.returncode = 1 - subprocess_mock.stderr = "ERROR: ...\nERROR: ..." - - with patch("subprocess.run", return_value=subprocess_mock): # noqa: SIM117 - with pytest.raises(RuntimeError, match="ERROR: ...\nERROR: ..."): - execute_pip_command("dashai-tabular-classification-package", "install") - - -def test_execute_incorrect_pip_command(): - incorrect_pip_action = "incorrect" - with pytest.raises( - ValueError, match=f"Pip action {incorrect_pip_action} not supported" - ): - execute_pip_command( - "dashai-tabular-classification-package", incorrect_pip_action - ) - - def test_uninstall_plugin(): entry_points_mock = Mock() entry_points_mock.side_effect = [ @@ -260,19 +216,70 @@ def test_uninstall_plugin(): ], [Mock(load=lambda: DummyComponent2, name="Plugin2")], ] - execute_pip_command_mock = Mock() - execute_pip_command_mock.return_value = 0 + uninstall_requirement_mock = Mock(return_value=["plugin1"]) with patch("DashAI.back.plugins.utils.entry_points", entry_points_mock): # noqa: SIM117 with patch( - "DashAI.back.plugins.utils.execute_pip_command", execute_pip_command_mock + "DashAI.back.plugins.utils.uninstall_requirement", + uninstall_requirement_mock, ): uninsalled_plugins = uninstall_plugin("Plugin1") assert uninsalled_plugins == {DummyComponent1} - assert execute_pip_command_mock.call_count == 1 assert entry_points_mock.call_count == 2 - execute_pip_command_mock.assert_called_once_with("Plugin1", "uninstall") + uninstall_requirement_mock.assert_called_once_with("Plugin1") + + +def test_install_plugin(): + entry_points_mock = Mock() + entry_points_mock.side_effect = [ + [Mock(load=lambda: DummyComponent1, name="Plugin1")], + [ + Mock(load=lambda: DummyComponent1, name="Plugin1"), + Mock(load=lambda: DummyComponent2, name="Plugin2"), + ], + ] + install_requirement_mock = Mock(return_value=["plugin2"]) + + with patch("DashAI.back.plugins.utils.entry_points", entry_points_mock): # noqa: SIM117 + with patch( + "DashAI.back.plugins.utils.install_requirement", install_requirement_mock + ): + installed_plugins = install_plugin("Plugin2") + + assert installed_plugins == {DummyComponent2} + install_requirement_mock.assert_called_once_with("Plugin2") + + +def test_get_available_plugins_skips_entry_points_that_fail_to_load(): + def broken_load(): + raise ImportError("missing dependency") + + entry_points_mock = Mock( + return_value=[ + Mock(load=broken_load, name="Broken"), + Mock(load=lambda: DummyComponent2, name="Plugin2"), + ] + ) + + with patch("DashAI.back.plugins.utils.entry_points", entry_points_mock): + plugins = get_available_plugins() + + assert plugins == [DummyComponent2] + + +def test_get_available_plugins_deduplicates_repeated_entry_points(): + entry_points_mock = Mock( + return_value=[ + Mock(load=lambda: DummyComponent1, name="Plugin1"), + Mock(load=lambda: DummyComponent1, name="Plugin1"), + ] + ) + + with patch("DashAI.back.plugins.utils.entry_points", entry_points_mock): + plugins = get_available_plugins() + + assert plugins == [DummyComponent1] def test_unregister_plugin_components(): From 8ede68272c0e073d806bfe4ebdd7dbbc5641364c Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:12:10 -0400 Subject: [PATCH 05/10] feat: activate the plugins directory on startup create_app has to make the directory importable before build_config_dict runs, because collecting the initial components already enumerates the installed plugin entry points. The CLI activates it before copying the environment for the Huey consumer, which imports plugin components in its own process and therefore needs the directory on its PYTHONPATH. --- DashAI/__main__.py | 8 ++++++++ DashAI/back/app.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/DashAI/__main__.py b/DashAI/__main__.py index c2bc3717d..47d441528 100644 --- a/DashAI/__main__.py +++ b/DashAI/__main__.py @@ -273,6 +273,14 @@ def main( resolved_local = pathlib.Path(local_path).expanduser().absolute() os.environ["DASHAI_LOCAL_PATH"] = str(resolved_local) os.environ["DASHAI_LOGGING_LEVEL"] = logging_level.value + + # Installed plugins live outside the app environment, so put their + # directory on PYTHONPATH before copying the environment for the Huey + # consumer: the consumer imports plugin components too. + from DashAI.back.plugins.environment import activate_plugins_directory + + activate_plugins_directory(resolved_local) + child_env = os.environ.copy() logger.info("Starting Huey consumer.") diff --git a/DashAI/back/app.py b/DashAI/back/app.py index 328d0af98..b199f8050 100644 --- a/DashAI/back/app.py +++ b/DashAI/back/app.py @@ -1,6 +1,7 @@ """FastAPI Application module.""" import logging +import os import pathlib from typing import Literal, Union @@ -17,6 +18,7 @@ backfill_explorer_artifacts, ) from DashAI.back.dependencies.database.migrate import migrate_on_startup +from DashAI.back.plugins.environment import activate_plugins_directory from DashAI.back.seeds import seed_datasets_if_first_run logger = logging.getLogger(__name__) @@ -65,6 +67,15 @@ def create_app( FastAPI The created FastAPI application. """ + # Plugins live in a writable per user directory that has to be importable + # before the initial components are collected, since building the config + # dict already enumerates the installed plugin entry points. + if local_path is not None: + os.environ["DASHAI_LOCAL_PATH"] = str( + pathlib.Path(local_path).expanduser().absolute() + ) + activate_plugins_directory() + # generating config dict and setting logging level config = build_config_dict( local_path=local_path, From b630e01b646b92f78b89707a4973b6c7705399c6 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:12:28 -0400 Subject: [PATCH 06/10] feat: report plugin install failures to the client A failed install only raised out of the endpoint, so the frontend got a bare Internal Server Error with nothing to act on. Catch PluginInstallError in the install and upgrade paths and return the pip error text as the response detail. --- DashAI/back/api/api_v1/endpoints/plugins.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/DashAI/back/api/api_v1/endpoints/plugins.py b/DashAI/back/api/api_v1/endpoints/plugins.py index 0f35d0783..fa1bb5394 100644 --- a/DashAI/back/api/api_v1/endpoints/plugins.py +++ b/DashAI/back/api/api_v1/endpoints/plugins.py @@ -245,6 +245,7 @@ async def update_plugin( """ from DashAI.back.credentials.sync import sync_credentials_status from DashAI.back.plugins.utils import ( + PluginInstallError, install_plugin, register_plugin_components, uninstall_plugin, @@ -293,6 +294,12 @@ async def update_plugin( db.commit() db.refresh(plugin) return plugin + except PluginInstallError as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Could not install the plugin: {e}", + ) from e except exc.SQLAlchemyError as e: logger.exception(e) raise HTTPException( @@ -329,6 +336,7 @@ async def upgrade_plugin( """ from DashAI.back.dependencies.database.utils import upgrade_plugin_info_in_db from DashAI.back.plugins.utils import ( + PluginInstallError, get_plugin_by_name_from_pypi, install_plugin, register_plugin_components, @@ -365,6 +373,12 @@ async def upgrade_plugin( ) return plugin + except PluginInstallError as e: + logger.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Could not upgrade the plugin: {e}", + ) from e except exc.SQLAlchemyError as e: logger.exception(e) raise HTTPException( From 5909e50d9ddc5e6efd7882c4018a87221b2754d6 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Tue, 18 Aug 2026 10:12:48 -0400 Subject: [PATCH 07/10] feat: make plugin installation work from a frozen build A PyInstaller bundle ships no python executable, so "sys.executable -m pip" re-enters the launcher. Add a runtime hook that answers the two interpreter invocations that path needs, "-m " and a script path, and falls through to the normal CLI for anything else. pip uses the same form to spawn its own build isolation, so both are required. The spec now carries pip itself, plus the dist-info metadata of every dependency collected recursively. Without the metadata the resolution phase considers torch and friends missing and downloads the whole tree into the user's plugins directory. --- dashai.spec | 24 ++++-- hooks/rthook_python_surrogate.py | 36 +++++++++ .../plugins/test_python_surrogate_hook.py | 80 +++++++++++++++++++ 3 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 hooks/rthook_python_surrogate.py create mode 100644 tests/back/plugins/test_python_surrogate_hook.py diff --git a/dashai.spec b/dashai.spec index 6bbf94822..70224644c 100644 --- a/dashai.spec +++ b/dashai.spec @@ -1,10 +1,24 @@ import os import platform import site -from PyInstaller.utils.hooks import collect_all +from PyInstaller.utils.hooks import collect_all, copy_metadata SITEPKG = str(site.getsitepackages()[-1]) +# Plugins are installed at runtime with "sys.executable -m pip" (see +# hooks/rthook_python_surrogate.py), so pip has to travel inside the bundle. +pip_datas, pip_binaries, pip_hiddenimports = collect_all("pip") + +# pip resolves a plugin against what dashAI already ships before downloading +# anything, and that check reads *.dist-info metadata. Without the metadata pip +# considers torch & friends missing and re-downloads the whole dependency tree +# into the user's plugins directory. +try: + dependency_metadata = copy_metadata("dashAI", recursive=True) +except Exception as error: # pragma: no cover - build time diagnostics only + print(f"WARNING: could not collect dependency metadata for plugins: {error}") + dependency_metadata = copy_metadata("dashAI") + # Check for llama_cpp/lib presence to determine if we can include the binaries llama_lib = os.path.exists(os.path.join(SITEPKG, "llama_cpp/lib")) @@ -20,7 +34,7 @@ if not os.path.exists(os.path.join("DashAI/back/user_models/temp_checkpoints")): a = Analysis( platform.system() == "Windows" and ["DashAI/__main__.py"] or ["DashAI/webview.py"], pathex=["."], - binaries=(llama_lib and [(f"{SITEPKG}/llama_cpp/lib/*", "llama_cpp/lib")] or []) + webview_binaries, + binaries=(llama_lib and [(f"{SITEPKG}/llama_cpp/lib/*", "llama_cpp/lib")] or []) + webview_binaries + pip_binaries, datas=[ ("DashAI/__main__.py", "DashAI/__main__.py"), ("DashAI/alembic", "DashAI/alembic"), @@ -39,10 +53,10 @@ a = Analysis( # via inspect.getsource, so these must be shipped as data dirs. (f"{SITEPKG}/diffusers", "diffusers"), (f"{SITEPKG}/controlnet_aux", "controlnet_aux"), - ] + webview_datas, - hiddenimports=webview_hiddenimports, + ] + webview_datas + pip_datas + dependency_metadata, + hiddenimports=webview_hiddenimports + pip_hiddenimports, hookspath=["hooks"], - runtime_hooks=None, + runtime_hooks=["hooks/rthook_python_surrogate.py"], excludes=None, ) pyz = PYZ(a.pure) diff --git a/hooks/rthook_python_surrogate.py b/hooks/rthook_python_surrogate.py new file mode 100644 index 000000000..e427c01af --- /dev/null +++ b/hooks/rthook_python_surrogate.py @@ -0,0 +1,36 @@ +"""Make the frozen dashAI launcher usable as a Python interpreter. + +A PyInstaller bundle ships no ``python`` executable: ``sys.executable`` is the +launcher itself. dashAI installs plugins with ``sys.executable -m pip``, and pip +in turn re-invokes ``sys.executable`` to build isolated wheels, so the launcher +has to honour the two interpreter invocations those paths rely on: + +* ``dashAI -m [args...]`` behaves like ``python -m ...`` +* ``dashAI