Fix plugin installation across development and frozen environments

DashAI/__main__.py

@@ -273,6 +273,14 @@ def main(
     resolved_local = pathlib.Path(local_path).expanduser().absolute()
     os.environ["DASHAI_LOCAL_PATH"] = str(resolved_local)
     os.environ["DASHAI_LOGGING_LEVEL"] = logging_level.value
+
+    # Installed plugins live outside the app environment, so put their
+    # directory on PYTHONPATH before copying the environment for the Huey
+    # consumer: the consumer imports plugin components too.
+    from DashAI.back.plugins.environment import activate_plugins_directory
+
+    activate_plugins_directory(resolved_local)
+
     child_env = os.environ.copy()
 
     logger.info("Starting Huey consumer.")

DashAI/back/api/api_v1/endpoints/plugins.py

@@ -245,6 +245,7 @@ async def update_plugin(
     """
     from DashAI.back.credentials.sync import sync_credentials_status
     from DashAI.back.plugins.utils import (
+        PluginInstallError,
         install_plugin,
         register_plugin_components,
         uninstall_plugin,
@@ -293,6 +294,12 @@ async def update_plugin(
             db.commit()
             db.refresh(plugin)
             return plugin
+        except PluginInstallError as e:
+            logger.exception(e)
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Could not install the plugin: {e}",
+            ) from e
         except exc.SQLAlchemyError as e:
             logger.exception(e)
             raise HTTPException(
@@ -329,6 +336,7 @@ async def upgrade_plugin(
     """
     from DashAI.back.dependencies.database.utils import upgrade_plugin_info_in_db
     from DashAI.back.plugins.utils import (
+        PluginInstallError,
         get_plugin_by_name_from_pypi,
         install_plugin,
         register_plugin_components,
@@ -365,6 +373,12 @@ async def upgrade_plugin(
                 )
 
             return plugin
+        except PluginInstallError as e:
+            logger.exception(e)
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Could not upgrade the plugin: {e}",
+            ) from e
         except exc.SQLAlchemyError as e:
             logger.exception(e)
             raise HTTPException(

DashAI/back/app.py

@@ -1,6 +1,7 @@
 """FastAPI Application module."""
 
 import logging
+import os
 import pathlib
 from typing import Literal, Union
 
@@ -17,6 +18,7 @@
     backfill_explorer_artifacts,
 )
 from DashAI.back.dependencies.database.migrate import migrate_on_startup
+from DashAI.back.plugins.environment import activate_plugins_directory
 from DashAI.back.seeds import seed_datasets_if_first_run
 
 logger = logging.getLogger(__name__)
@@ -65,6 +67,15 @@ def create_app(
     FastAPI
         The created FastAPI application.
     """
+    # Plugins live in a writable per user directory that has to be importable
+    # before the initial components are collected, since building the config
+    # dict already enumerates the installed plugin entry points.
+    if local_path is not None:
+        os.environ["DASHAI_LOCAL_PATH"] = str(
+            pathlib.Path(local_path).expanduser().absolute()
+        )
+    activate_plugins_directory()
+
     # generating config dict and setting logging level
     config = build_config_dict(
         local_path=local_path,

DashAI/back/plugins/environment.py

@@ -0,0 +1,92 @@
+"""Writable location where DashAI plugin distributions live.
+
+Plugins are never installed into the interpreter's own environment. The
+packaged distributions ship a read only environment (a PyInstaller bundle, or
+the squashfs image inside an AppImage), and in a uv managed checkout any
+``uv run`` re-syncs ``.venv`` against ``uv.lock`` and uninstalls everything
+that is not locked, plugins included. Instead every plugin is installed into a
+per user directory that is added to ``sys.path`` on startup.
+
+The directory is scoped by interpreter version because plugins may ship
+compiled extension modules, which are only importable by the CPython version
+they were built for.
+"""
+
+import importlib
+import logging
+import os
+import pathlib
+import site
+import sys
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LOCAL_PATH = "~/.DashAI"
+PLUGINS_DIR_NAME = "plugins"
+
+
+def get_plugins_directory(local_path: Optional[os.PathLike] = None) -> pathlib.Path:
+    """Resolve the directory that holds the installed plugin distributions.
+
+    Parameters
+    ----------
+    local_path : Optional[os.PathLike]
+        Base dashAI data directory. Defaults to the ``DASHAI_LOCAL_PATH``
+        environment variable, and to ``~/.DashAI`` when that is unset.
+
+    Returns
+    -------
+    pathlib.Path
+        Absolute path of the version scoped plugins directory. The directory is
+        not created by this function.
+    """
+    if local_path is None:
+        local_path = os.environ.get("DASHAI_LOCAL_PATH") or DEFAULT_LOCAL_PATH
+
+    base = pathlib.Path(local_path).expanduser().absolute()
+    interpreter = f"py{sys.version_info.major}.{sys.version_info.minor}"
+    return base / PLUGINS_DIR_NAME / interpreter
+
+
+def activate_plugins_directory(
+    local_path: Optional[os.PathLike] = None,
+) -> pathlib.Path:
+    """Create the plugins directory and make it importable.
+
+    The directory is appended to ``sys.path`` (so distributions shipped with
+    dashAI always win over a plugin's copy of the same package) and exported
+    through ``PYTHONPATH`` so that child processes, such as the Huey consumer,
+    see the plugins too. Import caches are invalidated so that plugins
+    installed while the app is running are discoverable without a restart.
+
+    Parameters
+    ----------
+    local_path : Optional[os.PathLike]
+        Base dashAI data directory, forwarded to
+        :func:`get_plugins_directory`.
+
+    Returns
+    -------
+    pathlib.Path
+        Absolute path of the activated plugins directory.
+    """
+    directory = get_plugins_directory(local_path)
+    try:
+        directory.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        logger.exception("Could not create the plugins directory %s", directory)
+        return directory
+
+    path = str(directory)
+    if path not in sys.path:
+        site.addsitedir(path)
+
+    entries = [
+        entry for entry in os.environ.get("PYTHONPATH", "").split(os.pathsep) if entry
+    ]
+    if path not in entries:
+        os.environ["PYTHONPATH"] = os.pathsep.join([path, *entries])
+
+    importlib.invalidate_caches()
+    return directory