From 09401a08d30cb63fab8c10ff7495568ee1025cbc Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 17 Nov 2026 11:28:16 -0500 Subject: [PATCH 1/2] Add a global agent environment selector --- .github/workflows/ci.yaml | 3 + README.md | 12 +- client/oci_env/agent.py | 153 ++--------------------- client/oci_env/commands.py | 32 ++++- client/oci_env/main.py | 108 ++++------------- client/oci_env/utils.py | 33 ++++- client/tests/test_agent_selection.py | 154 ++++++++++++++++++++++++ docs/dev/guides/agent-envs.md | 68 +++++++++-- docs/dev/guides/create-multiple-envs.md | 5 +- profiles/lean/README.md | 14 ++- 10 files changed, 327 insertions(+), 255 deletions(-) create mode 100644 client/tests/test_agent_selection.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2143cc7..84826c9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -87,6 +87,9 @@ jobs: run: | pip install -e oci_env/client/ + - name: Test client CLI + run: python -m unittest discover -s oci_env/client/tests -v + - name: Build and run the environment run: | oci-env compose build diff --git a/README.md b/README.md index bb1f2df..c8a5a7f 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,11 @@ and the `lean` profile (`oci-env profile docs lean`). ```bash oci-env agent create pr-123 --plugins pulpcore -oci-env agent up pr-123 -oci-env agent test pr-123 -p pulpcore -- -k test_crud_repos +oci-env -a pr-123 compose up -d +oci-env -a pr-123 poll --attempts 30 --wait 10 +oci-env -a pr-123 generate-client pulpcore +oci-env -a pr-123 generate-client pulp_file +oci-env -a pr-123 test -p pulpcore functional -- -k test_crud_repos oci-env agent destroy pr-123 # Or reuse a Cursor / other AI tool worktree for plugins under test: @@ -23,6 +26,11 @@ oci-env agent create pr-123 \ --plugins pulpcore=~/.cursor/worktrees/pulpcore/my-wt ``` +Use `-a AGENT_ID` (or `--agent AGENT_ID`) before the command to select an agent for +`compose`, `exec`, `shell`, `test`, `generate-client`, `poll`, or any other operational +command. The selector loads the agent's environment and worktree mounts. `-a` and `-e` +are mutually exclusive. The `agent` subcommands are `create`, `ls`, and `destroy`. + ## Multiple environments oci-env supports running multiple environments simultaneously. To do this, simply create a new .env file such as: diff --git a/client/oci_env/agent.py b/client/oci_env/agent.py index 1c0fd04..0e4c18e 100644 --- a/client/oci_env/agent.py +++ b/client/oci_env/agent.py @@ -15,7 +15,6 @@ read_env_file, ) - AGENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") DEFAULT_PORT_START = 5100 DEFAULT_PORT_END = 5199 @@ -83,10 +82,14 @@ def list_agent_ids(oci_env_path): def load_agent_env(oci_env_path, agent_id): + validate_agent_id(agent_id) path = agent_env_path(oci_env_path, agent_id) if not os.path.isfile(path): exit_with_error(f"Agent {agent_id!r} does not exist (missing {path})") - return path, read_env_file(path) + data = read_env_file(path) + if data.get("OCI_AGENT_ID") != agent_id: + exit_with_error(f"Agent id in {path} does not match {agent_id!r}") + return path, data def write_env_file(path, values): @@ -336,16 +339,6 @@ def write_plugin_volume_overlay(overlay_path, plugin_paths): return overlay_path -def compose_for_agent(is_verbose, env_file): - client = Compose(is_verbose, env_file) - overlay = client.config.get("OCI_AGENT_VOLUME_OVERLAY") - if overlay and os.path.isfile(overlay): - client.compose_files.append(overlay) - if is_verbose: - logger.info(f"Including agent volume overlay: {overlay}") - return client - - def agent_status(oci_env_path, agent_id, env_data, is_verbose=False): """Return a short status string for an agent environment.""" project = env_data.get("COMPOSE_PROJECT_NAME", agent_project_name(agent_id)) @@ -395,7 +388,7 @@ def agent_create(args): logger.warning( f"No {GENERATOR_REPO} checkout at " f"{os.path.join(host_src_dir, GENERATOR_REPO)}. " - "oci-env agent generate-client will fail until you clone it and " + f"oci-env -a {args.agent_id} generate-client will fail until you clone it and " "destroy/recreate this agent." ) @@ -434,48 +427,17 @@ def agent_create(args): print(f" {GENERATOR_REPO}: {generator}") for key in sorted(env_overrides.keys()): print(f" {key}={env_overrides[key]}") - print(f"Start with: oci-env agent up {args.agent_id}") - - -def agent_up(args): - oci_env_path = get_oci_env_path(args.is_verbose) - env_path, _ = load_agent_env(oci_env_path, args.agent_id) - client = compose_for_agent(args.is_verbose, env_path) - - up_cmd = ["up", "-d"] - if args.build: - up_cmd.append("--build") - rc = client.compose_command(up_cmd, interactive=True) - if rc != 0: - exit(rc) - - client.poll(args.attempts, args.wait) - api = "{}://{}:{}/".format( - client.config["API_PROTOCOL"], - client.config["API_HOST"], - client.config["API_PORT"], - ) - print(f"Agent {args.agent_id} is up at {api}") - - -def agent_down(args): - oci_env_path = get_oci_env_path(args.is_verbose) - env_path, _ = load_agent_env(oci_env_path, args.agent_id) - try: - client = compose_for_agent(args.is_verbose, env_path) - rc = client.compose_command(["down"], interactive=True) - except FileNotFoundError as exc: - exit_with_error(f"compose binary unavailable: {exc}") - exit(rc) + print(f"Start with: oci-env -a {args.agent_id} compose up -d") + print(f"Wait for the API: oci-env -a {args.agent_id} poll --attempts 30 --wait 10") def agent_destroy(args): oci_env_path = get_oci_env_path(args.is_verbose) - env_path, env_data = load_agent_env(oci_env_path, args.agent_id) + env_path, _ = load_agent_env(oci_env_path, args.agent_id) compiled_dir = agent_compiled_dir(oci_env_path, args.agent_id) try: - client = compose_for_agent(args.is_verbose, env_path) + client = Compose(args.is_verbose, env_path) rc = client.compose_command(["down", "--volumes"], interactive=True) if rc != 0: logger.warning( @@ -514,106 +476,11 @@ def agent_ls(args): ) -def agent_test(args): - """ - Run tests against an agent environment. - - Functional tests execute inside the agent container, so they use that - environment's live API (API_PORT/ creds from the agent env file). - """ - oci_env_path = get_oci_env_path(args.is_verbose) - env_path, env_data = load_agent_env(oci_env_path, args.agent_id) - client = compose_for_agent(args.is_verbose, env_path) - - # Expose connection hints for tooling that reads the process environment. - os.environ.setdefault("OCI_AGENT_ID", args.agent_id) - os.environ.setdefault("PULP_API_HOST", env_data.get("API_HOST", "localhost")) - os.environ.setdefault("PULP_API_PORT", str(env_data.get("API_PORT", ""))) - os.environ.setdefault( - "PULP_API_ROOT", - os.environ.get("PULP_API_ROOT", "/pulp/"), - ) - os.environ.setdefault( - "DJANGO_SUPERUSER_USERNAME", - env_data.get("DJANGO_SUPERUSER_USERNAME", "admin"), - ) - os.environ.setdefault( - "DJANGO_SUPERUSER_PASSWORD", - env_data.get("DJANGO_SUPERUSER_PASSWORD", "password"), - ) - - from oci_env.commands import test as run_test - - # Build a namespace compatible with commands.test - class _TestArgs: - pass - - test_args = _TestArgs() - test_args.install_deps = args.install_deps - test_args.test = args.test - test_args.plugin = args.plugin - # Drop a leading "--" separator commonly used before pytest paths. - pytest_args = list(args.args or []) - if pytest_args and pytest_args[0] == "--": - pytest_args = pytest_args[1:] - test_args.args = pytest_args - test_args.privileged = args.privileged - test_args.is_verbose = args.is_verbose - - if not test_args.plugin: - # Default to the first DEV_SOURCE_PATH plugin for agent workflows. - plugins = [p for p in env_data.get("DEV_SOURCE_PATH", "").split(":") if p] - if len(plugins) == 1: - test_args.plugin = plugins[0] - else: - exit_with_error( - "Specify -p PLUGIN for agent test when DEV_SOURCE_PATH has multiple plugins" - ) - - print( - f"Running {test_args.test} tests for {test_args.plugin} on agent {args.agent_id} " - f"(API {env_data.get('API_PROTOCOL', 'http')}://{env_data.get('API_HOST', 'localhost')}:" - f"{env_data.get('API_PORT')})" - ) - run_test(test_args, client) - - -def agent_generate_client(args): - oci_env_path = get_oci_env_path(args.is_verbose) - env_path, env_data = load_agent_env(oci_env_path, args.agent_id) - src_dir = env_data.get("SRC_DIR") or agent_compiled_dir(oci_env_path, args.agent_id) - if not agent_has_generator_copy(src_dir): - exit_with_error( - f"{GENERATOR_REPO} was not copied when agent {args.agent_id} was created. " - "Clone github.com/pulp/pulp-openapi-generator into the host source directory, " - "then destroy and recreate the agent." - ) - - client = compose_for_agent(args.is_verbose, env_path) - - from oci_env.commands import generate_client - - class _Args: - pass - - gargs = _Args() - gargs.plugin = args.plugin - gargs.language = args.language - gargs.install_client = args.install_client - gargs.is_verbose = args.is_verbose - gargs.api_version = getattr(args, "api_version", "v3") - generate_client(gargs, client) - - def agent_dispatch(args): actions = { "create": agent_create, - "up": agent_up, - "down": agent_down, "destroy": agent_destroy, "ls": agent_ls, - "test": agent_test, - "generate-client": agent_generate_client, } action = getattr(args, "agent_action", None) if action not in actions: diff --git a/client/oci_env/commands.py b/client/oci_env/commands.py index 098ec3c..1b1365c 100644 --- a/client/oci_env/commands.py +++ b/client/oci_env/commands.py @@ -1,15 +1,15 @@ -import subprocess import os import pathlib +import subprocess from oci_env.logger import logger -from oci_env.utils import exit_if_failed, exit_with_error from oci_env.templates import profile_templates +from oci_env.utils import exit_if_failed, exit_with_error def compose(args, client): logger.info(f'COMPOSE {args}') - client.compose_command(args.command, interactive=True) + exit_if_failed(client.compose_command(args.command, interactive=True)) def exec(args, client): @@ -67,6 +67,20 @@ def shell(args, client): def test(args, client): + agent_id = client.config.get("OCI_AGENT_ID") + if agent_id: + if not args.plugin: + plugins = [p for p in client.config["DEV_SOURCE_PATH"].split(":") if p] + if len(plugins) != 1: + exit_with_error("Specify -p PLUGIN before the test suite unless the agent has exactly one source plugin") + args.plugin = plugins[0] + + print(f"Running {args.test} tests for {args.plugin} on agent {agent_id}") + + pytest_args = list(args.args or []) + if pytest_args and pytest_args[0] == "--": + pytest_args = pytest_args[1:] + test_script = f"install_{args.test}_requirements.sh" if args.plugin: @@ -91,7 +105,7 @@ def test(args, client): exit_if_failed( client.exec_container_script( f"run_{args.test}_tests.sh", - args=[args.plugin] + args.args, + args=[args.plugin] + pytest_args, interactive=True, privileged=args.privileged, ) @@ -99,6 +113,16 @@ def test(args, client): def generate_client(args, client): + if agent_id := client.config.get("OCI_AGENT_ID"): + from oci_env.agent import agent_has_generator_copy + + if not agent_has_generator_copy(client.config["SRC_DIR"]): + exit_with_error( + f"pulp-openapi-generator was not copied when agent {agent_id!r} was created. " + "Clone github.com/pulp/pulp-openapi-generator into the host source directory, " + "then destroy and recreate the agent." + ) + api_root = client.get_dynaconf_variable("API_ROOT") container_name = client.container_name(service="pulp") diff --git a/client/oci_env/main.py b/client/oci_env/main.py index 75c9a9b..6261bb7 100644 --- a/client/oci_env/main.py +++ b/client/oci_env/main.py @@ -1,22 +1,22 @@ import argparse import os +from oci_env.agent import agent_dispatch, load_agent_env from oci_env.commands import ( compose, - exec, db, - shell, - test, + exec, generate_client, - pulpcore_manager, - profile, + phelper, poll, - phelper + profile, + pulpcore_manager, + shell, + test, ) - -from oci_env.agent import agent_dispatch from oci_env.utils import ( - Compose + Compose, + get_oci_env_path, ) @@ -42,7 +42,9 @@ def get_env_bool(key, default=False): def get_parser(): parser = argparse.ArgumentParser(description='Pulp OCI image developer environment.') parser.add_argument('-v', action='store_true', dest='is_verbose', default=get_env_bool("OCI_VERBOSE"), help="Print extra debug information.") - parser.add_argument('-e', type=str, default="", dest='env_file', help="Specify an environment file to use other than the default.") + target = parser.add_mutually_exclusive_group() + target.add_argument('-e', type=str, default="", dest='env_file', help="Specify an environment file to use other than the default.") + target.add_argument('-a', '--agent', dest='target_agent', metavar='AGENT_ID', help="Run the command against an existing agent environment.") subparsers = parser.add_subparsers() @@ -104,7 +106,7 @@ def parse_test_command(subparsers): parser = subparsers.add_parser('test', help='Run tests and install requirements.') parser.add_argument('test', choices=["functional", "unit", "lint", "performance"]) parser.add_argument('-i', action='store_true', dest='install_deps', help="Deprecated no-op. Test dependencies are always installed.") - parser.add_argument('-p', type=str, default="", dest='plugin', help="Plugin to test. Tests won't run unless this is specified.") + parser.add_argument('-p', type=str, default="", dest='plugin', help="Plugin to test; place before the suite. Defaults to the sole source plugin in an agent environment.") parser.add_argument('args', nargs=argparse.REMAINDER, help='Arguments to pass to pytest. Use -k/-m to select tests (file paths are not supported with --pyargs).') parser.add_argument("--privileged", action="store_true", dest="privileged") parser.set_defaults(func=test) @@ -206,83 +208,13 @@ def parse_agent_command(subparsers): ) create.set_defaults(func=agent_dispatch, agent_action="create") - up = agent_sub.add_parser("up", help="Start an agent environment and wait for the API.") - up.add_argument("agent_id", help="Agent id to start.") - up.add_argument("--build", action="store_true", help="Build images before starting.") - up.add_argument("--attempts", type=int, default=30, help="Poll attempts for API readiness.") - up.add_argument("--wait", type=int, default=10, help="Seconds between poll attempts.") - up.set_defaults(func=agent_dispatch, agent_action="up") - - down = agent_sub.add_parser("down", help="Stop an agent environment (keeps volumes).") - down.add_argument("agent_id", help="Agent id to stop.") - down.set_defaults(func=agent_dispatch, agent_action="down") + ls = agent_sub.add_parser("ls", help="List agent environments.") + ls.set_defaults(func=agent_dispatch, agent_action="ls") destroy = agent_sub.add_parser("destroy", help="Stop an agent environment and remove volumes/env.") destroy.add_argument("agent_id", help="Agent id to destroy.") destroy.set_defaults(func=agent_dispatch, agent_action="destroy") - ls = agent_sub.add_parser("ls", help="List agent environments.") - ls.set_defaults(func=agent_dispatch, agent_action="ls") - - test_cmd = agent_sub.add_parser("test", help="Run tests against an agent environment's live API.") - test_cmd.add_argument("agent_id", help="Agent id to test.") - test_cmd.add_argument( - "test", - nargs="?", - default="functional", - choices=["functional", "unit", "lint", "performance"], - help="Test suite (default: functional).", - ) - test_cmd.add_argument( - "-i", - action="store_true", - dest="install_deps", - help="Deprecated no-op. Test dependencies are always installed.", - ) - test_cmd.add_argument( - "-p", - type=str, - default="", - dest="plugin", - help="Plugin to test (defaults to the sole DEV_SOURCE_PATH plugin).", - ) - test_cmd.add_argument("args", nargs=argparse.REMAINDER, help="Arguments to pass to pytest.") - test_cmd.add_argument("--privileged", action="store_true", dest="privileged") - test_cmd.set_defaults(func=agent_dispatch, agent_action="test") - - gen = agent_sub.add_parser( - "generate-client", - help="Generate (and optionally install) API clients for an agent environment.", - ) - gen.add_argument("agent_id", help="Agent id.") - gen.add_argument( - "plugin", - nargs="?", - default=None, - help="Plugin to generate a client for (default: all in DEV_SOURCE_PATH).", - ) - gen.add_argument( - "-l", - "--language", - default="python", - choices=["python", "ruby"], - help="Client language.", - ) - gen.add_argument( - "-i", - action="store_true", - dest="install_client", - help="Deprecated no-op. Clients are always installed into the agent container.", - ) - gen.add_argument( - "--api-version", - default="v3", - choices=["v3", "v4"], - dest="api_version", - help="API version to generate the clients for (default: v3).", - ) - gen.set_defaults(func=agent_dispatch, agent_action="generate-client") - def main(): parser = get_parser() @@ -292,8 +224,10 @@ def main(): parser.print_help() exit() - # Agent create/ls manage env files themselves and must not require compose.env. + # Agent management handles its own env files and must not require compose.env. if getattr(args, "func", None) is agent_dispatch: + if args.target_agent is not None or args.env_file: + parser.error("-a/--agent and -e select a target for operational commands, not agent management") try: args.func(args) except KeyboardInterrupt: @@ -301,7 +235,11 @@ def main(): exit(1) return - client = Compose(args.is_verbose, args.env_file) + env_file = args.env_file + if args.target_agent is not None: + env_file, _ = load_agent_env(get_oci_env_path(args.is_verbose), args.target_agent) + + client = Compose(args.is_verbose, env_file) try: args.func(args, client) except KeyboardInterrupt: diff --git a/client/oci_env/utils.py b/client/oci_env/utils.py index 084a151..312121e 100644 --- a/client/oci_env/utils.py +++ b/client/oci_env/utils.py @@ -3,7 +3,6 @@ import subprocess import sys import time - from urllib import request from oci_env.logger import logger @@ -124,9 +123,21 @@ def get_config(env_file): # override any defaults that the user set. cfg = {**config, **user_preferences, **constant_vals} - # allow env overrides except for constants + fixed_keys = set(constant_vals) + if user_preferences.get("OCI_AGENT_ID"): + # Selecting an agent must not inherit another stack's identity or mounts + # from the caller's shell. Change these in the agent env file instead. + fixed_keys.update({ + "OCI_AGENT_ID", "COMPOSE_PROJECT_NAME", "COMPOSE_BINARY", "SRC_DIR", + "DEV_SOURCE_PATH", "OCI_AGENT_PLUGIN_PATHS", "OCI_AGENT_VOLUME_OVERLAY", + "OCI_AGENT_HOST_SRC_DIR", "API_HOST", "API_PORT", "API_PROTOCOL", + "NGINX_PORT", "NGINX_SSL_PORT", "API_CONTAINER", "DB_CONTAINER", + "CONTENT_APP_CONTAINER", "WORKER_CONTAINER", + }) + + # Allow env overrides except for constants and agent target settings. for key in sorted(list(cfg.keys())): - if key in os.environ and key not in constant_vals: + if key in os.environ and key not in fixed_keys: val = os.environ.get(key) cfg[key] = val else: @@ -333,9 +344,25 @@ class Compose: def __init__(self, is_verbose, env_file): self.path = get_oci_env_path(is_verbose) self.config = get_config(get_env_file(self.path, env_file)) + overlay = self.config.get("OCI_AGENT_VOLUME_OVERLAY") + agent_id = self.config.get("OCI_AGENT_ID") + if agent_id and not overlay: + exit_with_error(f"Agent {agent_id!r} has no plugin volume overlay configured") + if overlay and not os.path.isfile(overlay): + exit_with_error(f"Agent plugin volume overlay does not exist: {overlay}") + self.compose_files = parse_profiles(self.config) + if overlay: + self.compose_files.append(overlay) self.is_verbose = is_verbose + if agent_id: + logger.info( + f"Using agent {agent_id} (project {self.config['COMPOSE_PROJECT_NAME']}, " + f"API {self.config['API_PROTOCOL']}://{self.config['API_HOST']}:" + f"{self.config['API_PORT']})" + ) + if self.is_verbose: for key in sorted(self.config.keys()): logger.info(f'OCI CFG {key} = {self.config[key]}') diff --git a/client/tests/test_agent_selection.py b/client/tests/test_agent_selection.py new file mode 100644 index 0000000..f0ad948 --- /dev/null +++ b/client/tests/test_agent_selection.py @@ -0,0 +1,154 @@ +import contextlib +import io +import os +import shlex +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from oci_env import main as cli +from oci_env.utils import Compose + + +class AgentSelectionTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.agent_dir = self.root / ".compiled" / "agent_review" + self.agent_dir.mkdir(parents=True) + self.overlay = self.agent_dir / "plugin_volumes_compose.yaml" + self.overlay.write_text("services: {}") + self.env_file = self.agent_dir / "compose.env" + self.config = { + "OCI_AGENT_ID": "review", + "OCI_AGENT_VOLUME_OVERLAY": str(self.overlay), + "COMPOSE_PROJECT_NAME": "agent_review", + "SRC_DIR": str(self.agent_dir), + "DEV_SOURCE_PATH": "pulpcore", + "API_PORT": "5117", + } + self.write_config() + (self.root / "base").mkdir() + (self.root / "profiles").mkdir() + (self.root / "base" / "pulp_config.env").write_text("API_PORT={API_PORT}") + (self.root / "compose.env").write_text("COMPOSE_PROJECT_NAME=default_dev") + self.enterContext( + patch.dict(os.environ, {"OCI_ENV_PATH": str(self.root)}, clear=True) + ) + self.enterContext(contextlib.redirect_stdout(io.StringIO())) + + def write_config(self, **updates): + self.config.update(updates) + self.env_file.write_text( + "".join(f"{key}={value}\n" for key, value in self.config.items()) + ) + + def run_cli(self, command): + with patch.object(sys, "argv", ["oci-env", *shlex.split(command)]): + cli.main() + + def test_agent_selection_preserves_project_mounts_and_api_address(self): + with patch.dict( + os.environ, + { + "COMPOSE_PROJECT_NAME": "wrong_project", + "SRC_DIR": "/wrong/source", + "OCI_AGENT_VOLUME_OVERLAY": "/wrong/overlay", + "API_PORT": "5999", + }, + ): + for selector in ["-a review", "--agent review", f"-e {self.env_file}"]: + with ( + self.subTest(selector=selector), + patch("oci_env.utils.subprocess.call", return_value=0) as call, + ): + self.run_cli(f"{selector} compose up -d") + command = call.call_args.args[0] + self.assertEqual( + command[:3], ["podman-compose", "-p", "agent_review"] + ) + self.assertEqual( + command[-4:], ["-f", str(self.overlay), "up", "-d"] + ) + self.assertEqual( + (self.agent_dir / "combined.env").read_text(), "API_PORT=5117" + ) + self.assertFalse((self.root / ".compiled" / "default_dev").exists()) + + def test_default_target_is_unchanged(self): + with patch("oci_env.utils.subprocess.call", return_value=0) as call: + self.run_cli("compose ps") + command = call.call_args.args[0] + self.assertEqual(command[:3], ["podman-compose", "-p", "default_dev"]) + self.assertNotIn(str(self.overlay), command) + + def test_invalid_agent_never_falls_back_to_default(self): + for agent_id in ["missing", "../review"]: + with ( + self.subTest(agent_id=agent_id), + patch("oci_env.main.Compose") as compose, + ): + with self.assertRaises(SystemExit): + self.run_cli(f"-a {agent_id} compose down") + compose.assert_not_called() + + def test_missing_overlay_stops_before_compiling(self): + self.overlay.unlink() + with patch("oci_env.utils.parse_profiles") as compile_profiles: + with self.assertRaises(SystemExit): + self.run_cli("-a review compose up -d") + compile_profiles.assert_not_called() + + def test_compose_failures_propagate(self): + with ( + patch("oci_env.utils.subprocess.call", return_value=17), + self.assertRaises(SystemExit) as error, + ): + self.run_cli("-a review compose up -d") + self.assertEqual(error.exception.code, 17) + + def test_test_command_preserves_plugin_selection_and_pytest_arguments(self): + cases = [ + ( + "pulpcore", + "unit -- -p no:cacheprovider", + ["pulpcore", "-p", "no:cacheprovider"], + ), + ( + "pulpcore:pulp_python", + "-p pulp_python unit -- -k catalog", + ["pulp_python", "-k", "catalog"], + ), + ] + for plugins, arguments, expected in cases: + self.write_config(DEV_SOURCE_PATH=plugins) + with ( + self.subTest(plugins=plugins), + patch.object( + Compose, + "exec_container_script", + side_effect=[subprocess.CompletedProcess([], 0), 0], + ) as run, + ): + self.run_cli(f"-a review test {arguments}") + self.assertEqual( + run.call_args_list[0].args, ("install_unit_requirements.sh",) + ) + self.assertEqual(run.call_args_list[0].kwargs["args"], expected[:1]) + self.assertEqual(run.call_args_list[1].args, ("run_unit_tests.sh",)) + self.assertEqual(run.call_args_list[1].kwargs["args"], expected) + + def test_multiple_plugins_require_selection_before_installing(self): + self.write_config(DEV_SOURCE_PATH="pulpcore:pulp_python") + with patch.object(Compose, "exec_container_script") as run: + with self.assertRaises(SystemExit): + self.run_cli("-a review test functional") + run.assert_not_called() + + def test_missing_generator_copy_fails_before_container_access(self): + with patch.object(Compose, "exec") as execute: + with self.assertRaises(SystemExit): + self.run_cli("-a review generate-client") + execute.assert_not_called() diff --git a/docs/dev/guides/agent-envs.md b/docs/dev/guides/agent-envs.md index 3754b0e..7432efe 100644 --- a/docs/dev/guides/agent-envs.md +++ b/docs/dev/guides/agent-envs.md @@ -24,7 +24,7 @@ Each agent owns a single directory under `.compiled/`: compose.env # agent env file plugin_volumes_compose.yaml # bind mounts for plugin checkouts pulp-openapi-generator/ # copy of the host generator checkout - … # compose files written here on `up` (parse_profiles) + … # generated compose files and profile settings ``` `SRC_DIR` for the agent is that directory. Plugin checkouts are always bind-mounted into @@ -57,7 +57,7 @@ at `/src/pulp-openapi-generator` through the base `{SRC_DIR}:/src` mount. ``` `pulp_file` ships inside the `pulpcore` checkout — do not list it in `--plugins`. - `pulp-openapi-generator` is required for `oci-env agent generate-client`. + `pulp-openapi-generator` is required for `oci-env -a AGENT_ID generate-client`. ## Typical agent loop @@ -69,14 +69,16 @@ oci-env agent create pr-123 --plugins pulpcore oci-env agent create pr-123 \ --plugins pulpcore=~/.cursor/worktrees/pulpcore/my-wt -# 2. Start and wait for the live API -oci-env agent up pr-123 +# 2. Start the containers, then wait for the live API +oci-env -a pr-123 compose up -d +oci-env -a pr-123 poll --attempts 30 --wait 10 # 3. Generate/install clients for plugins under test -oci-env agent generate-client pr-123 +oci-env -a pr-123 generate-client pulpcore +oci-env -a pr-123 generate-client pulp_file # 4. Run narrowly scoped functional tests against that agent's API -oci-env agent test pr-123 -p pulpcore -- -k test_crud_repos +oci-env -a pr-123 test -p pulpcore functional -- -k test_crud_repos # 5. Tear down containers, volumes, and the agent compiled directory oci-env agent destroy pr-123 @@ -88,6 +90,39 @@ List allocated agents: oci-env agent ls ``` +## Select an agent for a command + +Put `-a AGENT_ID` or `--agent AGENT_ID` before any operational command. It loads the +agent's saved environment file and plugin volume overlay, so commands use the same +container, API port, and worktree mounts: + +```bash +oci-env -a pr-123 compose logs --tail 50 pulp +oci-env -a pr-123 exec printenv API_PORT +oci-env -a pr-123 shell +oci-env -a pr-123 pulpcore-manager showmigrations +oci-env -a pr-123 prestart +oci-env -a pr-123 pstatus + +# Stop the stack but keep its volumes and configuration for later +oci-env -a pr-123 compose down +``` + +An unknown agent fails before running the command. `-a` and `-e` are mutually exclusive; +without either selector, commands use the normal default environment. The `agent` +management commands are only `create`, `ls`, and `destroy`; they do not take a target +selector. + +The selected agent's saved project name, runtime, source paths, service names, and API +address take precedence over shell variables for another environment. Set these values +with `agent create --env` where supported, or edit the agent's `compose.env`. Other +configuration variables retain their usual environment overrides. Each command prints +the selected agent, compose project, and API URL. + +Starting containers and waiting for readiness are separate operations. `compose up -d` +does not poll the API; follow it with `poll` before generating clients or running functional +tests. Service-level restarts still use `prestart`. + ## Create options | Flag | Purpose | @@ -121,7 +156,7 @@ oci-env agent create pr-123 \ is copied into the agent `SRC_DIR` when present under `--host-src-dir`; if it is missing, `create` warns and `generate-client` will fail until you clone it and recreate the agent. -`oci-env agent generate-client` runs the generator against that per-agent copy and +`oci-env -a AGENT_ID generate-client` runs the generator against that per-agent copy and installs the resulting packages from `/src/pulp-openapi-generator` inside the container. Destroy removes the entire `.compiled/agent_/` directory, including the generator @@ -129,24 +164,31 @@ copy. It never deletes the plugin checkouts that were bind-mounted. ## Parallelism tips -- Build images once; agent `up` should not rebuild unless you pass `--build`. +- Build images once; `oci-env -a AGENT_ID compose up -d` reuses them unless you pass `--build`. - Keep `DEV_SOURCE_PATH` limited to plugins under test — that dominates startup cost after workers. -- Prefer scoped pytest paths/markers; full suites multiply host load. +- Prefer scoped pytest markers or `-k` selections; full suites multiply host load. - Destroy agents when finished so ports and volumes are reclaimed. - Default credentials are `admin` / `password` unless overridden with `--env`. ## How tests hit the live API -`oci-env agent test` loads that agent's env file (`API_PORT`, credentials, `SRC_DIR`, project name) -and runs the existing `oci-env test` path inside **that** container. Functional tests therefore +`oci-env -a AGENT_ID test -p PLUGIN SUITE` loads that agent's env file (`API_PORT`, credentials, +`SRC_DIR`, project name) and runs tests inside **that** container. Functional tests therefore exercise the agent's live API, not a shared default on port 5001. +Specify the suite explicitly (`functional`, `unit`, `lint`, or `performance`). Put `-p PLUGIN` +and `--privileged`, when needed, before the suite. Everything after the suite is forwarded +to pytest; a leading `--` separator is removed. If the agent has only one source plugin, +`-p` can be omitted. With multiple source plugins, it is required before any dependencies +are installed. + The functional test runner prints the API URL and auth user before invoking pytest. ## Manual env files -Agent commands write env files to `.compiled/agent_/compose.env`. You can still use the generic -multi-env flow with `-e` if you prefer hand-written files; see +`agent create` writes env files to `.compiled/agent_/compose.env`. Selecting that file +directly with `-e` also loads the agent's plugin overlay. You can still use the generic +multi-env flow with `-e` for hand-written files; see [Run multiple environments](create-multiple-envs.md). ## When not to use lean agents diff --git a/docs/dev/guides/create-multiple-envs.md b/docs/dev/guides/create-multiple-envs.md index 517bfd6..b3347c9 100644 --- a/docs/dev/guides/create-multiple-envs.md +++ b/docs/dev/guides/create-multiple-envs.md @@ -3,8 +3,9 @@ You can running multiple environments simultaneously. For parallel coding agents that need lean, short-lived stacks with automatic port allocation and -git worktrees, prefer [`oci-env agent`](agent-envs.md) with the `lean` profile instead of -hand-writing env files. +git worktrees, use [`oci-env agent create`](agent-envs.md) with the `lean` profile instead of +hand-writing env files. Select an allocated environment with `-a AGENT_ID` before the +command, for example `oci-env -a pr-123 compose up -d`. `-a` and `-e` are mutually exclusive. ## Create an `.env` file diff --git a/profiles/lean/README.md b/profiles/lean/README.md index 85f4f90..872ad07 100644 --- a/profiles/lean/README.md +++ b/profiles/lean/README.md @@ -21,12 +21,15 @@ API_PORT=5101 COMPOSE_PROJECT_NAME=agent_example ``` -Prefer the `oci-env agent` commands to allocate ports/project names and manage lifecycle: +Use `oci-env agent` to allocate ports/project names, then select the environment with `-a`: ```bash oci-env agent create example --plugins pulpcore -oci-env agent up example -oci-env agent test example -p pulpcore functional -k test_crud_repos +oci-env -a example compose up -d +oci-env -a example poll --attempts 30 --wait 10 +oci-env -a example generate-client pulpcore +oci-env -a example generate-client pulp_file +oci-env -a example test -p pulpcore functional -- -k test_crud_repos oci-env agent destroy example ``` @@ -37,6 +40,11 @@ oci-env agent create example \ --plugins pulpcore=/path/to/external/worktree ``` +`-a` also selects the agent for logs, shells, and service restarts, for example +`oci-env -a example compose logs pulp` and `oci-env -a example prestart`. +Use `oci-env -a example compose down` to stop the stack while keeping its volumes and +configuration. `agent destroy` removes them. `agent ls` lists allocated environments. + ## Extra variables - `PULP_WORKERS` From 32ec75cc81fca63859edc664b1470aaf5c7d5f8b Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 17 Nov 2026 11:35:26 -0500 Subject: [PATCH 2/2] Use the standard plugin selection behavior for agents --- client/oci_env/commands.py | 10 ---------- client/oci_env/main.py | 2 +- client/tests/test_agent_selection.py | 24 ++++++++++++++++++------ docs/dev/guides/agent-envs.md | 6 +++--- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/client/oci_env/commands.py b/client/oci_env/commands.py index 1b1365c..173eb79 100644 --- a/client/oci_env/commands.py +++ b/client/oci_env/commands.py @@ -67,16 +67,6 @@ def shell(args, client): def test(args, client): - agent_id = client.config.get("OCI_AGENT_ID") - if agent_id: - if not args.plugin: - plugins = [p for p in client.config["DEV_SOURCE_PATH"].split(":") if p] - if len(plugins) != 1: - exit_with_error("Specify -p PLUGIN before the test suite unless the agent has exactly one source plugin") - args.plugin = plugins[0] - - print(f"Running {args.test} tests for {args.plugin} on agent {agent_id}") - pytest_args = list(args.args or []) if pytest_args and pytest_args[0] == "--": pytest_args = pytest_args[1:] diff --git a/client/oci_env/main.py b/client/oci_env/main.py index 6261bb7..7f66739 100644 --- a/client/oci_env/main.py +++ b/client/oci_env/main.py @@ -106,7 +106,7 @@ def parse_test_command(subparsers): parser = subparsers.add_parser('test', help='Run tests and install requirements.') parser.add_argument('test', choices=["functional", "unit", "lint", "performance"]) parser.add_argument('-i', action='store_true', dest='install_deps', help="Deprecated no-op. Test dependencies are always installed.") - parser.add_argument('-p', type=str, default="", dest='plugin', help="Plugin to test; place before the suite. Defaults to the sole source plugin in an agent environment.") + parser.add_argument('-p', type=str, default="", dest='plugin', help="Plugin to test; place before the suite. Without -p, only test requirements are installed.") parser.add_argument('args', nargs=argparse.REMAINDER, help='Arguments to pass to pytest. Use -k/-m to select tests (file paths are not supported with --pyargs).') parser.add_argument("--privileged", action="store_true", dest="privileged") parser.set_defaults(func=test) diff --git a/client/tests/test_agent_selection.py b/client/tests/test_agent_selection.py index f0ad948..835f0ff 100644 --- a/client/tests/test_agent_selection.py +++ b/client/tests/test_agent_selection.py @@ -113,7 +113,7 @@ def test_test_command_preserves_plugin_selection_and_pytest_arguments(self): cases = [ ( "pulpcore", - "unit -- -p no:cacheprovider", + "-p pulpcore unit -- -p no:cacheprovider", ["pulpcore", "-p", "no:cacheprovider"], ), ( @@ -140,12 +140,24 @@ def test_test_command_preserves_plugin_selection_and_pytest_arguments(self): self.assertEqual(run.call_args_list[1].args, ("run_unit_tests.sh",)) self.assertEqual(run.call_args_list[1].kwargs["args"], expected) - def test_multiple_plugins_require_selection_before_installing(self): - self.write_config(DEV_SOURCE_PATH="pulpcore:pulp_python") - with patch.object(Compose, "exec_container_script") as run: - with self.assertRaises(SystemExit): + def test_without_plugin_only_installs_requirements(self): + for plugins in ["pulpcore", "pulpcore:pulp_python"]: + self.write_config(DEV_SOURCE_PATH=plugins) + with ( + self.subTest(plugins=plugins), + patch.object( + Compose, + "exec_container_script", + return_value=subprocess.CompletedProcess([], 0), + ) as run, + ): self.run_cli("-a review test functional") - run.assert_not_called() + self.assertEqual(run.call_count, len(plugins.split(":"))) + for invocation, plugin in zip(run.call_args_list, plugins.split(":")): + self.assertEqual( + invocation.args, ("install_functional_requirements.sh",) + ) + self.assertEqual(invocation.kwargs["args"], [plugin]) def test_missing_generator_copy_fails_before_container_access(self): with patch.object(Compose, "exec") as execute: diff --git a/docs/dev/guides/agent-envs.md b/docs/dev/guides/agent-envs.md index 7432efe..40e74d4 100644 --- a/docs/dev/guides/agent-envs.md +++ b/docs/dev/guides/agent-envs.md @@ -178,9 +178,9 @@ exercise the agent's live API, not a shared default on port 5001. Specify the suite explicitly (`functional`, `unit`, `lint`, or `performance`). Put `-p PLUGIN` and `--privileged`, when needed, before the suite. Everything after the suite is forwarded -to pytest; a leading `--` separator is removed. If the agent has only one source plugin, -`-p` can be omitted. With multiple source plugins, it is required before any dependencies -are installed. +to pytest; a leading `--` separator is removed. As with the default environment, omitting +`-p` only installs test requirements for all plugins in `DEV_SOURCE_PATH`; it does not run +tests. The functional test runner prints the API URL and auth user before invoking pytest.