From 6dfd348968986ef308066758312df97b50c3eb5c Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 26 Aug 2026 16:23:19 -0400 Subject: [PATCH] Copy pulp-openapi-generator into each agent so generate-client can run in parallel. Agent SRC_DIR is the compiled directory, so the shared host generator checkout cannot be used as a working tree. A per-agent copy keeps generation isolated and still visible at /src/pulp-openapi-generator for install. Co-authored-by: Cursor --- base/local_scripts/generate_client.sh | 2 +- client/oci_env/agent.py | 73 +++++++++++++++++++++++++-- client/oci_env/main.py | 7 +++ docs/dev/guides/agent-envs.md | 19 +++++-- 4 files changed, 93 insertions(+), 8 deletions(-) diff --git a/base/local_scripts/generate_client.sh b/base/local_scripts/generate_client.sh index 3585a23..0571a6f 100755 --- a/base/local_scripts/generate_client.sh +++ b/base/local_scripts/generate_client.sh @@ -10,7 +10,7 @@ echo "generate_client ${API_VERSION}" if [ ! -d "${SRC_DIR}/pulp-openapi-generator/" ] then - echo "Please clone github.com/pulp/pulp-openapi-generator into ${pwd}/pulp-openapi-generator/" + echo "Please clone github.com/pulp/pulp-openapi-generator into ${SRC_DIR}/pulp-openapi-generator/" exit 1 fi diff --git a/client/oci_env/agent.py b/client/oci_env/agent.py index 2cf372c..1c0fd04 100644 --- a/client/oci_env/agent.py +++ b/client/oci_env/agent.py @@ -21,6 +21,7 @@ DEFAULT_PORT_END = 5199 AGENT_ENV_FILENAME = "compose.env" AGENT_OVERLAY_FILENAME = "plugin_volumes_compose.yaml" +GENERATOR_REPO = "pulp-openapi-generator" # Agent environments always use podman (rootless-friendly for parallel stacks). AGENT_COMPOSE_BINARY = "podman" @@ -264,6 +265,53 @@ def encode_plugin_paths(plugin_paths): return ",".join(f"{plugin}={path}" for plugin, path in sorted(plugin_paths.items())) +def generator_checkout_path(host_src_dir): + """Return the real path to pulp-openapi-generator under host_src_dir, or None.""" + if not host_src_dir: + return None + path = os.path.join(host_src_dir, GENERATOR_REPO) + if os.path.isdir(path): + return os.path.realpath(path) + return None + + +def agent_generator_dir(src_dir): + return os.path.join(src_dir, GENERATOR_REPO) + + +def agent_has_generator_copy(src_dir): + """True if create copied pulp-openapi-generator into the agent SRC_DIR.""" + return os.path.isfile(os.path.join(agent_generator_dir(src_dir), "generate.sh")) + + +def copy_generator_checkout(host_src_dir, compiled_dir): + """ + Copy pulp-openapi-generator into the agent compiled dir. + + Skips .git and previously generated *-client trees so each agent gets an + isolated working copy without sharing host generator state. + """ + src = generator_checkout_path(host_src_dir) + if not src: + return None + dest = agent_generator_dir(compiled_dir) + ignore = shutil.ignore_patterns( + ".git", + ".github", + "*-client", + "__pycache__", + ".pytest_cache", + "api.json", + "patched-api.json", + ) + try: + shutil.copytree(src, dest, ignore=ignore) + except OSError as exc: + exit_with_error(f"Failed to copy {GENERATOR_REPO} from {src} to {dest}: {exc}") + logger.info(f"Copied {GENERATOR_REPO} to {dest}") + return dest + + def write_plugin_volume_overlay(overlay_path, plugin_paths): """ Write a compose overlay that bind-mounts plugin checkouts into /src/. @@ -342,6 +390,14 @@ def agent_create(args): agent_volume_overlay_path(oci_env_path, args.agent_id), plugin_paths, ) + generator = copy_generator_checkout(host_src_dir, compiled_dir) + if not generator: + logger.warning( + f"No {GENERATOR_REPO} checkout at " + f"{os.path.join(host_src_dir, GENERATOR_REPO)}. " + "oci-env agent generate-client will fail until you clone it and " + "destroy/recreate this agent." + ) values = { "OCI_AGENT_ID": args.agent_id, @@ -374,6 +430,8 @@ def agent_create(args): print(f" plugins: {':'.join(plugins)}") for plugin in plugins: print(f" {plugin}: {plugin_paths[plugin]}") + if generator: + print(f" {GENERATOR_REPO}: {generator}") for key in sorted(env_overrides.keys()): print(f" {key}={env_overrides[key]}") print(f"Start with: oci-env agent up {args.agent_id}") @@ -428,8 +486,8 @@ def agent_destroy(args): except SystemExit: logger.warning("compose client setup failed; continuing with agent cleanup") - # Remove the entire agent compiled directory (env, overlay, and any files - # written by parse_profiles). Never deletes bind-mount targets. + # Remove the entire agent compiled directory (env, overlay, generator copy, + # and any files written by parse_profiles). Never deletes bind-mount targets. if os.path.isdir(compiled_dir): shutil.rmtree(compiled_dir) logger.info(f"Removed agent compiled directory {compiled_dir}") @@ -522,7 +580,15 @@ class _TestArgs: def agent_generate_client(args): oci_env_path = get_oci_env_path(args.is_verbose) - env_path, _ = load_agent_env(oci_env_path, args.agent_id) + env_path, env_data = load_agent_env(oci_env_path, args.agent_id) + src_dir = env_data.get("SRC_DIR") or agent_compiled_dir(oci_env_path, args.agent_id) + if not agent_has_generator_copy(src_dir): + exit_with_error( + f"{GENERATOR_REPO} was not copied when agent {args.agent_id} was created. " + "Clone github.com/pulp/pulp-openapi-generator into the host source directory, " + "then destroy and recreate the agent." + ) + client = compose_for_agent(args.is_verbose, env_path) from oci_env.commands import generate_client @@ -535,6 +601,7 @@ class _Args: gargs.language = args.language gargs.install_client = args.install_client gargs.is_verbose = args.is_verbose + gargs.api_version = getattr(args, "api_version", "v3") generate_client(gargs, client) diff --git a/client/oci_env/main.py b/client/oci_env/main.py index cc1de81..75c9a9b 100644 --- a/client/oci_env/main.py +++ b/client/oci_env/main.py @@ -274,6 +274,13 @@ def parse_agent_command(subparsers): dest="install_client", help="Deprecated no-op. Clients are always installed into the agent container.", ) + gen.add_argument( + "--api-version", + default="v3", + choices=["v3", "v4"], + dest="api_version", + help="API version to generate the clients for (default: v3).", + ) gen.set_defaults(func=agent_dispatch, agent_action="generate-client") diff --git a/docs/dev/guides/agent-envs.md b/docs/dev/guides/agent-envs.md index 85399c4..4ece123 100644 --- a/docs/dev/guides/agent-envs.md +++ b/docs/dev/guides/agent-envs.md @@ -23,11 +23,15 @@ Each agent owns a single directory under `.compiled/`: .compiled/agent_/ compose.env # agent env file plugin_volumes_compose.yaml # bind mounts for plugin checkouts + pulp-openapi-generator/ # copy of the host generator checkout … # compose files written here on `up` (parse_profiles) ``` `SRC_DIR` for the agent is that directory. Plugin checkouts are always bind-mounted into `/src/` via the volume overlay (never by sharing the host `SRC_DIR` directly). +If `pulp-openapi-generator` exists under the host source directory, `create` copies it +into the agent `SRC_DIR` so each agent generates clients in isolation. That copy appears +at `/src/pulp-openapi-generator` through the base `{SRC_DIR}:/src` mount. ## Prerequisites @@ -48,10 +52,12 @@ Each agent owns a single directory under `.compiled/`: ~/devel/ ├── oci_env ├── pulpcore # includes pulp_file - └── pulp_rpm + ├── pulp_rpm + └── pulp-openapi-generator ``` `pulp_file` ships inside the `pulpcore` checkout — do not list it in `--plugins`. + `pulp-openapi-generator` is required for `oci-env agent generate-client`. ## Typical agent loop @@ -111,10 +117,15 @@ oci-env agent create pr-123 \ ``` `create` always writes a compose volume overlay that bind-mounts each real checkout at -`/src/` (nested under the base `{SRC_DIR}:/src` mount). +`/src/` (nested under the base `{SRC_DIR}:/src` mount). `pulp-openapi-generator` +is copied into the agent `SRC_DIR` when present under `--host-src-dir`; if it is missing, +`create` warns and `generate-client` will fail until you clone it and recreate the agent. -Destroy removes the entire `.compiled/agent_/` directory; it never deletes the plugin -checkouts that were bind-mounted. +`oci-env agent generate-client` runs the generator against that per-agent copy and +installs the resulting packages from `/src/pulp-openapi-generator` inside the container. + +Destroy removes the entire `.compiled/agent_/` directory, including the generator +copy. It never deletes the plugin checkouts that were bind-mounted. ## Parallelism tips