From 101a41c8bdb3da04b7dc90ded4e56948b4d607bc Mon Sep 17 00:00:00 2001 From: Yohta Kimura <38206553+rajyan@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:28:42 +0900 Subject: [PATCH 1/2] Add rawsetenv message type for provider plugins Providers can now send rawsetenv messages to inject environment variables into dependent services without the automatic service name prefix. This enables use cases where applications require exact variable names that cannot be altered. Closes #13727 Signed-off-by: Yohta Kimura <38206553+rajyan@users.noreply.github.com> Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/examples/provider.go | 1 + docs/extension.md | 13 ++++++- pkg/compose/plugins.go | 46 ++++++++++++++++------- pkg/e2e/fixtures/providers/rawsetenv.yaml | 13 +++++++ pkg/e2e/providers_test.go | 21 +++++++++++ 5 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 pkg/e2e/fixtures/providers/rawsetenv.yaml diff --git a/docs/examples/provider.go b/docs/examples/provider.go index e8bd898b24b..8fa5635e12b 100644 --- a/docs/examples/provider.go +++ b/docs/examples/provider.go @@ -96,6 +96,7 @@ func up(options options, args []string) { fmt.Printf(`{ "type": "info", "message": "Processing ... %d%%" }%s`, i*100/options.size, lineSeparator) } fmt.Printf(`{ "type": "setenv", "message": "URL=https://magic.cloud/%s" }%s`, servicename, lineSeparator) + fmt.Printf(`{ "type": "rawsetenv", "message": "CLOUD_REGION=us-east-1" }%s`, lineSeparator) } func down(_ *cobra.Command, _ []string) { diff --git a/docs/extension.md b/docs/extension.md index 9b1f40ab4b7..876f923c699 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -56,7 +56,8 @@ JSON messages MUST include a `type` and a `message` attribute. `type` can be either: - `info`: Reports status updates to the user. Compose will render message as the service state in the progress UI - `error`: Lets the user know something went wrong with details about the error. Compose will render the message as the reason for the service failure. -- `setenv`: Lets the plugin tell Compose how dependent services can access the created resource. See next section for further details. +- `setenv`: Lets the plugin tell Compose how dependent services can access the created resource. The variable is automatically prefixed with the service name. See next section for further details. +- `rawsetenv`: Same as `setenv`, but the variable is injected as-is without the service name prefix. Useful when applications require exact variable names that cannot be altered. - `debug`: Those messages could help debugging the provider, but are not rendered to the user by default. They are rendered when Compose is started with `--verbose` flag. ```mermaid @@ -99,6 +100,16 @@ automatically prefixing it with the service name. For example, if `awesomecloud Then the `app` service, which depends on the service managed by the provider, will receive a `DATABASE_URL` environment variable injected into its runtime environment. +When the provider command sends a `rawsetenv` JSON message, Compose injects the variable as-is without any prefix: +```json +{"type": "rawsetenv", "message": "SECRET_KEY=xxx"} +``` +The `app` service will receive `SECRET_KEY` exactly as specified, regardless of the provider service name. +This is useful when injecting secrets or configuration values that must match exact variable names expected by +applications or frameworks. Unlike `setenv`, which avoids collisions through automatic prefixing, `rawsetenv` keys +are the provider's responsibility to keep unique. If multiple providers emit the same `rawsetenv` key, the last one +to run will overwrite previous values. + > __Note:__ The `compose up` provider command _MUST_ be idempotent. If resource is already running, the command _MUST_ set > the same environment variables to ensure consistent configuration of dependent services. diff --git a/pkg/compose/plugins.go b/pkg/compose/plugins.go index 5b64855104c..29358bd478b 100644 --- a/pkg/compose/plugins.go +++ b/pkg/compose/plugins.go @@ -48,10 +48,16 @@ const ( ErrorType = "error" InfoType = "info" SetEnvType = "setenv" + RawSetEnvType = "rawsetenv" DebugType = "debug" providerMetadataDirectory = "compose/providers" ) +type pluginVariables struct { + prefixed types.Mapping + raw types.Mapping +} + var mux sync.Mutex func (s *composeService) runPlugin(ctx context.Context, project *types.Project, service types.ServiceConfig, command string) error { @@ -70,7 +76,7 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project, return nil } - variables, err := s.executePlugin(cmd, command, service) + vars, err := s.executePlugin(cmd, command, service) if err != nil { return err } @@ -84,16 +90,19 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project, for name, s := range project.Services { if _, ok := s.DependsOn[service.Name]; ok { prefix := strings.ToUpper(service.Name) + "_" - for key, val := range variables { + for key, val := range vars.prefixed { s.Environment[prefix+key] = &val } + for key, val := range vars.raw { + s.Environment[key] = &val + } project.Services[name] = s } } return nil } -func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service types.ServiceConfig) (types.Mapping, error) { //nolint:gocyclo +func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service types.ServiceConfig) (pluginVariables, error) { //nolint:gocyclo var action string switch command { case "up": @@ -106,23 +115,26 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty s.events.On(stoppingEvent(service.Name)) action = "stop" default: - return nil, fmt.Errorf("unsupported plugin command: %s", command) + return pluginVariables{}, fmt.Errorf("unsupported plugin command: %s", command) } stdout, err := cmd.StdoutPipe() if err != nil { - return nil, err + return pluginVariables{}, err } err = cmd.Start() if err != nil { - return nil, err + return pluginVariables{}, err } decoder := json.NewDecoder(stdout) defer func() { _ = stdout.Close() }() - variables := types.Mapping{} + vars := pluginVariables{ + prefixed: types.Mapping{}, + raw: types.Mapping{}, + } for { var msg JsonMessage @@ -131,31 +143,37 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty break } if err != nil { - return nil, err + return pluginVariables{}, err } switch msg.Type { case ErrorType: s.events.On(newEvent(service.Name, api.Error, firstLine(msg.Message))) - return nil, errors.New(msg.Message) + return pluginVariables{}, errors.New(msg.Message) case InfoType: s.events.On(newEvent(service.Name, api.Working, firstLine(msg.Message))) case SetEnvType: key, val, found := strings.Cut(msg.Message, "=") if !found { - return nil, fmt.Errorf("invalid response from plugin: %s", msg.Message) + return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) + } + vars.prefixed[key] = val + case RawSetEnvType: + key, val, found := strings.Cut(msg.Message, "=") + if !found { + return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) } - variables[key] = val + vars.raw[key] = val case DebugType: logrus.Debugf("%s: %s", service.Name, msg.Message) default: - return nil, fmt.Errorf("invalid response from plugin: %s", msg.Type) + return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Type) } } err = cmd.Wait() if err != nil { s.events.On(errorEvent(service.Name, err.Error())) - return nil, fmt.Errorf("failed to %s service provider: %s", action, err.Error()) + return pluginVariables{}, fmt.Errorf("failed to %s service provider: %s", action, err.Error()) } switch command { case "up": @@ -165,7 +183,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty case "stop": s.events.On(stoppedEvent(service.Name)) } - return variables, nil + return vars, nil } func (s *composeService) getPluginBinaryPath(provider string) (path string, err error) { diff --git a/pkg/e2e/fixtures/providers/rawsetenv.yaml b/pkg/e2e/fixtures/providers/rawsetenv.yaml new file mode 100644 index 00000000000..1dde88cf4c8 --- /dev/null +++ b/pkg/e2e/fixtures/providers/rawsetenv.yaml @@ -0,0 +1,13 @@ +services: + test: + image: alpine + command: env + depends_on: + - secrets + secrets: + provider: + type: example-provider + options: + name: secrets + type: test1 + size: 1 diff --git a/pkg/e2e/providers_test.go b/pkg/e2e/providers_test.go index aa1e31a0ff5..4f80f3b7adf 100644 --- a/pkg/e2e/providers_test.go +++ b/pkg/e2e/providers_test.go @@ -76,6 +76,27 @@ func TestDependsOnMultipleProviders(t *testing.T) { env := getEnv(res.Combined(), false) assert.Check(t, slices.Contains(env, "PROVIDER1_URL=https://magic.cloud/provider1"), env) assert.Check(t, slices.Contains(env, "PROVIDER2_URL=https://magic.cloud/provider2"), env) + assert.Check(t, slices.Contains(env, "CLOUD_REGION=us-east-1"), env) +} + +func TestProviderRawSetEnv(t *testing.T) { + provider, err := findExecutable("example-provider") + assert.NilError(t, err) + + path := fmt.Sprintf("%s%s%s", os.Getenv("PATH"), string(os.PathListSeparator), filepath.Dir(provider)) + c := NewParallelCLI(t, WithEnv("PATH="+path)) + const projectName = "rawsetenv" + t.Cleanup(func() { + c.cleanupWithDown(t, projectName) + }) + + res := c.RunDockerComposeCmd(t, "-f", "fixtures/providers/rawsetenv.yaml", "--project-name", projectName, "up") + res.Assert(t, icmd.Success) + env := getEnv(res.Combined(), false) + // setenv: prefixed with service name + assert.Check(t, slices.Contains(env, "SECRETS_URL=https://magic.cloud/secrets"), env) + // rawsetenv: injected as-is without prefix + assert.Check(t, slices.Contains(env, "CLOUD_REGION=us-east-1"), env) } func getEnv(out string, run bool) []string { From 16cc102d259fd0ff9578b0c8e4e370a6af60b092 Mon Sep 17 00:00:00 2001 From: Yohta Kimura <38206553+rajyan@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:48:49 +0900 Subject: [PATCH 2/2] Handle rawsetenv collisions with overwrite and warning rawsetenv injects provider variables without the service-name prefix, so a key can collide with a value already set on the dependent service, whether declared by the user in environment or emitted by another provider. Log a warning and overwrite on collision, document the precedence and the non-deterministic ordering between concurrent providers, and cover the user-environment override with an e2e test. Signed-off-by: Yohta Kimura <38206553+rajyan@users.noreply.github.com> Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/extension.md | 11 +++++++--- pkg/compose/plugins.go | 17 ++++++++------ .../providers/rawsetenv-override.yaml | 15 +++++++++++++ pkg/e2e/providers_test.go | 22 ++++++++++++++++++- 4 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 pkg/e2e/fixtures/providers/rawsetenv-override.yaml diff --git a/docs/extension.md b/docs/extension.md index 876f923c699..1234a3b93e4 100644 --- a/docs/extension.md +++ b/docs/extension.md @@ -106,9 +106,14 @@ When the provider command sends a `rawsetenv` JSON message, Compose injects the ``` The `app` service will receive `SECRET_KEY` exactly as specified, regardless of the provider service name. This is useful when injecting secrets or configuration values that must match exact variable names expected by -applications or frameworks. Unlike `setenv`, which avoids collisions through automatic prefixing, `rawsetenv` keys -are the provider's responsibility to keep unique. If multiple providers emit the same `rawsetenv` key, the last one -to run will overwrite previous values. +applications or frameworks. + +Unlike `setenv`, which avoids collisions through automatic prefixing, `rawsetenv` keys are the provider's +responsibility to keep unique. If a `rawsetenv` key collides with a variable already set on the dependent service, +the existing value is overwritten and Compose logs a warning. This includes variables declared by the user in the +service `environment` section as well as values emitted by other providers. Providers that are not linked by a +`depends_on` relationship may run concurrently, so when several of them emit the same `rawsetenv` key the resulting +value is not deterministic. > __Note:__ The `compose up` provider command _MUST_ be idempotent. If resource is already running, the command _MUST_ set > the same environment variables to ensure consistent configuration of dependent services. diff --git a/pkg/compose/plugins.go b/pkg/compose/plugins.go index 29358bd478b..504c31f62d9 100644 --- a/pkg/compose/plugins.go +++ b/pkg/compose/plugins.go @@ -76,7 +76,7 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project, return nil } - vars, err := s.executePlugin(cmd, command, service) + variables, err := s.executePlugin(cmd, command, service) if err != nil { return err } @@ -90,10 +90,13 @@ func (s *composeService) runPlugin(ctx context.Context, project *types.Project, for name, s := range project.Services { if _, ok := s.DependsOn[service.Name]; ok { prefix := strings.ToUpper(service.Name) + "_" - for key, val := range vars.prefixed { + for key, val := range variables.prefixed { s.Environment[prefix+key] = &val } - for key, val := range vars.raw { + for key, val := range variables.raw { + if existing, ok := s.Environment[key]; ok && existing != nil && *existing != val { + logrus.Warnf("provider %q overrides environment variable %q in service %q", service.Name, key, name) + } s.Environment[key] = &val } project.Services[name] = s @@ -131,7 +134,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty decoder := json.NewDecoder(stdout) defer func() { _ = stdout.Close() }() - vars := pluginVariables{ + variables := pluginVariables{ prefixed: types.Mapping{}, raw: types.Mapping{}, } @@ -156,13 +159,13 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty if !found { return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) } - vars.prefixed[key] = val + variables.prefixed[key] = val case RawSetEnvType: key, val, found := strings.Cut(msg.Message, "=") if !found { return pluginVariables{}, fmt.Errorf("invalid response from plugin: %s", msg.Message) } - vars.raw[key] = val + variables.raw[key] = val case DebugType: logrus.Debugf("%s: %s", service.Name, msg.Message) default: @@ -183,7 +186,7 @@ func (s *composeService) executePlugin(cmd *exec.Cmd, command string, service ty case "stop": s.events.On(stoppedEvent(service.Name)) } - return vars, nil + return variables, nil } func (s *composeService) getPluginBinaryPath(provider string) (path string, err error) { diff --git a/pkg/e2e/fixtures/providers/rawsetenv-override.yaml b/pkg/e2e/fixtures/providers/rawsetenv-override.yaml new file mode 100644 index 00000000000..9afc7ec92a8 --- /dev/null +++ b/pkg/e2e/fixtures/providers/rawsetenv-override.yaml @@ -0,0 +1,15 @@ +services: + test: + image: alpine + command: env + environment: + CLOUD_REGION: user-defined-region + depends_on: + - secrets + secrets: + provider: + type: example-provider + options: + name: secrets + type: test1 + size: 1 diff --git a/pkg/e2e/providers_test.go b/pkg/e2e/providers_test.go index 4f80f3b7adf..75470f41fea 100644 --- a/pkg/e2e/providers_test.go +++ b/pkg/e2e/providers_test.go @@ -76,7 +76,6 @@ func TestDependsOnMultipleProviders(t *testing.T) { env := getEnv(res.Combined(), false) assert.Check(t, slices.Contains(env, "PROVIDER1_URL=https://magic.cloud/provider1"), env) assert.Check(t, slices.Contains(env, "PROVIDER2_URL=https://magic.cloud/provider2"), env) - assert.Check(t, slices.Contains(env, "CLOUD_REGION=us-east-1"), env) } func TestProviderRawSetEnv(t *testing.T) { @@ -99,6 +98,27 @@ func TestProviderRawSetEnv(t *testing.T) { assert.Check(t, slices.Contains(env, "CLOUD_REGION=us-east-1"), env) } +func TestProviderRawSetEnvOverridesUserEnv(t *testing.T) { + provider, err := findExecutable("example-provider") + assert.NilError(t, err) + + path := fmt.Sprintf("%s%s%s", os.Getenv("PATH"), string(os.PathListSeparator), filepath.Dir(provider)) + c := NewParallelCLI(t, WithEnv("PATH="+path)) + const projectName = "rawsetenv-override" + t.Cleanup(func() { + c.cleanupWithDown(t, projectName) + }) + + res := c.RunDockerComposeCmd(t, "-f", "fixtures/providers/rawsetenv-override.yaml", "--project-name", projectName, "up") + res.Assert(t, icmd.Success) + env := getEnv(res.Combined(), false) + // rawsetenv overrides a user-defined environment variable + assert.Check(t, slices.Contains(env, "CLOUD_REGION=us-east-1"), env) + assert.Check(t, !slices.Contains(env, "CLOUD_REGION=user-defined-region"), env) + // the override is surfaced to the user rather than happening silently + assert.Check(t, strings.Contains(res.Combined(), "overrides environment variable"), res.Combined()) +} + func getEnv(out string, run bool) []string { var env []string scanner := bufio.NewScanner(strings.NewReader(out))