diff --git a/cmd/eirctl/run.go b/cmd/eirctl/run.go index b145d86b..bb295db2 100644 --- a/cmd/eirctl/run.go +++ b/cmd/eirctl/run.go @@ -21,6 +21,8 @@ var ( type runFlags struct { showGraphOnly, detailedSummary bool + contextName string + imports []string } type runCmd struct { @@ -52,6 +54,9 @@ func newRunCmd(rootCmd *EirCtlCmd) { if err != nil { return err } + if err := runner.applyImports(conf); err != nil { + return err + } // display selector if nothing is supplied if len(args) == 0 { selected, err := cmdutils.DisplayTaskSelection(rootCmd.ctx, conf, false) @@ -85,6 +90,9 @@ func newRunCmd(rootCmd *EirCtlCmd) { if err != nil { return err } + if err := runner.applyImports(conf); err != nil { + return err + } taskRunner, argsStringer, err := rootCmd.buildTaskRunner(args, conf) if err != nil { return err @@ -92,6 +100,9 @@ func newRunCmd(rootCmd *EirCtlCmd) { if argsStringer.pipelineName == nil { return fmt.Errorf("pipeline: %s is %w", args[0], ErrSpecifiedObjectIsNotFound) } + if runner.flags.contextName != "" { + return errors.New("the context flag can only be used when running a task") + } return runner.runPipeline(argsStringer.pipelineName, taskRunner, conf.Summary) }, }) @@ -108,6 +119,9 @@ func newRunCmd(rootCmd *EirCtlCmd) { if err != nil { return err } + if err := runner.applyImports(conf); err != nil { + return err + } runner.conf = conf taskRunner, argsStringer, err := rootCmd.buildTaskRunner(args, conf) if err != nil { @@ -116,7 +130,7 @@ func newRunCmd(rootCmd *EirCtlCmd) { if argsStringer.taskName == nil { return fmt.Errorf("task: %s is %w", args[0], ErrSpecifiedObjectIsNotFound) } - return runner.runTask(argsStringer.taskName, taskRunner) + return runner.runTask(runner.taskWithContext(argsStringer.taskName), taskRunner) }, }) @@ -138,6 +152,8 @@ func newRunCmd(rootCmd *EirCtlCmd) { rc.PersistentFlags().BoolVarP(&f.showGraphOnly, "graph-only", "", false, "Show only the denormalized graph") rc.PersistentFlags().BoolVarP(&f.detailedSummary, "detailed", "", false, "Show detailed summary, otherwise will be summarised by top level stages only") + rc.PersistentFlags().StringVarP(&f.contextName, "context", "", "", "override the context used when running a task") + rc.PersistentFlags().StringArrayVarP(&f.imports, "import", "", nil, "import an additional config file, can be repeated; entries in imported files take precedence over any existing ones with the same name") rootCmd.Cmd.AddCommand(rc) } @@ -145,11 +161,14 @@ func newRunCmd(rootCmd *EirCtlCmd) { func (r *runCmd) runTarget(taskRunner *runner.TaskRunner, conf *config.Config, argsStringer *argsToStringsMapper) (err error) { if argsStringer.pipelineName != nil { + if r.flags.contextName != "" { + return errors.New("the context flag can only be used when running a task") + } return r.runPipeline(argsStringer.pipelineName, taskRunner, conf.Summary) } if argsStringer.taskName != nil { - if err := r.runTask(argsStringer.taskName, taskRunner); err != nil { + if err := r.runTask(r.taskWithContext(argsStringer.taskName), taskRunner); err != nil { return fmt.Errorf("task `%s` failed: %w", argsStringer.taskOrPipelineName, err) } } @@ -157,6 +176,30 @@ func (r *runCmd) runTarget(taskRunner *runner.TaskRunner, conf *config.Config, a return nil } +// applyImports merges any files supplied via the repeatable --import flag into +// conf, with entries in those files taking precedence over any name clashes. +func (r *runCmd) applyImports(conf *config.Config) error { + if len(r.flags.imports) == 0 { + return nil + } + cl := config.NewConfigLoader(conf) + if _, err := cl.LoadImports(r.flags.imports); err != nil { + return err + } + return nil +} + +func (r *runCmd) taskWithContext(t *task.Task) *task.Task { + if r.flags.contextName == "" { + return t + } + + taskWithContext := task.NewTask(t.Name) + taskWithContext.FromTask(t) + taskWithContext.Context = r.flags.contextName + return taskWithContext +} + func (r *runCmd) runPipeline(g *scheduler.ExecutionGraph, taskRunner *runner.TaskRunner, summary bool) error { sd := scheduler.NewScheduler(taskRunner) defer sd.Finish() diff --git a/cmd/eirctl/run_test.go b/cmd/eirctl/run_test.go index bfea804d..85fcceb2 100644 --- a/cmd/eirctl/run_test.go +++ b/cmd/eirctl/run_test.go @@ -25,9 +25,27 @@ func Test_runCommand(t *testing.T) { t.Run("correct with task specified", func(t *testing.T) { cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/graph.yaml", "run", "task", "graph:task1", "--raw"}, exactOutput: "hello, world!\n"}) }) + t.Run("overrides the context for an implicit task", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/task.yaml", "run", "task:context:override", "--context", "context:env", "--raw"}, exactOutput: "supplied-by-context\n"}) + }) + t.Run("overrides the context for an explicit task", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/task.yaml", "run", "task", "task:context:override", "--context", "context:env", "--raw"}, exactOutput: "supplied-by-context\n"}) + }) + t.Run("import flag overrides an existing context on clash", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/task.yaml", "run", "task", "task:context:override", "--context", "context:env", "--import", "testdata/cli-import-override.yaml", "--raw"}, exactOutput: "supplied-by-cli-import\n"}) + }) + t.Run("import flag adds a new task", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/task.yaml", "run", "task", "task:from:cli:import", "--import", "testdata/cli-import-override.yaml", "--raw"}, exactOutput: "hello from cli import\n"}) + }) + t.Run("import flag not supplied does not affect existing behaviour", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/task.yaml", "run", "task", "task:context:override", "--context", "context:env", "--raw"}, exactOutput: "supplied-by-context\n"}) + }) t.Run("correct with pipeline specified", func(t *testing.T) { cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/graph.yaml", "run", "pipeline", "graph:pipeline1", "--raw"}, output: []string{"hello, world!\n"}}) }) + t.Run("rejects a context override for a pipeline", func(t *testing.T) { + cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"-c", "testdata/graph.yaml", "run", "pipeline", "graph:pipeline1", "--context", "context:env", "--raw"}, errored: true, output: []string{"context flag can only be used when running a task"}}) + }) t.Run("correct prefixed output", func(t *testing.T) { t.Setenv("EIRCTL_CONFIG_FILE", "testdata/graph.yaml") cmdRunTestHelper(t, &cmdRunTestInput{args: []string{"--output=prefixed", "run", "graph:pipeline1"}, output: []string{"graph:task1", "graph:task2", "graph:task3", "hello, world!"}}) diff --git a/cmd/eirctl/testdata/cli-import-override.yaml b/cmd/eirctl/testdata/cli-import-override.yaml new file mode 100644 index 00000000..3ca9b324 --- /dev/null +++ b/cmd/eirctl/testdata/cli-import-override.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=../../../schemas/schema_v1.json +# fixture used to verify entries supplied via `--import` on the CLI +# take precedence over any existing entries with the same name + +contexts: + context:env: + env: + FOO: supplied-by-cli-import + +tasks: + task:from:cli:import: + command: "echo 'hello from cli import'" diff --git a/cmd/eirctl/testdata/task.yaml b/cmd/eirctl/testdata/task.yaml index 31a63823..0d59b6c6 100644 --- a/cmd/eirctl/testdata/task.yaml +++ b/cmd/eirctl/testdata/task.yaml @@ -1,6 +1,9 @@ # yaml-language-server: $schema=../../../schemas/schema_v1.json contexts: + context:env: + env: + FOO: supplied-by-context context:v1: executable: bin: foo @@ -20,6 +23,8 @@ contexts: - HOME - PATH tasks: + task:context:override: + command: "echo '{{ .Env.FOO }}'" task:task1: command: "echo 'This is {{index .ArgsList 0}} argument'" env: diff --git a/docs/import.adoc b/docs/import.adoc index 638466d9..094a2ea9 100644 --- a/docs/import.adoc +++ b/docs/import.adoc @@ -50,3 +50,20 @@ Set `GIT_SSH_PASSPHRASE` when the selected SSH key is encrypted with a passphras === Filesystem Filesystem imports support relative and absolute paths. + +=== Command line + +`eirctl run` accepts a repeatable `--import` flag to bring in additional config files at execution time, without editing the project's config file: + +[source,bash] +---- +eirctl run task1 --import ./local/context-overrides.yaml +eirctl run pipeline pipeline1 --import ./local/context-overrides.yaml --import ./local/extra-tasks.yaml +---- + +This is useful when working with new or unverified contexts during development, avoiding the risk of accidentally committing them to the tracked config file. + +[IMPORTANT] +==== +Unlike the `import:` directive in a config file, which errors on a name clash, entries supplied via `--import` on the command line take precedence over any existing tasks, pipelines, or contexts with the same name. +==== diff --git a/docs/index.adoc b/docs/index.adoc index 8d5123e8..069a3f8c 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -13,6 +13,8 @@ include::installation.adoc[] include::import.adoc[] +include::tasks.adoc[] + include::artifacts.adoc[] include::watchers.adoc[] diff --git a/docs/tasks.adoc b/docs/tasks.adoc new file mode 100644 index 00000000..95016bfb --- /dev/null +++ b/docs/tasks.adoc @@ -0,0 +1,31 @@ +== Running tasks + +Tasks can use a context configured in the eirctl configuration file. When testing +a task in another context, use the `--context` option with `eirctl run`: + +[source,bash] +---- +eirctl run task task1 --context context:v2 +---- + +The option also works with the short form of the command, where the task name +is supplied without the `task` subcommand: + +[source,bash] +---- +eirctl run task1 --context context:v2 +---- + +`--context` overrides the context configured on the selected task for that +invocation. The value must be the name of a context defined in the loaded +configuration, including any files supplied with `--import`. + +The option applies only to tasks. It cannot be used when running a pipeline: + +[source,bash] +---- +eirctl run pipeline pipeline1 --context context:v2 +---- + +The command returns an error when a context override is supplied for a +pipeline. diff --git a/internal/config/config.go b/internal/config/config.go index ed0f89c6..36ad7867 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -68,6 +68,25 @@ func (cfg *Config) merge(src *Config) error { return nil } +// mergeOverride merges src into cfg, with values in src taking precedence +// over any existing entries in cfg on a name clash (e.g. contexts, tasks, pipelines). +// +// Used when merging config supplied via CLI (e.g. `--import`) on top of the +// already loaded config. +func (cfg *Config) mergeOverride(src *Config) error { + defer func() { + if err := recover(); err != nil { + logrus.Error(err) + } + }() + + if err := mergo.Merge(cfg, src, mergo.WithOverride); err != nil { + return err + } + + return nil +} + func buildFromDefinition(def *ConfigDefinition, lc *loaderContext) (cfg *Config, err error) { cfg = NewConfig() diff --git a/internal/config/loader.go b/internal/config/loader.go index 1b1aeba1..8a1bfb9d 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -130,6 +130,39 @@ func (cl *Loader) Load(file string) (*Config, error) { return cl.Validate() } +// LoadImports loads additional standalone config files, e.g. supplied via the +// `--import` CLI flag, and merges them into the already loaded config. +// +// Unlike the `import:` directive in a config file, entries in these files take +// precedence over any existing entries with the same name (contexts, tasks, +// pipelines etc.), allowing CLI supplied imports to override the loaded config. +func (cl *Loader) LoadImports(files []string) (*Config, error) { + for _, file := range files { + if !utils.IsURL(file) && !filepath.IsAbs(file) { + file = path.Join(cl.dir, file) + } + + def, err := cl.load(schema.ImportEntry{Src: file}) + if err != nil { + return nil, err + } + + importedCfg, err := buildFromDefinition(def, &loaderContext{Dir: cl.dir}) + if err != nil { + return nil, err + } + + if err := cl.dst.mergeOverride(importedCfg); err != nil { + return nil, err + } + logrus.Debugf("import %s loaded", file) + } + + cl.dst.Variables.Set("Root", cl.dir) + + return cl.Validate() +} + // LoadGlobalConfig load global config file - ~/.eirctl/config.yaml func (cl *Loader) LoadGlobalConfig() (*Config, error) { if cl.homeDir == "" {