diff --git a/src/specify_cli/workflows/engine.py b/src/specify_cli/workflows/engine.py index 14aaf80a69..8c58c53313 100644 --- a/src/specify_cli/workflows/engine.py +++ b/src/specify_cli/workflows/engine.py @@ -1190,6 +1190,9 @@ def execute( ) self._execution_state.set(state) + # Validate inputs before creating or replacing any run artifacts. + resolved_inputs = self._resolve_inputs(definition, inputs or {}) + # Persist the definition the run was started with. Storage contract # (see RunState.definition_path): read by resume and by # `specify workflow definition`; do not change location or content. @@ -1197,8 +1200,6 @@ def execute( with open(state.definition_path, "w", encoding="utf-8") as f: yaml.safe_dump(definition.data, f, sort_keys=False, allow_unicode=True) - # Resolve inputs - resolved_inputs = self._resolve_inputs(definition, inputs or {}) state.inputs = resolved_inputs workflow_dir = workflow_dir_for(definition) state.workflow_dir = workflow_dir diff --git a/tests/specify_cli/workflows/test_command_run.py b/tests/specify_cli/workflows/test_command_run.py index 316af276b5..e6ae075819 100644 --- a/tests/specify_cli/workflows/test_command_run.py +++ b/tests/specify_cli/workflows/test_command_run.py @@ -5,6 +5,8 @@ import json import os import shutil +import subprocess +import sys from pathlib import Path import pytest @@ -13,6 +15,176 @@ from tests.specify_cli.workflows.helpers import force_gate_stdin as _force_gate_stdin +class TestWorkflowStartupArtifacts: + @pytest.fixture(params=["missing", "numeric", "enum"]) + def invalid_input(self, request): + return { + "missing": ({"type": "string", "required": True}, [], "Required input"), + "numeric": ({"type": "number"}, ["--input", "value=invalid"], "expected a number"), + "enum": ({"type": "string", "enum": ["yes"]}, ["--input", "value=no"], "not in allowed"), + }[request.param] + + @pytest.fixture + def startup_workflows(self, project_dir, monkeypatch): + monkeypatch.chdir(project_dir) + monkeypatch.setenv("SPECKIT_WORKFLOW_RUN_ID", "reused-startup") + _force_gate_stdin(monkeypatch, tty=False) + original = project_dir / "original.yml" + original.write_text(yaml.safe_dump({ + "schema_version": "1.0", + "workflow": {"id": "original", "name": "Original", "version": "1.0.0"}, + "inputs": {"verdict": {"type": "string"}}, + "steps": [ + {"id": "prefix", "type": "shell", "run": "echo prefix>>prefix.txt"}, + {"id": "review", "type": "gate", "message": "Review", "verdict_input": "verdict"}, + {"id": "suffix", "type": "shell", "run": "echo suffix>>suffix.txt"}, + ], + }), encoding="utf-8") + return original + + def _replacement(self, project_dir, input_definition): + path = project_dir / "replacement.yml" + path.write_text(yaml.safe_dump({ + "schema_version": "1.0", + "workflow": {"id": "replacement", "name": "Replacement", "version": "1.0.0"}, + "inputs": {"value": input_definition}, + "steps": [ + {"id": "replacement", "type": "shell", "run": "echo ran>replacement.txt"}, + {"id": "replacement-after", "type": "shell", "run": "echo after>replacement-after.txt"}, + ], + }), encoding="utf-8") + return path + + def _invoke(self, project_dir, arguments, transport="runner"): + if transport == "subprocess": + executable = shutil.which("specify", path=str(Path(sys.executable).parent)) + assert executable is not None, "checkout's specify executable is required" + result = subprocess.run( + [str(Path(executable).absolute()), "workflow", *arguments], + cwd=project_dir, input="", + capture_output=True, text=True, timeout=30, + ) + return result.returncode, result.stdout, result.stderr + + from typer.testing import CliRunner + + from specify_cli import app + + result = CliRunner().invoke(app, ["workflow", *arguments], catch_exceptions=False) + return result.exit_code, result.stdout, result.stderr + + def _artifacts(self, run_dir): + return { + str(path.relative_to(run_dir)): path.read_bytes() if path.is_file() else None + for path in run_dir.rglob("*") + } + + @pytest.mark.parametrize("transport", ["runner", "subprocess"]) + def test_invalid_startup_preserves_paused_run_and_resume( + self, project_dir, startup_workflows, invalid_input, transport + ): + code, stdout, stderr = self._invoke( + project_dir, ["run", str(startup_workflows), "--json"], transport + ) + assert code == 0, (stdout, stderr) + paused = json.loads(stdout) + assert paused["status"] == "paused" + assert paused["run_id"] == "reused-startup" + run_dir = project_dir / ".specify" / "workflows" / "runs" / paused["run_id"] + before = self._artifacts(run_dir) + assert {"workflow.yml", "state.json", "inputs.json", "log.jsonl"} <= before.keys() + input_definition, arguments, diagnostic = invalid_input + replacement = self._replacement(project_dir, input_definition) + + code, stdout, stderr = self._invoke( + project_dir, ["run", str(replacement), "--json", *arguments], transport + ) + + assert code == 1 + assert stdout == "" + assert diagnostic in stderr + assert "Traceback" not in stderr + after = self._artifacts(run_dir) + assert not (project_dir / "replacement.txt").exists() + assert not (project_dir / "replacement-after.txt").exists() + # Attempt resume even on the broken baseline to expose snapshot damage. + resume_code, resume_stdout, resume_stderr = self._invoke( + project_dir, ["resume", paused["run_id"], "--input", "verdict=approve", "--json"], + transport, + ) + changed = sorted(key for key in before.keys() | after.keys() if before.get(key) != after.get(key)) + assert before.keys() == after.keys() + assert changed == [], f"Changed artifacts: {changed}; resume: {resume_code}, {resume_stdout}, {resume_stderr}" + assert resume_code == 0, (resume_stdout, resume_stderr) + resumed = json.loads(resume_stdout) + assert resumed["status"] == "completed" + assert resumed["workflow_id"] == "original" + assert resumed["run_id"] == paused["run_id"] + assert (project_dir / "prefix.txt").read_text().splitlines() == ["prefix"] + assert (project_dir / "suffix.txt").read_text().splitlines() == ["suffix"] + + @pytest.mark.parametrize("json_output", [True, False]) + @pytest.mark.parametrize("automatic_id", [True, False]) + def test_invalid_startup_creates_no_run_directory( + self, project_dir, startup_workflows, invalid_input, monkeypatch, + json_output, automatic_id, + ): + if automatic_id: + monkeypatch.delenv("SPECKIT_WORKFLOW_RUN_ID") + input_definition, arguments, diagnostic = invalid_input + replacement = self._replacement(project_dir, input_definition) + code, stdout, stderr = self._invoke( + project_dir, ["run", str(replacement), *(["--json"] if json_output else []), *arguments] + ) + + assert code == 1 + if json_output: + assert stdout == "" + assert diagnostic in stderr + else: + assert "Running workflow:" in stdout + assert diagnostic in stdout + assert stderr == "" + assert "▸" not in stdout + assert "Status:" not in stdout + assert "Traceback" not in stdout + stderr + assert not (project_dir / "replacement.txt").exists() + assert not (project_dir / ".specify" / "workflows" / "runs").exists() + + @pytest.mark.parametrize("initial_status", ["paused", "completed"]) + def test_successful_startup_reuses_run_id( + self, project_dir, startup_workflows, initial_status + ): + arguments = ["--input", "verdict=approve"] if initial_status == "completed" else [] + code, stdout, stderr = self._invoke( + project_dir, ["run", str(startup_workflows), "--json", *arguments] + ) + assert code == 0, (stdout, stderr) + original = json.loads(stdout) + assert original["status"] == initial_status + run_dir = project_dir / ".specify" / "workflows" / "runs" / original["run_id"] + previous_log = (run_dir / "log.jsonl").read_bytes() + replacement = self._replacement(project_dir, {"type": "number", "default": "7"}) + + code, stdout, stderr = self._invoke(project_dir, ["run", str(replacement), "--json"]) + + assert code == 0, (stdout, stderr) + outcome = json.loads(stdout) + assert outcome["run_id"] == original["run_id"] + assert outcome["workflow_id"] == "replacement" + assert outcome["status"] == "completed" + assert (project_dir / "replacement.txt").read_text().splitlines() == ["ran"] + assert yaml.safe_load((run_dir / "workflow.yml").read_text())["workflow"]["id"] == "replacement" + state = json.loads((run_dir / "state.json").read_text()) + assert state["workflow_id"] == "replacement" + assert state["inputs"] == {"value": 7} + assert set(state["step_results"]) == {"replacement", "replacement-after"} + assert json.loads((run_dir / "inputs.json").read_text()) == {"inputs": {"value": 7}} + current_log = (run_dir / "log.jsonl").read_bytes() + assert current_log.startswith(previous_log) + assert len(current_log) > len(previous_log) + + class TestWorkflowRunNumberInputs: @pytest.mark.parametrize("source", ["cli", "default", "enum-default"]) def test_run_preserves_integer_input_in_file_and_state( diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 57fba0a17b..56baaa7b16 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7019,6 +7019,115 @@ def test_malformed_inputs_block_no_cascade(self): class TestWorkflowEngine: """Test WorkflowEngine execution.""" + @pytest.mark.parametrize("run_id", [None, "invalid-start"]) + @pytest.mark.parametrize( + ("input_definition", "provided", "error"), + [ + ({"type": "string", "required": True}, {}, "Required input"), + ({"type": "number"}, {"value": "invalid"}, "expected a number"), + ({"type": "string", "enum": ["yes"]}, {"value": "no"}, "not in allowed"), + ({"type": "number"}, {"value": True}, "expected a number"), + ({"type": "string"}, {"value": 5}, "expected a string"), + ({"type": "number", "default": True}, {}, "expected a number"), + ({"type": "string", "default": 5}, {}, "expected a string"), + ({"type": "string", "default": "no", "enum": ["yes"]}, {}, "not in allowed"), + ], + ids=["missing", "numeric", "enum", "bool-number", "typed", "number-default", + "string-default", "enum-default"], + ) + def test_invalid_startup_inputs_create_no_artifacts( + self, project_dir, monkeypatch, run_id, input_definition, provided, error + ): + from copy import deepcopy + + from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine + + monkeypatch.delenv("SPECKIT_WORKFLOW_RUN_ID", raising=False) + definition = WorkflowDefinition({ + "workflow": {"id": "invalid-start", "name": "Invalid start", "version": "1.0.0"}, + "inputs": {"value": input_definition}, + "steps": [{"id": "unexpected", "type": "shell", "run": "echo ran > unexpected.txt"}], + }) + original_data = deepcopy(definition.data) + original_inputs = deepcopy(provided) + started = [] + engine = WorkflowEngine(project_dir) + engine.on_step_start = lambda *args: started.append(args) + + with pytest.raises(ValueError, match=error): + engine.execute(definition, provided, run_id=run_id) + + assert started == [] + assert not (project_dir / "unexpected.txt").exists() + assert definition.data == original_data + assert provided == original_inputs + assert not (project_dir / ".specify" / "workflows" / "runs").exists() + + @pytest.mark.parametrize("env_id", [None, " ", " trimmed-id "]) + def test_startup_run_id_environment_selection(self, project_dir, monkeypatch, env_id): + from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine + + if env_id is None: + monkeypatch.delenv("SPECKIT_WORKFLOW_RUN_ID", raising=False) + else: + monkeypatch.setenv("SPECKIT_WORKFLOW_RUN_ID", env_id) + definition = WorkflowDefinition({"workflow": {"id": "identity"}, "steps": []}) + + state = WorkflowEngine(project_dir).execute(definition) + + if env_id and env_id.strip(): + assert state.run_id == "trimmed-id" + else: + assert len(state.run_id) == 8 + assert all(char in "0123456789abcdef" for char in state.run_id) + assert state.definition_path.is_file() + + def test_startup_preserves_coercion_defaults_and_caller_mappings(self, project_dir): + from copy import deepcopy + + from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine + + number = 1792234567890123456 + definition = WorkflowDefinition({ + "workflow": {"id": "defaults"}, + "inputs": { + "value": {"type": "number", "enum": [number], "default": str(number)}, + "enabled": {"type": "boolean", "default": "true"}, + "scope": {"type": "string", "enum": ["full"], "default": "full"}, + }, + "steps": [], + }) + provided = {"value": str(number)} + original_data = deepcopy(definition.data) + original_inputs = provided.copy() + + state = WorkflowEngine(project_dir).execute(definition) + assert state.inputs == {"value": number, "enabled": True, "scope": "full"} + state = WorkflowEngine(project_dir).execute(definition, provided) + + assert state.inputs == {"value": number, "enabled": True, "scope": "full"} + assert isinstance(state.inputs["value"], int) + assert definition.data == original_data + assert provided == original_inputs + + @pytest.mark.parametrize("run_id", ["", "../unsafe"]) + def test_startup_run_id_validation_precedes_input_resolution( + self, project_dir, monkeypatch, run_id + ): + from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine + + monkeypatch.setenv("SPECKIT_WORKFLOW_RUN_ID", "valid-env-id") + definition = WorkflowDefinition({ + "workflow": {"id": "identity"}, + "inputs": {"value": {"type": "string", "required": True}}, + "steps": [], + }) + + with pytest.raises(ValueError, match="[Rr]un.*ID|run_id"): + WorkflowEngine(project_dir).execute(definition, run_id=run_id) + + assert not (project_dir / ".specify" / "workflows" / "runs").exists() + @pytest.mark.parametrize( ("field", "value"), [