Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/harness-gateway-client/src/wire/request-tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ fn tool_round(offered: &[Offered]) -> Round {
let catalog = ToolCatalog::new(tools).expect("the offered tools are distinct");
let (ctx, requirements) = ctx.with_tools(catalog).prepare(&prompt, Some(model));
assert!(requirements.refusal().is_none(), "the round prepares");
let mut run = Run::new(Arc::new(prompt), "", ctx);
let mut run = Run::new(Arc::new(prompt), "".into(), ctx);
let mut rounds = Vec::new();
let result = loop {
let effects = match run.step() {
Expand Down
7 changes: 4 additions & 3 deletions crates/harness-internal/runner/src/harness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ pub struct RunRequest {
/// The prompt's Markdown source. The Harness reads the prompt only
/// from this text.
pub source: String,
/// The run's argument text.
pub args: String,
/// The run's arguments: a JSON string is the prompt's argument text as
/// it stands, and any other value is its JSON text.
pub args: serde_json::Value,
/// Text written to the prompt's declared `input:` file before the run
/// starts.
pub input_text: Option<String>,
Expand Down Expand Up @@ -235,7 +236,7 @@ impl Harness {
model,
ui: Some(host.ui()),
};
let prepared = match prepare_noting_cancel(&source, &args, services).await {
let prepared = match prepare_noting_cancel(&source, args, services).await {
Ok(prepared) => prepared,
Err((PrepareError::Refused { run_id, .. }, true)) => {
return Ok(RunReport {
Expand Down
4 changes: 2 additions & 2 deletions crates/harness-internal/runner/src/prepare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ impl PrepareError {
/// refuses a write.
pub async fn prepare(
source: &str,
args: &str,
args: serde_json::Value,
services: Services,
) -> Result<Prepared, PrepareError> {
prepare_noting_cancel(source, args, services)
Expand All @@ -231,7 +231,7 @@ pub async fn prepare(
)]
pub(crate) async fn prepare_noting_cancel(
source: &str,
args: &str,
args: serde_json::Value,
services: Services,
) -> Result<Prepared, (PrepareError, bool)> {
let Services {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ fn infers_then_chats() -> Run {
requirements.is_satisfied(),
"the bound model satisfies the fixture's one role: {requirements:?}"
);
Run::new(prompt, "", ctx)
Run::new(prompt, "".into(), ctx)
}

/// One round as the broker saw it: the last message's text and the round.
Expand Down
2 changes: 1 addition & 1 deletion crates/harness-internal/runner/tests/it/effect_loop-cap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ fn run_of(sections: &str) -> Run {
requirements.is_satisfied(),
"the bound model and the catalog satisfy the fixture: {requirements:?}"
);
Run::new(prompt, "", ctx)
Run::new(prompt, "".into(), ctx)
}

/// Parks every call until the test adds a permit to `gate`, then answers
Expand Down
45 changes: 44 additions & 1 deletion crates/harness-internal/runner/tests/it/harness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pub(crate) fn request(source: &str) -> RunRequest {
RunRequest {
name: NAME.to_owned(),
source: source.to_owned(),
args: String::new(),
args: "".into(),
input_text: None,
vfs: VfsRef::default(),
host: HostSnapshot {
Expand Down Expand Up @@ -161,6 +161,49 @@ const INFERS_AND_STORES: &str = "---\nname: infers\ndescription: d\nrunvil: 0\n\
store.write('note.md', 'kept')\n\
return models.infer('asked') .. '|' .. store.read('note.md')\n```\n";

/// A prompt under the default `args:` declaration that returns its args
/// text beside its `argv.prose`.
const ECHOES_PROSE: &str = "---\nname: prose\ndescription: d\nrunvil: 0\n---\n\n\
# Title\n\n## Only\n\n```lua\nreturn args .. '|' .. argv.prose\n```\n";

/// A prompt declaring a structured `topic` argument that returns
/// `argv.topic` beside its args text.
const ECHOES_TOPIC: &str = "---\nname: topic\ndescription: d\nrunvil: 0\n\
args:\n topic:\n type: string\n description: the topic\n---\n\n\
# Title\n\n## Only\n\n```lua\nreturn argv.topic .. '|' .. args\n```\n";

/// The final text of a run of `source` with `args`.
async fn final_text(source: &str, args: Value) -> String {
let recorder = Arc::new(MemoryRecorder::new());
let report = plain_harness(&recorder, ScriptedBroker::replying())
.run(RunRequest {
args,
..request(source)
})
.await
.expect("the run reaches an outcome");
let RunOutcome::Completed { final_text } = report.outcome else {
panic!("the run completes: {:?}", report.outcome);
};
final_text
}

#[tokio::test]
async fn a_string_argument_is_the_prose_of_a_default_declared_prompt() {
assert_eq!(
final_text(ECHOES_PROSE, json!("two words")).await,
"two words|two words"
);
}

#[tokio::test]
async fn an_object_argument_is_the_argv_of_a_structured_prompt() {
assert_eq!(
final_text(ECHOES_TOPIC, json!({ "topic": "cats" })).await,
r#"cats|{"topic":"cats"}"#
);
}

#[tokio::test]
async fn run_returns_the_declared_output_and_the_outcome_under_the_requests_name() {
let recorder = Arc::new(MemoryRecorder::new());
Expand Down
2 changes: 1 addition & 1 deletion crates/harness-internal/runner/tests/it/prepare-files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ async fn prepare_over(
let mut services = services(&recorder, bare());
services.vfs = vfs.clone();
services.input_text = input_text.map(str::to_owned);
let prepared = prepare(source, "", services).await;
let prepared = prepare(source, "".into(), services).await;
(recorder, prepared)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ async fn prepare_greeter(
let recorder = recorder();
let mut services = services(&recorder, building(GREETER));
services.services = run;
let prepared = prepare(GREETS, "", services).await;
let prepared = prepare(GREETS, "".into(), services).await;
(recorder, prepared)
}

Expand Down
4 changes: 2 additions & 2 deletions crates/harness-internal/runner/tests/it/prepare-input.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ fn with_input(input: Option<Arc<dyn AskBroker>>) -> HostServices {
/// Prepares `source` under `services` and drives the run to its end.
async fn drive_prompt(source: &str, services: Services) -> RunOutcome {
let recorder = Arc::clone(&services.recorder);
let prepared = prepare(source, "", services)
let prepared = prepare(source, "".into(), services)
.await
.expect("the prompt prepares");
drive_run(
Expand Down Expand Up @@ -158,7 +158,7 @@ async fn a_required_user_input_declaration_on_a_host_without_a_broker_is_refused
let recorder = recorder();
let error = prepare(
&user_input_prompt(REQUIRED, ASKS_ONCE),
"",
"".into(),
user_input_services(&recorder, None),
)
.await
Expand Down
2 changes: 1 addition & 1 deletion crates/harness-internal/runner/tests/it/prepare-prelude.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ fn script_call(value: &str, section: &str, entry: &str) -> serde_json::Value {
#[tokio::test]
async fn a_preludes_tool_calls_are_recorded_as_script_calls_from_the_section_that_made_each() {
let recorder = recorder();
let prepared = prepare(SPEAKS, "", services(&recorder, building(SPEAKER)))
let prepared = prepare(SPEAKS, "".into(), services(&recorder, building(SPEAKER)))
.await
.unwrap();
let run_id = prepared.run_id;
Expand Down
17 changes: 10 additions & 7 deletions crates/harness-internal/runner/tests/it/prepare-ready.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ fn request(source: &str) -> RunRequest {
RunRequest {
name: "session-1".to_owned(),
source: source.to_owned(),
args: String::new(),
args: "".into(),
input_text: None,
vfs: VfsRef::default(),
host: HostSnapshot {
Expand Down Expand Up @@ -211,10 +211,13 @@ async fn a_pack_whose_ready_resolves_late_has_its_tools_in_the_run() {
let (host, moves, startup) = slow_host(Phase::Starting);
let recorder = recorder();
let (prepared, ()) = tokio::time::timeout(PATIENCE, async {
tokio::join!(prepare(CALLS_SLOW, "", services(&recorder, host)), async {
startup.asked.notified().await;
moves.send(Phase::Ready).unwrap();
})
tokio::join!(
prepare(CALLS_SLOW, "".into(), services(&recorder, host)),
async {
startup.asked.notified().await;
moves.send(Phase::Ready).unwrap();
}
)
})
.await
.expect("the Harness waits on the Pack's ready");
Expand All @@ -237,7 +240,7 @@ async fn every_starting_pack_is_asked_to_be_ready_before_any_of_them_answers() {
let recorder = recorder();
let (prepared, ()) = tokio::time::timeout(PATIENCE, async {
tokio::join!(
prepare(COUNTS_OFFERED, "", services(&recorder, host)),
prepare(COUNTS_OFFERED, "".into(), services(&recorder, host)),
async {
for (_, startup) in &handles {
startup.asked.notified().await;
Expand Down Expand Up @@ -274,7 +277,7 @@ async fn a_declared_pack_whose_ready_fails_refuses_the_run_naming_its_reason() {
let reason = "the fixture server refused the handshake";
let (host, _moves, _startup) = slow_host(Phase::Failed(reason));
let recorder = recorder();
let error = prepare(DECLARES_SLOW, "", services(&recorder, host))
let error = prepare(DECLARES_SLOW, "".into(), services(&recorder, host))
.await
.expect_err("a declared Pack that will not be ready refuses the run");

Expand Down
22 changes: 13 additions & 9 deletions crates/harness-internal/runner/tests/it/prepare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ fn completed(outcome: RunOutcome) -> String {
#[tokio::test]
async fn an_unmet_requirement_is_refused_with_the_engines_notice_and_its_run_ended_as_failed() {
let recorder = recorder();
let error = prepare(NEEDS_WEB, "", services(&recorder, bare()))
let error = prepare(NEEDS_WEB, "".into(), services(&recorder, bare()))
.await
.expect_err("a missing required Pack refuses the run");

Expand Down Expand Up @@ -221,7 +221,7 @@ async fn an_unmet_requirement_is_refused_with_the_engines_notice_and_its_run_end
#[tokio::test]
async fn a_prompt_that_does_not_parse_fails_preparation_and_its_run_ends_as_a_parse_failure() {
let recorder = recorder();
let error = prepare(UNCLOSED, "", services(&recorder, bare()))
let error = prepare(UNCLOSED, "".into(), services(&recorder, bare()))
.await
.expect_err("a prompt without a closed frontmatter does not parse");

Expand All @@ -248,10 +248,10 @@ async fn a_prompt_that_does_not_parse_fails_preparation_and_its_run_ends_as_a_pa
async fn two_prepared_runs_draw_different_seeds_and_both_begin_at_the_recorder() {
let recorder = recorder();
let before = unix_millis_now();
let first = prepare(PLAIN, "", services(&recorder, bare()))
let first = prepare(PLAIN, "".into(), services(&recorder, bare()))
.await
.unwrap();
let second = prepare(PLAIN, "", services(&recorder, bare()))
let second = prepare(PLAIN, "".into(), services(&recorder, bare()))
.await
.unwrap();
let after = unix_millis_now();
Expand Down Expand Up @@ -310,7 +310,7 @@ async fn a_prepared_run_drives_to_its_end_under_its_own_performers() {
run_id,
performers,
..
} = prepare(PLAIN, "", services(&recorder, bare()))
} = prepare(PLAIN, "".into(), services(&recorder, bare()))
.await
.unwrap();
let outcome = drive_run(
Expand All @@ -333,7 +333,7 @@ async fn a_prepared_run_drives_to_its_end_under_its_own_performers() {
/// Prepares `source` over `host` and drives the run to its end.
pub(crate) async fn drive_over(source: &str, host: HostContext) -> RunOutcome {
let recorder = recorder();
let prepared = prepare(source, "", services(&recorder, host))
let prepared = prepare(source, "".into(), services(&recorder, host))
.await
.unwrap();
drive_run(
Expand Down Expand Up @@ -387,9 +387,13 @@ async fn a_bare_entry_another_vendor_answers_is_refused_naming_its_fqn() {
let recorder = recorder();
let declares_bare = "---\nname: bare\ndescription: d\nrunvil: 0\n\
uses:\n - tools\n---\n\n# Title\n\n## Only\n\nDone.\n";
let error = prepare(declares_bare, "", services(&recorder, building(TOOLS)))
.await
.expect_err("a bare id means the runvil Pack, which the Host lacks");
let error = prepare(
declares_bare,
"".into(),
services(&recorder, building(TOOLS)),
)
.await
.expect_err("a bare id means the runvil Pack, which the Host lacks");
let PrepareError::Refused { error, .. } = error else {
panic!("the refusal is a requirements refusal: {error}");
};
Expand Down
2 changes: 1 addition & 1 deletion crates/harness-internal/runner/tests/it/support.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ fn prepared(prompt: Arc<Prompt>, vfs: VfsRef) -> Run {
requirements.is_satisfied(),
"the runner fixture prompt declares nothing the environment must supply: {requirements:?}"
);
Run::new(prompt, "", ctx)
Run::new(prompt, "".into(), ctx)
}

/// A Pack-free run over `lua` with a fixed seed and start, over a
Expand Down
10 changes: 7 additions & 3 deletions crates/harness-internal/runner/tests/it/tool_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,13 @@ impl Pack for Record {
#[tokio::test]
async fn a_tool_writes_through_its_context_and_the_calling_script_reads_it_back() {
let recorder = Arc::new(MemoryRecorder::new());
let prepared = prepare(CALLS_RECORD, "", services(&recorder, building(CONTEXT)))
.await
.unwrap();
let prepared = prepare(
CALLS_RECORD,
"".into(),
services(&recorder, building(CONTEXT)),
)
.await
.unwrap();
let outcome = drive_run(
prepared.run,
prepared.performers,
Expand Down
2 changes: 1 addition & 1 deletion crates/harness/examples/run-prompt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let request = RunRequest {
name: "desk-greet-1".to_owned(),
source: GREET.to_owned(),
args: String::new(),
args: "".into(),
input_text: Some("desk".to_owned()),
vfs: VfsRef::default(),
host: HostSnapshot::default(),
Expand Down
2 changes: 1 addition & 1 deletion crates/harness/tests/suite/broker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ async fn a_sections_round_reaches_the_broker_as_chat_and_a_nested_infer_round_as
.run(RunRequest {
name: "rounds-1".to_owned(),
source: INFERS_THEN_CHATS.to_owned(),
args: String::new(),
args: "".into(),
input_text: None,
vfs: VfsRef::default(),
host: HostSnapshot {
Expand Down
2 changes: 1 addition & 1 deletion crates/harness/tests/suite/host.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ fn request(name: &str, source: &str) -> RunRequest {
RunRequest {
name: name.to_owned(),
source: source.to_owned(),
args: String::new(),
args: "".into(),
input_text: None,
vfs: VfsRef::default(),
host: HostSnapshot::default(),
Expand Down
2 changes: 1 addition & 1 deletion crates/harness/tests/suite/launch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fn request(source: &str, vfs: VfsRef) -> RunRequest {
RunRequest {
name: "shouts-1".to_owned(),
source: source.to_owned(),
args: String::new(),
args: "".into(),
input_text: Some("hello".to_owned()),
vfs,
host: HostSnapshot::default(),
Expand Down
2 changes: 1 addition & 1 deletion crates/harness/tests/suite/vfs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ async fn a_hosts_edit_of_a_file_the_run_read_conflicts_while_the_run_waits_on_it
let request = RunRequest {
name: "desk-review-1".to_owned(),
source: REVIEW.to_owned(),
args: String::new(),
args: "".into(),
input_text: None,
vfs: vfs.clone(),
host: HostSnapshot::default(),
Expand Down
2 changes: 1 addition & 1 deletion crates/runvil-internal/engine/benches/models_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ fn run_once(prompt: &Prompt, chat: &ScriptedChat, context: u32) -> RunResult {
if let Some(refusal) = requirements.refusal() {
return RunResult::Failure(refusal);
}
let run = Run::new(Arc::new(prompt.clone()), "", ctx);
let run = Run::new(Arc::new(prompt.clone()), "".into(), ctx);
let (result, _) = drive(run, |_, effect| {
perform_locally(effect, &mut |round| {
let Effect::Chat {
Expand Down
9 changes: 9 additions & 0 deletions crates/runvil-internal/engine/src/execute/context-bound.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,15 @@ pub(super) fn bound_model_set(prompt: &Prompt, bindings: &ModelBindings) -> Mode
set
}

/// The run's argument text for the arguments `args`: a JSON string as it
/// stands, and any other value as its JSON text.
pub(super) fn args_text(args: serde_json::Value) -> String {
match args {
serde_json::Value::String(text) => text,
value => value.to_string(),
}
}

/// Derives a section's `argv` from the args string under the prompt's
/// declaration: a default-declared prompt wraps the interface prose into
/// the default shape (`argv.prose`, with the empty string present, not
Expand Down
Loading
Loading