diff --git a/CHANGELOG.md b/CHANGELOG.md index 125542c..902e9ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add versioned machine-readable CLI success results and JSON failure diagnostics, plus bounded + JSON Lines standard-input spooling and unambiguous completed-journal standard-output pipelines. - Publish scenario/journal contract v16 and strategy protocol v14 with typed, dimensioned strategy metrics while retaining string-only metric compatibility through v15 and protocol v13. diff --git a/README.md b/README.md index 1a1b207..e93feda 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,19 @@ opam exec -- dune exec trading-engine -- \ --journal demo.journal.jsonl ``` +Compose a JSON Lines producer and journal consumer without mixing streams: + +```sh +produce-scenario | trading-engine --input - --input-format jsonl --journal - | consume-journal +``` + +Standard input is spooled to a private temporary file, limited to 1 GiB, then hashed and validated +before replay. Standard output contains only journal records. The engine stages and verifies the +complete journal before copying it to the pipe; its final `run_completed` record and a zero exit +status signal completion. Pipe output cannot provide exclusive no-replace publication, atomic +linking, retained partial files, directory synchronization, or restart-durability guarantees. +`--durable-artifacts` is therefore invalid with `--journal -`. + Run an external strategy against an empty-schedule scenario: ```sh @@ -162,6 +175,13 @@ diagnostic contract identified by each diagnostic's `diagnostic_version`. Human the default. Use `--diagnostic-format json` to receive one JSON diagnostic on standard error with a stable code, phase, typed context, and sanitized underlying cause. +Use `--output-format json` for the versioned +[CLI result contract](contracts/cli/v1/README.md). A success document includes the run identity, +scenario and artifact hashes, replay counts, normalized current valuation, and artifact locations. +This option also selects JSON failure diagnostics. For file journals the success document is written +to standard output. With `--journal -`, the journal owns standard output and the success document +moves to standard error. + The final and `.partial` journal paths must not already exist. Batch JSON hashes the same complete document it parses. JSON Lines input is hashed and validated in a bounded-memory pass before the journal is created, then replayed from the same open file and hashed again before publication. The @@ -238,6 +258,7 @@ do not provide reducer snapshots or restart recovery. - [Contributing](CONTRIBUTING.md) - [Architecture](docs/architecture.md) - [Diagnostic contract](docs/diagnostics.md) +- [CLI result contract](contracts/cli/v1/README.md) - [Scenario contract](docs/scenario.md) - [Contract conformance corpus](contracts/conformance/README.md) - [Current contract v16 and conformance fixtures](contracts/v16/README.md) @@ -246,6 +267,7 @@ do not provide reducer snapshots or restart recovery. - [Scenario JSON Schema](contracts/v16/scenario.schema.json) - [Scenario stream record JSON Schema](contracts/v16/scenario-stream.schema.json) - [Journal record JSON Schema](contracts/v16/journal.schema.json) +- [CLI result JSON Schema](contracts/cli/v1/result.schema.json) - [External strategy protocol v14](contracts/strategy/v14/README.md) - [Historical strategy protocol v3](contracts/strategy/v3/README.md) - [Historical strategy protocol v2](contracts/strategy/v2/README.md) diff --git a/bin/dune b/bin/dune index cb97f95..d1392d9 100644 --- a/bin/dune +++ b/bin/dune @@ -4,4 +4,12 @@ (package trading_engine) (instrumentation (backend bisect_ppx)) - (libraries trading_engine cmdliner eio_main fmt.tty logs.fmt logs.cli)) + (libraries + trading_engine + cmdliner + eio_main + fmt.tty + logs.fmt + logs.cli + yojson + unix)) diff --git a/bin/main.ml b/bin/main.ml index dad181a..14e0fbb 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -1,6 +1,37 @@ open Cmdliner type diagnostic_format = Human | Json +type output_format = Human_output | Json_output + +type counts = { + instruments : int64; + schedule_batches : int64; + slices : int64; + audits : int64; + orders : int64; + active_orders : int64; + filled_orders : int64; + rejected_orders : int64; +} + +type success = { + operation : string; + run_id : Trading_engine.Id.Run.t; + scenario_sha256 : string; + journal_sha256 : string option; + transcript_sha256 : string option; + counts : counts; + valuation : Trading_engine.Account.valuation; + journal : string option; + transcript : string option; +} + +type journal_destination = { + replay_path : string; + public_path : string; + writes_stdout : bool; + cleanup : unit -> unit; +} let cli_error message = Trading_engine.Diagnostic.make @@ -17,65 +48,243 @@ let count predicate values = (fun total value -> total + Bool.to_int (predicate value)) 0 values -let run_replay scenario_sha256 scenario journal durability = +let int64 value = `Intlit (Int64.to_string value) +let option_string = function None -> `Null | Some value -> `String value + +let success_to_yojson success = + let counts = success.counts in + `Assoc + [ + ("result_version", `String "1"); + ("status", `String "success"); + ("operation", `String success.operation); + ("run_id", `String (Trading_engine.Id.Run.to_string success.run_id)); + ( "hashes", + `Assoc + [ + ("scenario_sha256", `String success.scenario_sha256); + ("journal_sha256", option_string success.journal_sha256); + ( "strategy_transcript_sha256", + option_string success.transcript_sha256 ); + ] ); + ( "counts", + `Assoc + [ + ("instruments", int64 counts.instruments); + ("schedule_batches", int64 counts.schedule_batches); + ("slices", int64 counts.slices); + ("audits", int64 counts.audits); + ("orders", int64 counts.orders); + ("active_orders", int64 counts.active_orders); + ("filled_orders", int64 counts.filled_orders); + ("rejected_orders", int64 counts.rejected_orders); + ] ); + ( "valuation", + Trading_engine.Codec.account_valuation_to_yojson ~contract_version:"16" + success.valuation ); + ( "artifacts", + `Assoc + [ + ("journal", option_string success.journal); + ("strategy_transcript", option_string success.transcript); + ] ); + ] + +let order_counts orders = + let active = count Trading_engine.Order.is_active orders in + let filled = + count + (fun order -> + order.Trading_engine.Order.status = Trading_engine.Order.Filled) + orders + in + let rejected = + count + (fun order -> + match order.Trading_engine.Order.status with + | Trading_engine.Order.Rejected _ -> true + | _ -> false) + orders + in + (active, filled, rejected) + +let emit_success format ~to_stderr success = + let formatter = + if to_stderr then Format.err_formatter else Format.std_formatter + in + match format with + | Json_output -> + Fmt.pf formatter "%s@." + (Yojson.Safe.to_string (success_to_yojson success)) + | Human_output -> + let counts = success.counts in + Fmt.pf formatter + "run=%a audits=%Ld orders=%Ld active=%Ld filled=%Ld rejected=%Ld@." + Trading_engine.Id.Run.pp success.run_id counts.audits counts.orders + counts.active_orders counts.filled_orders counts.rejected_orders; + Fmt.pf formatter "%a@." Trading_engine.Account.pp_valuation + success.valuation; + Option.iter (Fmt.pf formatter "journal=%s@.") success.journal; + Option.iter + (Fmt.pf formatter "strategy_transcript=%s@.") + success.transcript + +let digest_file path = Trading_engine.Sha256.digest_file path + +let remove_if_exists path = + try if Sys.file_exists path then Sys.remove path with Sys_error _ -> () + +let temporary_journal_destination () = + try + let directory = + Filename.temp_dir ~perms:0o700 "trading-engine-journal-" "" + in + let path = Filename.concat directory "journal.jsonl" in + Ok + { + replay_path = path; + public_path = "stdout"; + writes_stdout = true; + cleanup = + (fun () -> + remove_if_exists path; + remove_if_exists (path ^ ".partial"); + remove_if_exists (path ^ ".partial.cleanup"); + try Unix.rmdir directory with Unix.Unix_error _ -> ()); + } + with exception_ -> + Error + (Trading_engine.Diagnostic.of_exception + ~code:Trading_engine.Diagnostic.Artifact_io + ~phase:Trading_engine.Diagnostic.Artifact + ~message:"could not create temporary journal spool" exception_) + +let journal_destination path = + if String.equal path "-" then temporary_journal_destination () + else + Ok + { + replay_path = path; + public_path = path; + writes_stdout = false; + cleanup = Fun.id; + } + +let copy_file_to_stdout path = + try + In_channel.with_open_bin path (fun channel -> + let buffer = Bytes.create 65_536 in + let rec loop () = + match input channel buffer 0 (Bytes.length buffer) with + | 0 -> () + | length -> + output stdout buffer 0 length; + loop () + in + loop ()); + flush stdout; + Ok () + with exception_ -> + Error + (Trading_engine.Diagnostic.of_exception + ~code:Trading_engine.Diagnostic.Artifact_io + ~phase:Trading_engine.Diagnostic.Artifact + ~message:"could not write journal to standard output" exception_) + +let finish_success output_format destination success = + let result = + match digest_file destination.replay_path with + | Error _ as error -> error + | Ok journal_sha256 -> + let success = + { + success with + journal_sha256 = Some journal_sha256; + journal = Some destination.public_path; + } + in + if destination.writes_stdout then ( + match copy_file_to_stdout destination.replay_path with + | Error _ as error -> error + | Ok () -> + emit_success output_format ~to_stderr:true success; + Ok ()) + else ( + emit_success output_format ~to_stderr:false success; + Ok ()) + in + destination.cleanup (); + result + +let run_replay scenario_sha256 scenario destination durability output_format = match - Trading_engine.Replay.run ~scenario_sha256 ~journal_path:journal ~durability - scenario + Trading_engine.Replay.run ~scenario_sha256 + ~journal_path:destination.replay_path ~durability scenario with - | Error message -> Error message + | Error message -> + destination.cleanup (); + Error message | Ok result -> - let active = count Trading_engine.Order.is_active result.orders in - let filled = - count - (fun order -> - order.Trading_engine.Order.status = Trading_engine.Order.Filled) - result.orders + let active, filled, rejected = order_counts result.orders in + let success = + { + operation = "replay"; + run_id = scenario.Trading_engine.Scenario.run_id; + scenario_sha256; + journal_sha256 = None; + transcript_sha256 = None; + counts = + { + instruments = Int64.of_int (List.length scenario.instruments); + schedule_batches = Int64.of_int (List.length scenario.schedule); + slices = Int64.of_int (List.length scenario.slices); + audits = Int64.of_int (List.length result.audits); + orders = Int64.of_int (List.length result.orders); + active_orders = Int64.of_int active; + filled_orders = Int64.of_int filled; + rejected_orders = Int64.of_int rejected; + }; + valuation = result.valuation; + journal = None; + transcript = None; + } in - let rejected = - count - (fun order -> - match order.Trading_engine.Order.status with - | Trading_engine.Order.Rejected _ -> true - | _ -> false) - result.orders - in - Fmt.pr "run=%a audits=%d orders=%d active=%d filled=%d rejected=%d@." - Trading_engine.Id.Run.pp scenario.Trading_engine.Scenario.run_id - (List.length result.audits) - (List.length result.orders) - active filled rejected; - Fmt.pr "%a@." Trading_engine.Account.pp_valuation result.valuation; - Fmt.pr "journal=%s@." journal; - Ok () - -let run_stream input journal durability = + finish_success output_format destination success + +let run_stream input destination durability output_format = match - Trading_engine.Replay.run_stream ~journal_path:journal ~durability input + Trading_engine.Replay.run_stream ~journal_path:destination.replay_path + ~durability input with - | Error message -> Error message + | Error message -> + destination.cleanup (); + Error message | Ok result -> - let active = count Trading_engine.Order.is_active result.orders in - let filled = - count - (fun order -> - order.Trading_engine.Order.status = Trading_engine.Order.Filled) - result.orders - in - let rejected = - count - (fun order -> - match order.Trading_engine.Order.status with - | Trading_engine.Order.Rejected _ -> true - | _ -> false) - result.orders + let active, filled, rejected = order_counts result.orders in + let success = + { + operation = "replay"; + run_id = result.run_id; + scenario_sha256 = result.scenario_sha256; + journal_sha256 = None; + transcript_sha256 = None; + counts = + { + instruments = Int64.of_int result.instrument_count; + schedule_batches = result.schedule_count; + slices = result.slice_count; + audits = result.audit_count; + orders = Int64.of_int (List.length result.orders); + active_orders = Int64.of_int active; + filled_orders = Int64.of_int filled; + rejected_orders = Int64.of_int rejected; + }; + valuation = result.valuation; + journal = None; + transcript = None; + } in - Fmt.pr "run=%a audits=%Ld orders=%d active=%d filled=%d rejected=%d@." - Trading_engine.Id.Run.pp result.run_id result.audit_count - (List.length result.orders) - active filled rejected; - Fmt.pr "%a@." Trading_engine.Account.pp_valuation result.valuation; - Fmt.pr "journal=%s@." journal; - Ok () + finish_success output_format destination success type external_strategy = { command : string list; @@ -83,75 +292,117 @@ type external_strategy = { transcript : string; } -let run_external_replay environment scenario_sha256 scenario journal strategy - durability = +let run_external_replay environment scenario_sha256 scenario destination + strategy durability output_format = match Trading_engine.External_replay.run ~durability ~env:environment - ~scenario_sha256 ~journal_path:journal + ~scenario_sha256 ~journal_path:destination.replay_path ~transcript_path:strategy.transcript ~strategy_command:strategy.command ~strategy_timeout:strategy.timeout scenario with - | Error message -> Error message - | Ok result -> - let active = count Trading_engine.Order.is_active result.orders in - let filled = - count - (fun order -> - order.Trading_engine.Order.status = Trading_engine.Order.Filled) - result.orders - in - let rejected = - count - (fun order -> - match order.Trading_engine.Order.status with - | Trading_engine.Order.Rejected _ -> true - | _ -> false) - result.orders - in - Fmt.pr "run=%a audits=%d orders=%d active=%d filled=%d rejected=%d@." - Trading_engine.Id.Run.pp scenario.Trading_engine.Scenario.run_id - (List.length result.audits) - (List.length result.orders) - active filled rejected; - Fmt.pr "%a@." Trading_engine.Account.pp_valuation result.valuation; - Fmt.pr "journal=%s@." journal; - Fmt.pr "strategy_transcript=%s@." strategy.transcript; - Ok () - -let run_external_stream environment input journal strategy durability = + | Error message -> + destination.cleanup (); + Error message + | Ok result -> ( + let active, filled, rejected = order_counts result.orders in + match digest_file strategy.transcript with + | Error _ as error -> + destination.cleanup (); + error + | Ok transcript_sha256 -> + finish_success output_format destination + { + operation = "replay"; + run_id = scenario.Trading_engine.Scenario.run_id; + scenario_sha256; + journal_sha256 = None; + transcript_sha256 = Some transcript_sha256; + counts = + { + instruments = Int64.of_int (List.length scenario.instruments); + schedule_batches = 0L; + slices = Int64.of_int (List.length scenario.slices); + audits = Int64.of_int (List.length result.audits); + orders = Int64.of_int (List.length result.orders); + active_orders = Int64.of_int active; + filled_orders = Int64.of_int filled; + rejected_orders = Int64.of_int rejected; + }; + valuation = result.valuation; + journal = None; + transcript = Some strategy.transcript; + }) + +let run_external_stream environment input destination strategy durability + output_format = match Trading_engine.External_replay.run_stream ~durability ~env:environment - ~journal_path:journal ~transcript_path:strategy.transcript + ~journal_path:destination.replay_path ~transcript_path:strategy.transcript ~strategy_command:strategy.command ~strategy_timeout:strategy.timeout input with - | Error message -> Error message - | Ok result -> - let active = count Trading_engine.Order.is_active result.orders in - let filled = - count - (fun order -> - order.Trading_engine.Order.status = Trading_engine.Order.Filled) - result.orders - in - let rejected = - count - (fun order -> - match order.Trading_engine.Order.status with - | Trading_engine.Order.Rejected _ -> true - | _ -> false) - result.orders - in - Fmt.pr "run=%a audits=%Ld orders=%d active=%d filled=%d rejected=%d@." - Trading_engine.Id.Run.pp result.run_id result.audit_count - (List.length result.orders) - active filled rejected; - Fmt.pr "%a@." Trading_engine.Account.pp_valuation result.valuation; - Fmt.pr "journal=%s@." journal; - Fmt.pr "strategy_transcript=%s@." strategy.transcript; - Ok () - -let execute_json environment input journal validate_only strategy durability = + | Error message -> + destination.cleanup (); + Error message + | Ok result -> ( + let active, filled, rejected = order_counts result.orders in + match digest_file strategy.transcript with + | Error _ as error -> + destination.cleanup (); + error + | Ok transcript_sha256 -> + finish_success output_format destination + { + operation = "replay"; + run_id = result.run_id; + scenario_sha256 = result.scenario_sha256; + journal_sha256 = None; + transcript_sha256 = Some transcript_sha256; + counts = + { + instruments = Int64.of_int result.instrument_count; + schedule_batches = 0L; + slices = result.slice_count; + audits = result.audit_count; + orders = Int64.of_int (List.length result.orders); + active_orders = Int64.of_int active; + filled_orders = Int64.of_int filled; + rejected_orders = Int64.of_int rejected; + }; + valuation = result.valuation; + journal = None; + transcript = Some strategy.transcript; + }) + +let emit_validation output_format ~run_id ~scenario_sha256 ~instrument_count + ~schedule_count ~slice_count ~orders ~valuation ~audit_count = + let active, filled, rejected = order_counts orders in + emit_success output_format ~to_stderr:false + { + operation = "validate"; + run_id; + scenario_sha256; + journal_sha256 = None; + transcript_sha256 = None; + counts = + { + instruments = instrument_count; + schedule_batches = schedule_count; + slices = slice_count; + audits = audit_count; + orders = Int64.of_int (List.length orders); + active_orders = Int64.of_int active; + filled_orders = Int64.of_int filled; + rejected_orders = Int64.of_int rejected; + }; + valuation; + journal = None; + transcript = None; + }; + Ok () + +let execute_json environment input journal validate_only strategy durability + output_format = let document = try Ok (In_channel.with_open_bin input In_channel.input_all) with Sys_error message as exception_ -> @@ -175,16 +426,28 @@ let execute_json environment input journal validate_only strategy durability = | None -> ( match Trading_engine.Replay.run ~scenario_sha256 scenario with | Error message -> Error message - | Ok _ -> - Fmt.pr - "valid run=%a instruments=%d schedule=%d slices=%d \ - scenario_sha256=%s@." - Trading_engine.Id.Run.pp scenario.run_id - (List.length scenario.instruments) - (List.length scenario.schedule) - (List.length scenario.slices) - scenario_sha256; - Ok ()) + | Ok result -> + if output_format = Human_output then ( + Fmt.pr + "valid run=%a instruments=%d schedule=%d slices=%d \ + scenario_sha256=%s@." + Trading_engine.Id.Run.pp scenario.run_id + (List.length scenario.instruments) + (List.length scenario.schedule) + (List.length scenario.slices) + scenario_sha256; + Ok ()) + else + emit_validation output_format ~run_id:scenario.run_id + ~scenario_sha256 + ~instrument_count: + (Int64.of_int (List.length scenario.instruments)) + ~schedule_count: + (Int64.of_int (List.length scenario.schedule)) + ~slice_count: + (Int64.of_int (List.length scenario.slices)) + ~orders:result.orders ~valuation:result.valuation + ~audit_count:(Int64.of_int (List.length result.audits))) else match journal with | None -> @@ -192,13 +455,19 @@ let execute_json environment input journal validate_only strategy durability = (cli_error "--journal is required unless --validate-only is set") | Some path -> ( - match strategy with - | None -> run_replay scenario_sha256 scenario path durability - | Some strategy -> - run_external_replay environment scenario_sha256 scenario - path strategy durability))) + match journal_destination path with + | Error _ as error -> error + | Ok destination -> ( + match strategy with + | None -> + run_replay scenario_sha256 scenario destination + durability output_format + | Some strategy -> + run_external_replay environment scenario_sha256 scenario + destination strategy durability output_format)))) -let execute_jsonl environment input journal validate_only strategy durability = +let execute_jsonl environment input journal validate_only strategy durability + output_format = if validate_only then match journal with | Some _ -> @@ -207,30 +476,44 @@ let execute_jsonl environment input journal validate_only strategy durability = match Trading_engine.Replay.run_stream input with | Error message -> Error message | Ok result -> - Fmt.pr - "valid run=%a instruments=%d schedule=%Ld slices=%Ld \ - scenario_sha256=%s@." - Trading_engine.Id.Run.pp result.run_id result.instrument_count - result.schedule_count result.slice_count result.scenario_sha256; - Ok ()) + if output_format = Human_output then ( + Fmt.pr + "valid run=%a instruments=%d schedule=%Ld slices=%Ld \ + scenario_sha256=%s@." + Trading_engine.Id.Run.pp result.run_id result.instrument_count + result.schedule_count result.slice_count result.scenario_sha256; + Ok ()) + else + emit_validation output_format ~run_id:result.run_id + ~scenario_sha256:result.scenario_sha256 + ~instrument_count:(Int64.of_int result.instrument_count) + ~schedule_count:result.schedule_count + ~slice_count:result.slice_count ~orders:result.orders + ~valuation:result.valuation ~audit_count:result.audit_count) else match journal with | None -> Error (cli_error "--journal is required unless --validate-only is set") | Some path -> ( - match strategy with - | None -> run_stream input path durability - | Some strategy -> - run_external_stream environment input path strategy durability) + match journal_destination path with + | Error _ as error -> error + | Ok destination -> ( + match strategy with + | None -> run_stream input destination durability output_format + | Some strategy -> + run_external_stream environment input destination strategy + durability output_format)) type input_format = Json | Jsonl let execute_scenario environment input journal validate_only strategy durability - = function + output_format = function | Json -> execute_json environment input journal validate_only strategy durability + output_format | Jsonl -> execute_jsonl environment input journal validate_only strategy durability + output_format let external_strategy executable arguments timeout transcript = match (executable, transcript, arguments, timeout) with @@ -250,9 +533,78 @@ let external_strategy executable arguments timeout transcript = Error (cli_error "--strategy-timeout must be finite and positive") else Ok (Some { command = executable :: arguments; timeout; transcript }) +let spool_standard_input () = + let path, channel = + Filename.open_temp_file ~mode:[ Open_binary ] "trading-engine-stdin-" + ".jsonl" + in + let fail diagnostic = + close_out_noerr channel; + remove_if_exists path; + Error diagnostic + in + try + let buffer = Bytes.create 65_536 in + let rec loop total = + match input stdin buffer 0 (Bytes.length buffer) with + | 0 -> Ok total + | length -> + if + total + > Trading_engine.Resource_limits.scenario_stream_bytes - length + then + Error + (Trading_engine.Diagnostic.make + ~code:Trading_engine.Diagnostic.Resource_limit + ~phase:Trading_engine.Diagnostic.Input + (Printf.sprintf + "standard-input scenario stream exceeds %d bytes" + Trading_engine.Resource_limits.scenario_stream_bytes)) + else ( + output channel buffer 0 length; + loop (total + length)) + in + match loop 0 with + | Error diagnostic -> fail diagnostic + | Ok _ -> + close_out channel; + Ok path + with exception_ -> + fail + (Trading_engine.Diagnostic.of_exception + ~code:Trading_engine.Diagnostic.Input_io + ~phase:Trading_engine.Diagnostic.Input + ~message:"could not spool scenario stream from standard input" + exception_) + +let with_input_path input input_format function_ = + if not (String.equal input "-") then function_ input + else + match input_format with + | Json -> + Error + (cli_error + "standard input requires --input-format jsonl; batch JSON is not \ + supported") + | Jsonl -> ( + try + match spool_standard_input () with + | Error _ as error -> error + | Ok path -> + Fun.protect + ~finally:(fun () -> remove_if_exists path) + (fun () -> function_ path) + with exception_ -> + Error + (Trading_engine.Diagnostic.of_exception + ~code:Trading_engine.Diagnostic.Input_io + ~phase:Trading_engine.Diagnostic.Input + ~message:"could not prepare standard-input scenario stream" + exception_)) + let execute environment input journal validate_only capabilities input_format strategy_executable strategy_arguments strategy_timeout strategy_transcript - durable_artifacts = + durable_artifacts output_format = if capabilities then match ( input, @@ -273,6 +625,16 @@ let execute environment input journal validate_only capabilities input_format "--capabilities cannot be combined with replay or strategy options") else if validate_only && durable_artifacts then Error (cli_error "--durable-artifacts cannot be used with --validate-only") + else if durable_artifacts && Option.equal String.equal journal (Some "-") then + Error + (cli_error + "--durable-artifacts cannot be used when --journal writes to standard \ + output") + else if Option.equal String.equal strategy_transcript (Some "-") then + Error + (cli_error + "--strategy-transcript does not support standard output; choose a \ + file path") else match input with | None -> @@ -292,13 +654,16 @@ let execute environment input journal validate_only capabilities input_format if durable_artifacts then Trading_engine.Artifact_writer.Durable else Trading_engine.Artifact_writer.Buffered in - execute_scenario environment path journal validate_only strategy - durability input_format) + with_input_path path input_format (fun input_path -> + execute_scenario environment input_path journal validate_only + strategy durability output_format input_format)) let input = let doc = "Read the replay scenario from $(docv)." in Arg.( - value & opt (some file) None & info [ "input"; "i" ] ~docv:"SCENARIO" ~doc) + value + & opt (some string) None + & info [ "input"; "i" ] ~docv:"SCENARIO|-" ~doc) let input_format = let formats = Arg.enum [ ("json", Json); ("jsonl", Jsonl) ] in @@ -306,7 +671,10 @@ let input_format = Arg.(value & opt formats Json & info [ "input-format" ] ~docv:"FORMAT" ~doc) let journal = - let doc = "Create the append-only JSON Lines audit journal at $(docv)." in + let doc = + "Create the append-only JSON Lines audit journal at $(docv). Use '-' to \ + write a completed journal to standard output." + in Arg.( value & opt (some string) None @@ -333,6 +701,16 @@ let diagnostic_format = Arg.( value & opt formats Human & info [ "diagnostic-format" ] ~docv:"FORMAT" ~doc) +let output_format = + let formats = Arg.enum [ ("human", Human_output); ("json", Json_output) ] in + let doc = + "Render successful validation and replay summaries as $(docv) (default: \ + human). JSON output also selects JSON diagnostics." + in + Arg.( + value & opt formats Human_output + & info [ "output-format" ] ~docv:"FORMAT" ~doc) + let strategy_executable = let doc = "Launch $(docv) as the external strategy process without using a shell." @@ -398,27 +776,32 @@ let command environment = strategy_transcript durable_artifacts diagnostic_format + output_format -> ( diagnostic_format, + output_format, execute environment input journal validate_only capabilities input_format strategy_executable strategy_arguments - strategy_timeout strategy_transcript durable_artifacts )) + strategy_timeout strategy_transcript durable_artifacts + output_format )) $ input $ journal $ validate_only $ capabilities $ input_format $ strategy_executable $ strategy_argument $ strategy_timeout - $ strategy_transcript $ durable_artifacts $ diagnostic_format) + $ strategy_transcript $ durable_artifacts $ diagnostic_format + $ output_format) let () = Fmt_tty.setup_std_outputs (); Eio_main.run @@ fun environment -> match Cmd.eval_value' (command environment) with | `Exit code -> exit code - | `Ok (_, Ok ()) -> exit Cmd.Exit.ok - | `Ok (format, Error diagnostic) -> + | `Ok (_, _, Ok ()) -> exit Cmd.Exit.ok + | `Ok (diagnostic_format, output_format, Error diagnostic) -> let rendered = - match format with - | Human -> + match (diagnostic_format, output_format) with + | Human, Human_output -> "trading-engine: " ^ Trading_engine.Diagnostic.to_human diagnostic - | Json -> Trading_engine.Diagnostic.to_json diagnostic + | Json, _ | _, Json_output -> + Trading_engine.Diagnostic.to_json diagnostic in Fmt.epr "%s@." rendered; exit Cmd.Exit.some_error diff --git a/contracts/cli/v1/README.md b/contracts/cli/v1/README.md new file mode 100644 index 0000000..82a47e4 --- /dev/null +++ b/contracts/cli/v1/README.md @@ -0,0 +1,20 @@ +# CLI result contract v1 + +Pass `--output-format json` to receive one compact JSON success document. The +[`result.schema.json`](result.schema.json) schema defines its stable fields. The document identifies +the operation and run, binds scenario and artifact hashes, reports replay counts and the normalized +current valuation, and names any created artifacts. + +Failures use the existing +[diagnostic contract v1](../../diagnostic/v1/diagnostic.schema.json). Selecting JSON output also +selects JSON diagnostics, so automation does not need to combine two format flags. Diagnostics are +always written to standard error. + +When `--journal -` is selected, standard output contains only the complete JSON Lines journal. The +success document moves to standard error. Its journal artifact is named `stdout`, and the final +`run_completed` record signals successful completion. A consumer must also require a zero process +exit status. Strategy protocol messages remain confined to the supervised child process. + +This result contract is independent of the scenario contract version. Its `valuation` uses the +current v16 valuation shape so consumers receive one stable automation model for older accepted +scenarios. diff --git a/contracts/cli/v1/dune b/contracts/cli/v1/dune new file mode 100644 index 0000000..c85ad1c --- /dev/null +++ b/contracts/cli/v1/dune @@ -0,0 +1,7 @@ +(install + (section share) + (package trading_engine) + (files + (README.md as contracts/cli/v1/README.md) + (result.schema.json as contracts/cli/v1/result.schema.json) + (fixtures/demo.result.json as contracts/cli/v1/fixtures/demo.result.json))) diff --git a/contracts/cli/v1/fixtures/demo.result.json b/contracts/cli/v1/fixtures/demo.result.json new file mode 100644 index 0000000..efb13c6 --- /dev/null +++ b/contracts/cli/v1/fixtures/demo.result.json @@ -0,0 +1 @@ +{"result_version":"1","status":"success","operation":"validate","run_id":"demo","hashes":{"scenario_sha256":"a56b9b38f18d93e2953f90c8b052026d174e91c01a9465f8070a22040820a78d","journal_sha256":null,"strategy_transcript_sha256":null},"counts":{"instruments":1,"schedule_batches":2,"slices":4,"audits":29,"orders":3,"active_orders":0,"filled_orders":2,"rejected_orders":0},"valuation":{"base_currency":"USD","cash":"9846.979929","net_market_value":"265","long_market_value":"265","short_market_value":"0","gross_exposure":"265","cost_basis":"257.154644","realized_pnl":"19.134573","unrealized_pnl":"7.845356","equity":"10111.979929","dividend_pnl":"1","execution_fees":"2.907896","borrow_fees":"0.25","total_fees":"3.157896","cash_balances":[{"currency":"USD","amount":"9846.979929","fx_rate":"1","base_value":"9846.979929","interest":"0.285075","base_interest":"0.285075","settled_amount":"9846.979929","unsettled_amount":"0","base_settled_value":"9846.979929","base_unsettled_value":"0"}],"positions":[{"instrument_id":"demo-equity-acme","quote_currency":"USD","quantity":"2.5","mark":"106","fx_rate":"1","market_value":"265","base_market_value":"265","cost_basis":"257.154644","base_cost_basis":"257.154644","realized_pnl":"18.849498","base_realized_pnl":"18.849498","unrealized_pnl":"7.845356","base_unrealized_pnl":"7.845356","dividend_pnl":"1","base_dividend_pnl":"1","execution_fees":"2.907896","base_execution_fees":"2.907896","borrow_fees":"0.25","base_borrow_fees":"0.25","total_fees":"3.157896","base_total_fees":"3.157896","execution_fee_components":[{"name":"broker","kind":"fixed","currency":"USD","amount":"0.75","quote_currency":"USD","quote_amount":"0.75","base_amount":"0.75"},{"name":"exchange","kind":"notional_bps","currency":"USD","amount":"1.664894","quote_currency":"USD","quote_amount":"1.664894","base_amount":"1.664894"},{"name":"maximum_adjustment","kind":"maximum_adjustment","currency":"USD","amount":"-0.006998","quote_currency":"USD","quote_amount":"-0.006998","base_amount":"-0.006998"}],"settled_quantity":"2.5","unsettled_quantity":"0"}],"execution_fee_components":[{"name":"broker","kind":"fixed","currency":"USD","amount":"0.75","quote_currency":"USD","quote_amount":"0.75","base_amount":"0.75"},{"name":"exchange","kind":"notional_bps","currency":"USD","amount":"1.664894","quote_currency":"USD","quote_amount":"1.664894","base_amount":"1.664894"},{"name":"maximum_adjustment","kind":"maximum_adjustment","currency":"USD","amount":"-0.006998","quote_currency":"USD","quote_amount":"-0.006998","base_amount":"-0.006998"}],"cash_interest":"0.285075","settled_cash":"9846.979929","unsettled_cash":"0"},"artifacts":{"journal":null,"strategy_transcript":null}} diff --git a/contracts/cli/v1/result.schema.json b/contracts/cli/v1/result.schema.json new file mode 100644 index 0000000..2aea165 --- /dev/null +++ b/contracts/cli/v1/result.schema.json @@ -0,0 +1,130 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/fallblu/trading-engine/contracts/cli/v1/result.schema.json", + "title": "Trading Engine CLI success result v1", + "type": "object", + "additionalProperties": false, + "required": [ + "result_version", + "status", + "operation", + "run_id", + "hashes", + "counts", + "valuation", + "artifacts" + ], + "properties": { + "result_version": { "const": "1" }, + "status": { "const": "success" }, + "operation": { "enum": ["validate", "replay"] }, + "run_id": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/identifier" }, + "hashes": { + "type": "object", + "additionalProperties": false, + "required": ["scenario_sha256", "journal_sha256", "strategy_transcript_sha256"], + "properties": { + "scenario_sha256": { "$ref": "#/$defs/sha256" }, + "journal_sha256": { "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] }, + "strategy_transcript_sha256": { "oneOf": [{ "$ref": "#/$defs/sha256" }, { "type": "null" }] } + } + }, + "counts": { + "type": "object", + "additionalProperties": false, + "required": ["instruments", "schedule_batches", "slices", "audits", "orders", "active_orders", "filled_orders", "rejected_orders"], + "properties": { + "instruments": { "$ref": "#/$defs/count" }, + "schedule_batches": { "$ref": "#/$defs/count" }, + "slices": { "$ref": "#/$defs/count" }, + "audits": { "$ref": "#/$defs/count" }, + "orders": { "$ref": "#/$defs/count" }, + "active_orders": { "$ref": "#/$defs/count" }, + "filled_orders": { "$ref": "#/$defs/count" }, + "rejected_orders": { "$ref": "#/$defs/count" } + } + }, + "valuation": { "$ref": "#/$defs/accountValuation" }, + "artifacts": { + "type": "object", + "additionalProperties": false, + "required": ["journal", "strategy_transcript"], + "properties": { + "journal": { "oneOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] }, + "strategy_transcript": { "oneOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] } + } + } + }, + "allOf": [ + { + "if": { "properties": { "operation": { "const": "validate" } } }, + "then": { + "properties": { + "hashes": { "properties": { "journal_sha256": { "type": "null" }, "strategy_transcript_sha256": { "type": "null" } } }, + "artifacts": { "properties": { "journal": { "type": "null" }, "strategy_transcript": { "type": "null" } } } + } + } + }, + { + "if": { "properties": { "operation": { "const": "replay" } } }, + "then": { + "properties": { + "hashes": { "properties": { "journal_sha256": { "$ref": "#/$defs/sha256" } } }, + "artifacts": { "properties": { "journal": { "type": "string", "minLength": 1 } } } + } + } + } + ], + "$defs": { + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "count": { "type": "integer", "minimum": 0 }, + "accountValuation": { + "type": "object", + "additionalProperties": false, + "required": [ + "base_currency", + "cash", + "settled_cash", + "unsettled_cash", + "net_market_value", + "long_market_value", + "short_market_value", + "gross_exposure", + "cost_basis", + "realized_pnl", + "unrealized_pnl", + "equity", + "dividend_pnl", + "execution_fees", + "borrow_fees", + "cash_interest", + "total_fees", + "cash_balances", + "positions", + "execution_fee_components" + ], + "properties": { + "base_currency": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/identifier" }, + "cash": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "settled_cash": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "unsettled_cash": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "net_market_value": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "long_market_value": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/unsignedDecimal" }, + "short_market_value": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/unsignedDecimal" }, + "gross_exposure": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/unsignedDecimal" }, + "cost_basis": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "realized_pnl": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "unrealized_pnl": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "equity": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "dividend_pnl": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "execution_fees": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "borrow_fees": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "cash_interest": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "total_fees": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/signedDecimal" }, + "cash_balances": { "type": "array", "minItems": 1, "items": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/cashAttribution" } }, + "positions": { "type": "array", "items": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/positionAttribution" } }, + "execution_fee_components": { "type": "array", "items": { "$ref": "https://github.com/fallblu/trading-engine/contracts/v16/journal.schema.json#/$defs/feeComponentAttribution" } } + } + } + } +} diff --git a/contracts/conformance/manifest.json b/contracts/conformance/manifest.json index 6e7d864..fe01d3c 100644 --- a/contracts/conformance/manifest.json +++ b/contracts/conformance/manifest.json @@ -1242,6 +1242,18 @@ "format": "jsonl" } ] + }, + { + "name": "cli-result-v1", + "schema": "cli/v1/result.schema.json", + "version_field": "result_version", + "version": "1", + "sources": [ + { + "path": "cli/v1/fixtures/demo.result.json", + "format": "json" + } + ] } ] } diff --git a/contracts/diagnostic/v1/README.md b/contracts/diagnostic/v1/README.md index 7c62f8b..ecfb773 100644 --- a/contracts/diagnostic/v1/README.md +++ b/contracts/diagnostic/v1/README.md @@ -1,7 +1,7 @@ # Diagnostic contract v1 This directory defines the stable JSON emitted on standard error when the CLI uses -`--diagnostic-format json`. Validate each complete document against +`--diagnostic-format json` or `--output-format json`. Validate each complete document against [`diagnostic.schema.json`](diagnostic.schema.json). The `code` and typed `context` fields are the machine contract. Treat `message`, cause messages, diff --git a/docs/diagnostics.md b/docs/diagnostics.md index dad9b59..537b92e 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -2,7 +2,8 @@ Process and file boundaries return diagnostic contract version `1`. The CLI prints the concise `message` by default. Pass `--diagnostic-format json` to write one machine-readable diagnostic to -standard error. The process exits with status 123 for either format. +standard error. `--output-format json` also selects JSON diagnostics so an automation client needs +only one format option. The process exits with status 123 for either format. The versioned [diagnostic JSON Schema](../contracts/diagnostic/v1/diagnostic.schema.json) is the authoritative structural contract. The adjacent fixture demonstrates every optional context diff --git a/docs/scenario.md b/docs/scenario.md index c4c7292..a15d7a3 100644 --- a/docs/scenario.md +++ b/docs/scenario.md @@ -14,6 +14,10 @@ trading-engine --input scenario.json --validate-only trading-engine --input scenario.jsonl --input-format jsonl --validate-only ``` +Use `--input - --input-format jsonl` to read a stream from standard input. The CLI spools at most +1 GiB to a private temporary file so the same bytes can be hashed, validated, replayed, and hashed +again. Batch JSON cannot use standard input. + ## JSON Lines stream Use the stream for histories that should not be materialized inside the engine. The first record @@ -67,6 +71,14 @@ the scenario contract or the separate strategy protocol, never both. Each JSON Lines record is limited to 1 MiB, excluding its line feed. The reader accepts a final record without a line feed and drains an oversized record without retaining bytes above the limit. +Use `--journal -` to write a journal to standard output. The engine first creates and verifies a +complete temporary journal, then copies only journal bytes to the pipe. The final `run_completed` +record and a zero process exit status signal completeness. Success summaries move to standard error, +and diagnostics always use standard error, so protocol, journal, and summary bytes never share one +stream. Pipes do not provide exclusive no-replace publication, atomic linking, retained partial +artifacts, directory synchronization, or restart durability. They cannot be combined with +`--durable-artifacts`. + ## Instruments, risk, and execution Each instrument contains `instrument_id`, `symbol`, `quote_currency`, `tick_size`, and `lot_size`. diff --git a/lib/codec.mli b/lib/codec.mli index f48c608..ed39a34 100644 --- a/lib/codec.mli +++ b/lib/codec.mli @@ -16,5 +16,9 @@ val order_to_yojson_v8 : Order.t -> Yojson.Safe.t val fill_to_yojson : Fill.t -> Yojson.Safe.t val fill_to_yojson_v9 : Fill.t -> Yojson.Safe.t val initial_portfolio_to_yojson : Initial_portfolio.t -> Yojson.Safe.t + +val account_valuation_to_yojson : + ?contract_version:string -> Account.valuation -> Yojson.Safe.t + val audit_to_yojson : Audit.t -> Yojson.Safe.t val audit_to_string : Audit.t -> string diff --git a/lib/resource_limits.ml b/lib/resource_limits.ml index 6ba78ac..a2e0bd3 100644 --- a/lib/resource_limits.ml +++ b/lib/resource_limits.ml @@ -1,5 +1,6 @@ let version = "1" let scenario_record_bytes = 1_048_576 +let scenario_stream_bytes = 1_073_741_824 let strategy_message_bytes = 1_048_576 let internal_events = 100_000 let catalog_instruments = 4_096 diff --git a/lib/resource_limits.mli b/lib/resource_limits.mli index 682453c..15657a2 100644 --- a/lib/resource_limits.mli +++ b/lib/resource_limits.mli @@ -5,6 +5,7 @@ val version : string val scenario_record_bytes : int +val scenario_stream_bytes : int val strategy_message_bytes : int val internal_events : int val catalog_instruments : int diff --git a/mkdocs.yml b/mkdocs.yml index 0b4fb2f..ec574e8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,6 +26,8 @@ nav: - Security policy: SECURITY.md - Contracts: - Conformance corpus: contracts/conformance/README.md + - CLI results: + - Current v1: contracts/cli/v1/README.md - Diagnostics: - Current v1: contracts/diagnostic/v1/README.md - Scenario and journal: diff --git a/scripts/check-documentation.py b/scripts/check-documentation.py index 39a2038..5ae4a7b 100644 --- a/scripts/check-documentation.py +++ b/scripts/check-documentation.py @@ -26,6 +26,7 @@ "docs/persistra.md", "SECURITY.md", "contracts/conformance/README.md", + "contracts/cli/v1/README.md", "contracts/v16/README.md", "contracts/v5/README.md", "contracts/v4/README.md", diff --git a/scripts/release_artifacts.py b/scripts/release_artifacts.py index 6da23e8..aceeddc 100644 --- a/scripts/release_artifacts.py +++ b/scripts/release_artifacts.py @@ -378,6 +378,7 @@ def verify_release( "lib/trading_engine/opam", "share/trading_engine/contracts/v16/scenario.schema.json", "share/trading_engine/contracts/v16/fixtures/demo.scenario.json", + "share/trading_engine/contracts/cli/v1/result.schema.json", "doc/trading_engine/README.md", ), epoch, @@ -389,6 +390,7 @@ def verify_release( "trading_engine.opam", "contracts/v1/scenario.schema.json", "contracts/v16/fixtures/demo.scenario.json", + "contracts/cli/v1/result.schema.json", "docs/architecture.md", ".github/workflows/release-candidate.yml", ), @@ -402,6 +404,7 @@ def verify_release( "contracts/v1/scenario.schema.json", "contracts/v16/fixtures/demo.scenario.json", "contracts/strategy/v14/message.schema.json", + "contracts/cli/v1/result.schema.json", ), epoch, ) @@ -413,6 +416,7 @@ def verify_release( "docs/architecture/index.html", "contracts/v1/index.html", "contracts/v16/scenario.schema.json", + "contracts/cli/v1/result.schema.json", "api/trading_engine/Trading_engine/index.html", ), epoch, diff --git a/test/cli.t b/test/cli.t index 6ce3ee0..22bbca7 100644 --- a/test/cli.t +++ b/test/cli.t @@ -10,6 +10,11 @@ $ ../bin/main.exe --validate-only --input-format jsonl --input ../contracts/v8/fixtures/demo.scenario.jsonl valid run=demo instruments=1 schedule=2 slices=4 scenario_sha256=786f38d8bd10faac03b6b15c7aa8ae0a867eedc609ca6eaa75cfd93ae3ffdcae + $ ../bin/main.exe --output-format json --validate-only --input ../contracts/v8/fixtures/demo.scenario.json | python3 -c 'import json, sys; result=json.load(sys.stdin); print(result["result_version"], result["status"], result["operation"], result["run_id"]); print(result["counts"]["instruments"], result["counts"]["slices"], result["counts"]["audits"], result["valuation"]["equity"]); print(result["hashes"]["journal_sha256"], result["artifacts"]["journal"])' + 1 success validate demo + 1 4 22 10111.65392 + None None + $ ../bin/main.exe --input-format jsonl --input ../contracts/v8/fixtures/demo.scenario.jsonl --journal streamed.journal.jsonl --durable-artifacts run=demo audits=22 orders=3 active=0 filled=2 rejected=0 cash=9846.65392 equity=10111.65392 gross=265 realized=18.965682 unrealized=7.688238 fees=3.16608 @@ -32,6 +37,12 @@ 1 scenario_stream.invalid validation 6 6 None + $ diagnostic=$(../bin/main.exe --output-format json --validate-only --input-format jsonl --input truncated.scenario.jsonl 2>&1 >/dev/null); status=$?; test "$status" -eq 123; python3 -c 'import json, sys; diagnostic=json.loads(sys.argv[1]); print(diagnostic["diagnostic_version"], diagnostic["code"], diagnostic["phase"])' "$diagnostic" + 1 scenario_stream.invalid validation + + $ ../bin/main.exe --output-format json --validate-only --input missing.scenario.json 2>&1 >/dev/null | python3 -c 'import json, sys; diagnostic=json.load(sys.stdin); print(diagnostic["code"], diagnostic["phase"])' + input.io input + $ sed 's/"open": "100"/"open": "100.001"/' ../contracts/v8/fixtures/demo.scenario.json > invalid-tick.json $ ../bin/main.exe --validate-only --input invalid-tick.json trading-engine: market prices and volumes must align with instrument increments @@ -47,6 +58,36 @@ $ test ! -e validation.journal.jsonl + $ ../bin/main.exe --input-format jsonl --input ../contracts/v8/fixtures/demo.scenario.jsonl --journal piped-file.journal.jsonl >/dev/null + $ cat ../contracts/v8/fixtures/demo.scenario.jsonl | ../bin/main.exe --input-format jsonl --input - --journal - > piped-stdout.journal.jsonl 2> piped-summary.txt + $ cmp piped-file.journal.jsonl piped-stdout.journal.jsonl + $ python3 - piped-summary.txt piped-stdout.journal.jsonl <<'PY' + > import hashlib + > import json + > import sys + > summary_path, journal_path = sys.argv[1:] + > summary = open(summary_path, encoding="utf-8").read() + > journal_bytes = open(journal_path, "rb").read() + > completion = json.loads(journal_bytes.splitlines()[-1]) + > print("journal=stdout" in summary, completion["event_type"]) + > print(len(journal_bytes.splitlines()), hashlib.sha256(journal_bytes).hexdigest() == hashlib.sha256(open("piped-file.journal.jsonl", "rb").read()).hexdigest()) + > PY + True run_completed + 22 True + + $ ../bin/main.exe --output-format json --input-format jsonl --input ../contracts/v8/fixtures/demo.scenario.jsonl --journal - > piped-json.journal.jsonl 2> piped-json-summary.json + $ python3 -c 'import json; result=json.load(open("piped-json-summary.json")); print(result["result_version"], result["operation"], result["artifacts"]["journal"], result["hashes"]["journal_sha256"] is not None)' + 1 replay stdout True + $ cmp piped-file.journal.jsonl piped-json.journal.jsonl + + $ printf '{}\n' | ../bin/main.exe --input - --journal - + trading-engine: standard input requires --input-format jsonl; batch JSON is not supported + [123] + + $ ../bin/main.exe --input-format jsonl --input ../contracts/v8/fixtures/demo.scenario.jsonl --journal - --durable-artifacts + trading-engine: --durable-artifacts cannot be used when --journal writes to standard output + [123] + $ ../bin/main.exe --validate-only --durable-artifacts --input ../contracts/v8/fixtures/demo.scenario.json trading-engine: --durable-artifacts cannot be used with --validate-only [123] @@ -67,6 +108,16 @@ 12 14 $ diff -u ../contracts/strategy/v6/fixtures/external.strategy.jsonl external/run.strategy.jsonl + $ mkdir external-json + $ ../bin/main.exe --output-format json --input ../contracts/strategy/v6/fixtures/external.scenario.json --journal external-json/run.journal.jsonl --strategy-executable ./fake_strategy.py --strategy-transcript external-json/run.strategy.jsonl --strategy-timeout 5 | python3 -c 'import hashlib, json, sys; result=json.load(sys.stdin); digest=lambda path: hashlib.sha256(open(path, "rb").read()).hexdigest(); print(result["operation"], result["run_id"], result["artifacts"]["strategy_transcript"]); print(result["hashes"]["journal_sha256"] == digest(result["artifacts"]["journal"]), result["hashes"]["strategy_transcript_sha256"] == digest(result["artifacts"]["strategy_transcript"]))' + replay external-demo external-json/run.strategy.jsonl + True True + + $ ../bin/main.exe --input ../contracts/strategy/v6/fixtures/external.scenario.json --journal ignored-stdout.journal.jsonl --strategy-executable ./fake_strategy.py --strategy-transcript - + trading-engine: --strategy-transcript does not support standard output; choose a file path + [123] + $ test ! -e ignored-stdout.journal.jsonl + $ mkdir callback-ordering $ ../bin/main.exe --input ../contracts/strategy/v6/fixtures/external.scenario.json --journal callback-ordering/run.journal.jsonl --strategy-executable ./fake_strategy.py --strategy-arg cancel-next --strategy-transcript callback-ordering/run.strategy.jsonl --strategy-timeout 5 run=external-demo audits=12 orders=2 active=0 filled=1 rejected=0 diff --git a/test/dune b/test/dune index e68f723..aba6db0 100644 --- a/test/dune +++ b/test/dune @@ -755,3 +755,20 @@ ../contracts/v9/fixtures/demo.scenario.json) (action (run python3 %{dep:test_benchmark_replay.py}))) + +(rule + (alias runtest) + (deps + validate_cli_result.py + ../contracts/cli/v1/result.schema.json + ../contracts/v16/journal.schema.json + ../contracts/v16/fixtures/demo.scenario.json + ../bin/main.exe) + (action + (run + python3 + %{dep:validate_cli_result.py} + %{dep:../contracts/cli/v1/result.schema.json} + %{dep:../contracts/v16/journal.schema.json} + %{dep:../bin/main.exe} + %{dep:../contracts/v16/fixtures/demo.scenario.json}))) diff --git a/test/validate_cli_result.py b/test/validate_cli_result.py new file mode 100644 index 0000000..62664b6 --- /dev/null +++ b/test/validate_cli_result.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Validate live CLI success results against their versioned JSON Schema.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + + +def load(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def run(binary: Path, *arguments: str) -> tuple[object, str]: + completed = subprocess.run( + [str(binary), "--output-format", "json", *arguments], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout), completed.stderr + + +def main() -> None: + if len(sys.argv) != 5: + raise SystemExit( + "usage: validate_cli_result.py RESULT_SCHEMA JOURNAL_SCHEMA BINARY SCENARIO" + ) + result_path, journal_path, binary, scenario = map(Path, sys.argv[1:]) + result_schema = load(result_path) + journal_schema = load(journal_path) + Draft202012Validator.check_schema(result_schema) + registry = Registry().with_resource( + journal_schema["$id"], Resource.from_contents(journal_schema) + ) + validator = Draft202012Validator(result_schema, registry=registry) + + validation, validation_stderr = run( + binary, "--validate-only", "--input", str(scenario) + ) + validator.validate(validation) + assert validation_stderr == "" + assert validation["operation"] == "validate" + + with tempfile.TemporaryDirectory() as directory: + journal = Path(directory) / "run.journal.jsonl" + replay, replay_stderr = run( + binary, "--input", str(scenario), "--journal", str(journal) + ) + validator.validate(replay) + assert replay_stderr == "" + assert replay["operation"] == "replay" + assert replay["hashes"]["journal_sha256"] == hashlib.sha256( + journal.read_bytes() + ).hexdigest() + + +if __name__ == "__main__": + main()