diff --git a/src/core/resolver/hb_message.erl b/src/core/resolver/hb_message.erl index 425eadbfd..eea24fad9 100644 --- a/src/core/resolver/hb_message.erl +++ b/src/core/resolver/hb_message.erl @@ -58,6 +58,7 @@ -module(hb_message). -export([id/1, id/2, id/3]). -export([convert/3, convert/4, uncommitted/1, uncommitted/2, committed/3]). +-export([add_bundle_hint/2, add_bundle_hint/3]). -export([with_only_committers/2, with_only_committers/3, commitment_devices/2]). -export([verify/1, verify/2, verify/3, paranoid_verify/2, paranoid_verify/3]). -export([commit/2, commit/3, signers/2, type/1, minimize/1]). @@ -102,6 +103,7 @@ convert(Msg, TargetFormat, SourceFormat, Opts) -> true -> hb_maps:without([<<"priv">>], Msg, Opts); false -> Msg end, + TargetFormat, SourceFormat, Opts ), @@ -110,8 +112,9 @@ convert(Msg, TargetFormat, SourceFormat, Opts) -> _ -> from_tabm(TABM, TargetFormat, OldPriv, Opts) end. -to_tabm(Msg, SourceFormat, Opts) -> - {SourceCodecMod, Params} = conversion_spec_to_req(SourceFormat, Opts), +to_tabm(Msg, TargetFormat, SourceFormat, Opts) -> + {SourceCodecMod, Params0} = conversion_spec_to_req(SourceFormat, Opts), + Params = add_bundle_hint(Params0, TargetFormat, Opts), % We use _from_ here because the codecs are labelled from the perspective % of their own format. `dev_codec_ans104:from/1' will convert _from_ % an ANS-104 message _into_ a TABM. @@ -121,6 +124,51 @@ to_tabm(Msg, SourceFormat, Opts) -> {ok, OtherTypeRes} -> OtherTypeRes end. +%% @doc Extract the device value from a conversion spec. +conversion_spec_device(Spec, _Default, _Opts) + when is_binary(Spec) orelse (Spec == tabm) -> + Spec; +conversion_spec_device(Spec, Default, Opts) when is_map(Spec) -> + hb_maps:get(<<"device">>, Spec, Default, Opts); +conversion_spec_device(_Spec, Default, _Opts) -> + Default. + +%% @doc Extend a structured->tabm source spec with the `bundle' flag and +%% `hint-device' implied by a hint spec, so the structured codec can decide +%% whether to load or offload children and can call the target codec's +%% `to_hint/3' callback at each node of the tree. +%% +%% `Spec' is the spec being extended (the source spec when converting). +%% `HintSpec' is the spec from which we should infer bundling +%% (target spec when converting). +add_bundle_hint(Spec, Opts) -> + add_bundle_hint(Spec, Spec, Opts). +add_bundle_hint(Spec, HintSpec, Opts) -> + WithBundle = + case maps:is_key(<<"bundle">>, Spec) of + true -> + Spec; + false -> + case + is_map(HintSpec) + andalso hb_maps:find(<<"bundle">>, HintSpec, Opts) + of + {ok, Bundle} -> Spec#{ <<"bundle">> => Bundle }; + _ -> Spec + end + end, + case maps:is_key(<<"hint-device">>, WithBundle) of + true -> + WithBundle; + false -> + case conversion_spec_device(HintSpec, undefined, Opts) of + HintDevice when is_binary(HintDevice) -> + WithBundle#{ <<"hint-device">> => HintDevice }; + _ -> + WithBundle + end + end. + from_tabm(Msg, TargetFormat, OldPriv, Opts) -> {TargetCodecMod, Params} = conversion_spec_to_req(TargetFormat, Opts), % We use the _to_ function here because each of the codecs we may call in @@ -147,17 +195,15 @@ restore_priv(Msg, OldPriv, Opts) -> %% Expects conversion spec to either be a binary codec name, or a map with a %% `device' key and other parameters. Additionally honors the `always_bundle' %% key in the node message if present. -conversion_spec_to_req(Spec, Opts) when is_binary(Spec) or (Spec == tabm) -> +conversion_spec_to_req(Spec, Opts) when is_binary(Spec) orelse (Spec == tabm) -> conversion_spec_to_req(#{ <<"device">> => Spec }, Opts); conversion_spec_to_req(Spec, Opts) -> try - Device = - hb_maps:get( - <<"device">>, - Spec, - no_codec_device_in_conversion_spec, - Opts - ), + Device = conversion_spec_device( + Spec, + no_codec_device_in_conversion_spec, + Opts + ), { case Device of tabm -> tabm; diff --git a/src/dev_message_bundle_test_vectors.erl b/src/dev_message_bundle_test_vectors.erl new file mode 100644 index 000000000..30f7710a9 --- /dev/null +++ b/src/dev_message_bundle_test_vectors.erl @@ -0,0 +1,179 @@ +%%% @doc A battery of test vectors exercising the `bundle' / `hint-device' +%%% machinery of the `message@1.0' device across a three-level message tree. +%%% +%%% The tree is built bottom-up; each level holds the level below it as a +%%% sub-message: +%%% +%%% 
+%%%     L1 (root) --> L2 (middle) --> L3 (leaf) --> #{}
+%%%
+%%% +%%% Each level is built with one of four choices: +%%% +%%% - `bundle_true' -- committed (`ans104@1.0') with `bundle' => true +%%% - `bundle_false' -- committed with `bundle' => false +%%% - `no_bundle' -- committed with no `bundle' flag +%%% - `uncommitted' -- not committed at all; a plain, unsigned map +%%% +%%% For every 4x4x4 permutation of build choices the suite checks: +%%% +%%% - verify/3: every level verifies in the state it was committed in. +%%% - id/3: the root's id equals its sole commitment's key (or, for an +%%% uncommitted root, the content-addressed unsigned id). +%%% - convert/4 with target `bundle' none/true/false: the tree +%%% round-trips through the `ans104@1.0' codec -- the standard +%%% structured<->codec path -- and still verifies at every level. A +%%% `bundle' on the conversion target applies only to the root; nested +%%% subtrees follow their own commitments via `hint-device', so the +%%% committed shape survives the round-trip. +-module(dev_message_bundle_test_vectors). +-include_lib("eunit/include/eunit.hrl"). +-include("include/hb.hrl"). + +fresh_opts() -> + #{ + <<"priv-wallet">> => hb:wallet(), + <<"store">> => hb_test_utils:test_store() + }. + +%% @doc Build one tree level from its build choice: commit the message with +%% the `ans104@1.0' codec (with `bundle' => true, `bundle' => false, or no +%% `bundle' flag), or -- for `uncommitted' -- leave it as a plain map. +build_level(Msg, uncommitted, _Opts) -> + Msg; +build_level(Msg, no_bundle, Opts) -> + hb_message:commit(Msg, Opts, #{ <<"device">> => <<"ans104@1.0">> }); +build_level(Msg, bundle_true, Opts) -> + hb_message:commit( + Msg, + Opts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true } + ); +build_level(Msg, bundle_false, Opts) -> + hb_message:commit( + Msg, + Opts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false } + ). + +%% @doc Build a three-level tree with the given per-level build choices. +build_tree(B1, B2, B3, Opts) -> + L3 = + build_level( + #{ + <<"l3-tag">> => <<"l3-value">>, + <<"inner">> => #{ <<"deep">> => <<"deep-value">> } + }, + B3, + Opts + ), + L2 = + build_level(#{ <<"l2-tag">> => <<"l2-value">>, <<"l3">> => L3 }, B2, Opts), + build_level(#{ <<"l1-tag">> => <<"l1-value">>, <<"l2">> => L2 }, B1, Opts). + +%%% Test vector generator. + +%% @doc The {API, RequestBundle} operations run against every tree shape. +operations() -> + [ + {verify, none}, + {id, none}, + {convert, none}, + {convert, true}, + {convert, false} + ]. + +%% @doc The per-level build choices a tree level can take. +build_choices() -> + [bundle_true, bundle_false, no_bundle, uncommitted]. + +%% @doc Generate the full grid: 4x4x4 tree shapes x the operation list. +bundle_vectors_test_() -> + {timeout, 600, + [ + { + test_label(B1, B2, B3, Api, ReqBundle), + fun() -> run(B1, B2, B3, Api, ReqBundle) end + } + || + B1 <- build_choices(), + B2 <- build_choices(), + B3 <- build_choices(), + {Api, ReqBundle} <- operations() + ] + }. + +test_label(B1, B2, B3, Api, ReqBundle) -> + lists:flatten( + io_lib:format( + "L1=~p L2=~p L3=~p ~p req-bundle=~p", + [B1, B2, B3, Api, ReqBundle] + ) + ). + +%% @doc Build the tree and run_test the chosen API. +run(B1, B2, B3, Api, ReqBundle) -> + Opts = fresh_opts(), + Tree = build_tree(B1, B2, B3, Opts), + ?event(debug_test, {tree, + {label, test_label(B1, B2, B3, Api, ReqBundle)}, + {built, Tree}}), + % Every freshly built tree must verify via the reliable per-node path, + % whatever per-level bundle permutation it was signed with. + ?assert(hb_message:verify(Tree, all, Opts)), + run_test(Api, ReqBundle, B1, B2, B3, Tree, Opts). + +%%% Per-API run_tests. + +%% `verify': every level of a validly-built tree verifies. The bundle +%% state each subtree was committed in is reproduced per-node via +%% `hint-device', so the verify request carries no `bundle'. (`run/3' +%% already verifies the root, so this only adds the nested levels.) +run_test(verify, _ReqBundle, _B1, _B2, _B3, Tree, Opts) -> + L2 = hb_maps:get(<<"l2">>, Tree, undefined, Opts), + ?assert(hb_message:verify(L2, all, Opts)), + L3 = hb_maps:get(<<"l3">>, L2, undefined, Opts), + ?assert(hb_message:verify(L3, all, Opts)); + +%% `id': +%% - committed root: `id/3' with `all' committers accumulates to the +%% single commitment -- the id must equal the key under which it is +%% stored in the root's commitments map. +%% - uncommitted root: there are no commitments, so `id/3' falls back to +%% the (content-addressed) unsigned id -- `all' committers must give +%% the same result as the bare unsigned-id call. +run_test(id, _ReqBundle, uncommitted, _B2, _B3, Tree, Opts) -> + ?assertEqual( + hb_message:id(Tree, none, Opts), + hb_message:id(Tree, all, Opts) + ); +run_test(id, _ReqBundle, _B1, _B2, _B3, Tree, Opts) -> + Id = hb_message:id(Tree, all, Opts), + Commitments = hb_maps:get(<<"commitments">>, Tree, #{}, Opts), + ?assertEqual([Id], maps:keys(Commitments)); + +%% `convert': round-trip the tree through the `ans104@1.0' codec. Each subtree +%% converts in the state its own commitment dictates (per-node) via +%% `hint-device', so the `bundle' on the conversion target applies only to the +%% root and the committed shape is preserved. +run_test(convert, ReqBundle, _B1, _B2, _B3, Tree, Opts) -> + Encoded = hb_message:convert(Tree, convert_target(ReqBundle), Opts), + Restored = + hb_message:convert( + Encoded, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + ?assert(hb_message:verify(Restored, all, Opts)), + L2 = hb_maps:get(<<"l2">>, Restored, undefined, Opts), + ?assert(hb_message:verify(L2, all, Opts)), + L3 = hb_maps:get(<<"l3">>, L2, undefined, Opts), + ?assert(hb_message:verify(L3, all, Opts)). + +%% @doc The convert target for a request-bundle value: the bare `ans104@1.0' +%% codec, plus a forced `bundle' flag when one is given. +convert_target(none) -> + <<"ans104@1.0">>; +convert_target(ReqBundle) -> + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ReqBundle }. diff --git a/src/preloaded/arweave/dev_bundler.erl b/src/preloaded/arweave/dev_bundler.erl index 98112fdd9..126226df0 100644 --- a/src/preloaded/arweave/dev_bundler.erl +++ b/src/preloaded/arweave/dev_bundler.erl @@ -659,6 +659,109 @@ nested_bundle_test_parallel() -> stop_test_servers(ServerHandle, NodeOpts) end. +%% @doc End-to-end bundler test for a nested dataitem where the parent +%% has bundle=false. The chile is posted on its own first. +nested_unbundled_bundle_child_posted_test_parallel() -> + run_nested_unbundled_bundle_test(child_posted). + +%% @doc Like `nested_inlined_bundle_child_posted_test_parallel/0', but the +%% child is never posted on its own. +nested_unbundled_bundle_child_not_posted_test_parallel() -> + run_nested_unbundled_bundle_test(child_not_posted). + +run_nested_unbundled_bundle_test(Variant) -> + Anchor = rand:bytes(32), + Price = 12345, + % NodeOpts redirects arweave gateway requests to the mock server. + {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway( + #{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + } + ), + try + ClientOpts = #{ <<"priv-wallet">> => ar_wallet:new() }, + NodeOpts2 = maps:merge(NodeOpts, #{ <<"bundler-max-items">> => 3 }), + Node = hb_http_server:start_node(NodeOpts2#{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => hb_test_utils:test_store() + }), + %% Child: an `httpsig@1.0'-signed message + Child = hb_message:commit( + #{ + <<"event">> => <<"is_admissible">>, + <<"reference">> => <<"ref-value">>, + <<"status-class">> => <<"success">> + }, + ClientOpts, + #{ <<"device">> => <<"httpsig@1.0">> } + ), + ?assert(hb_message:verify(Child, all, ClientOpts)), + %% Parent: signed with `ans104@1.0' and `bundle' => false, so the + %% child is offloaded as a link in the parent's committed form. + Parent = hb_message:commit( + #{ + <<"data-protocol">> => <<"ao">>, + <<"type">> => <<"Assignment">>, + <<"body">> => Child + }, + ClientOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false } + ), + ?assert(hb_message:verify(Parent, all, ClientOpts)), + %% Post the first bundle slot (per `Variant'), then the nested + %% parent, then a plain data item. + ?assertMatch({ok, _}, post_first_item(Node, Variant, Child, ClientOpts)), + ?assertMatch({ok, _}, post_structured_item(Node, Parent, ClientOpts)), + ?assertMatch({ok, _}, + post_data_item(Node, new_data_item(2, 10), ClientOpts)), + %% The three items bundle into a single transaction. + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + Proofs = hb_mock_server:get_requests(chunk, 1, ServerHandle), + ?assert(length(Proofs) >= 1), + %% Reconstitute the bundle TX and verify it carries three valid items. + TX = reconstitute_tx(hd(TXs), Proofs), + ?event(debug_test, {tx, TX}), + ?assert(ar_tx:verify(TX)), + ?assertEqual(Anchor, TX#tx.anchor), + ?assertEqual(Price, TX#tx.reward), + Bundle = ar_bundles:deserialize(TX), + ?assertEqual(3, maps:size(Bundle#tx.data)), + %% Each bundled item must still verify once decoded back to + %% `structured@1.0'. + maps:foreach( + fun(_Key, BundledItem) -> + Structured = hb_message:convert( + BundledItem, + <<"structured@1.0">>, + <<"ans104@1.0">>, + ClientOpts + ), + ?assert(hb_message:verify(Structured, all, ClientOpts)) + end, + Bundle#tx.data + ), + %% The bundle TX must convert to `structured@1.0' and verify, then + %% round-trip back to `tx@1.0' without inflating its data. + TXStructured = hb_message:convert( + TX, <<"structured@1.0">>, <<"tx@1.0">>, ClientOpts), + ?assert(hb_message:verify(TXStructured, all, ClientOpts)), + TXRoundtrip = hb_message:convert( + TXStructured, <<"tx@1.0">>, <<"structured@1.0">>, ClientOpts), + ?assertEqual(byte_size(TX#tx.data), byte_size(TXRoundtrip#tx.data)), + ?assert(ar_tx:verify(TXRoundtrip)), + ok + after + %% Always cleanup, even if test fails + stop_test_servers(ServerHandle, NodeOpts) + end. + +post_first_item(Node, child_posted, Child, ClientOpts) -> + post_structured_item(Node, Child, ClientOpts); +post_first_item(Node, child_not_posted, _Child, ClientOpts) -> + post_data_item(Node, new_data_item(1, 10), ClientOpts). + price_error_test_parallel() -> test_api_error(#{ price => {500, <<"error">>}, @@ -1522,6 +1625,10 @@ post_data_item(Node, Item, Opts) -> <<"ans104@1.0">>, Opts ), + post_structured_item(Node, StructuredItem, Opts). + +%% @doc Post an already-`structured@1.0' message to the bundler endpoint. +post_structured_item(Node, StructuredItem, Opts) -> hb_http:post( Node, #{ @@ -1532,8 +1639,11 @@ post_data_item(Node, Item, Opts) -> Opts ). -assert_bundle(Node, ExpectedItems, Anchor, Price, TXRequest, Proofs, ClientOpts) -> - %% Reconstitute the transaction with its data from the POSTed payloads. +%% @doc Reconstitute a bundle transaction from a captured `tx' request and +%% its `chunk' proof requests: decode the header, validate every chunk's +%% merkle path, then concatenate the chunks in offset order to recover the +%% transaction data. +reconstitute_tx(TXRequest, Proofs) -> TXBinary = maps:get(<<"body">>, TXRequest), TXJSON = hb_json:decode(TXBinary), TXHeader = ar_tx:json_struct_to_tx(TXJSON), @@ -1558,8 +1668,11 @@ assert_bundle(Node, ExpectedItems, Anchor, Price, TXRequest, Proofs, ClientOpts) ), SortedChunks = lists:sort(fun({O1, _}, {O2, _}) -> O1 =< O2 end, ChunksWithOffsets), Chunks = [Chunk || {_Offset, Chunk} <- SortedChunks], - DataBinary = iolist_to_binary(Chunks), - TX = TXHeader#tx{ data = DataBinary }, + TXHeader#tx{ data = iolist_to_binary(Chunks) }. + +assert_bundle(Node, ExpectedItems, Anchor, Price, TXRequest, Proofs, ClientOpts) -> + %% Reconstitute the transaction with its data from the POSTed payloads. + TX = reconstitute_tx(TXRequest, Proofs), ?event(debug_test, {tx, TX}), ?assert(ar_tx:verify(TX)), ?assertEqual(Anchor, TX#tx.anchor), diff --git a/src/preloaded/arweave/dev_bundler_task.erl b/src/preloaded/arweave/dev_bundler_task.erl index cd00d989e..e7992ac9e 100644 --- a/src/preloaded/arweave/dev_bundler_task.erl +++ b/src/preloaded/arweave/dev_bundler_task.erl @@ -33,10 +33,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> case build_signed_tx(Items, Opts) of {ok, SignedTX} -> Committed = hb_message:convert( - SignedTX, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - Opts), + SignedTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ?event(bundler_short, log_task(posting_tx, Task, [{tx, {explicit, hb_message:id(Committed, signed, Opts)}}] @@ -185,7 +182,7 @@ build_signed_tx(Items, Opts) -> data_items_to_tx(Items, Opts) -> List = lists:map( - fun(Item) -> + fun(Item) -> hb_message:convert( Item, #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, @@ -251,71 +248,6 @@ format_timestamp() -> Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). -build_signed_tx_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - TestOpts = NodeOpts#{ - <<"priv-wallet">> => ar_wallet:new(), - <<"store">> => hb_test_utils:test_store() - }, - try - Timestamp = 12344567, - ListValue = [<<"a">>, <<"b">>, <<"c">>], - StructuredItems = [ - #{ - <<"body">> => <<"body1">>, - <<"tag1">> => <<"value1">>, - <<"timestamp">> => Timestamp - }, - #{ - <<"body">> => <<"body3">>, - <<"tag3">> => <<"value3">>, - <<"list">> => ListValue - }, - #{ - <<"body">> => <<"body2">>, - <<"tag2">> => <<"value2">> - } - ], - Items = [ - hb_message:commit( - Item, - TestOpts, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true } - ) - || Item <- StructuredItems], - {ok, SignedTX} = build_signed_tx(Items, TestOpts), - ?assert(ar_tx:verify(SignedTX)), - ?assertEqual(Anchor, SignedTX#tx.anchor), - ?assertEqual(Price, SignedTX#tx.reward), - ?event(debug_test, {signed_tx, SignedTX}), - BundledTX = ar_bundles:deserialize(SignedTX), - ?event(debug_test, {bundled_tx, BundledTX}), - BundledItems = hb_util:numbered_keys_to_list(BundledTX#tx.data, #{}), - lists:foreach( - fun(Item) -> - ?assert(ar_bundles:verify_item(Item)) - end, - BundledItems - ), - BundledStructuredItems = [ - hb_message:convert( - Item, - <<"structured@1.0">>, - <<"ans104@1.0">>, - TestOpts - ) - || Item <- BundledItems], - ?assertEqual(lists:reverse(Items), BundledStructuredItems), - ok - after - hb_mock_server:stop(ServerHandle) - end. - build_signed_tx_on_arbundles_js_test() -> Anchor = rand:bytes(32), Price = 12345, @@ -351,14 +283,14 @@ build_signed_tx_on_arbundles_js_test() -> ?assert(ar_bundles:verify_item(BundledItem)), % Convert both dataitems to structured messages ItemStructured = hb_message:convert(Item, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, + <<"ans104@1.0">>, TestOpts), ?event(debug_test, {item_structured, ItemStructured}), ?assert(hb_message:verify(ItemStructured, all, TestOpts)), BundledItemStructured = hb_message:convert(BundledItem, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, + <<"ans104@1.0">>, TestOpts), ?event(debug_test, {bundled_item_structured, BundledItemStructured}), ?assert(hb_message:verify(BundledItemStructured, all, TestOpts)), @@ -371,15 +303,15 @@ build_signed_tx_on_arbundles_js_test() -> ?assert(ar_tx:verify(SignedTX)), % Convert the signed TX to a structured message StructuredTX = hb_message:convert(SignedTX, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, + <<"tx@1.0">>, TestOpts), % ?event(debug_test, {structured_tx, StructuredTX}), ?assert(hb_message:verify(StructuredTX, all, TestOpts)), % Convert back to an L1 TX SignedTXRoundtrip = hb_message:convert(StructuredTX, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + <<"tx@1.0">>, + <<"structured@1.0">>, TestOpts), ?event(debug_test, {signed_tx_roundtrip, SignedTXRoundtrip}), ?assert(ar_tx:verify(SignedTXRoundtrip)), @@ -388,3 +320,197 @@ build_signed_tx_on_arbundles_js_test() -> after hb_mock_server:stop(ServerHandle) end. + +%% Test that a nested dataitem is handled correctly by the bundler flow. +%% This test focuses in on the conversion that happens between building +%% the signed bundle TX and building the bundle proofs. +bundle_convert_real_data_test() -> + Item = inlined_broken_item(), + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => hb_test_utils:test_store() + }, + try + {ok, SignedTX} = build_signed_tx([Item], TestOpts), + ?assert(ar_tx:verify(SignedTX)), + Committed = hb_message:convert( + SignedTX, <<"structured@1.0">>, <<"tx@1.0">>, TestOpts), + %% This convert is exactly what build_proofs runs. + TX = hb_message:convert( + Committed, <<"tx@1.0">>, <<"structured@1.0">>, TestOpts), + ?assert(ar_tx:verify(TX)) + after + hb_mock_server:stop(ServerHandle) + end. + +bundle_convert_minimal_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => hb_test_utils:test_store() + }, + try + Item = hb_message:commit( + #{ <<"key">> => <<"value">>, + <<"body">> => #{ <<"a">> => <<"b">> } }, + TestOpts, #{<<"device">> => <<"ans104@1.0">>}), + {ok, SignedTX} = build_signed_tx([Item], TestOpts), + ?assert(ar_tx:verify(SignedTX)), + Committed = hb_message:convert( + SignedTX, <<"structured@1.0">>, <<"tx@1.0">>, TestOpts), + TX = hb_message:convert( + Committed, <<"tx@1.0">>, <<"structured@1.0">>, TestOpts), + ?assert(ar_tx:verify(TX)) + after + hb_mock_server:stop(ServerHandle) + end. + +%% @doc Drive a nested tree of items signed in mixed bundle states through +%% the bundler flow: each child is signed with bundle=true OR bundle=false, +%% then we build the bundle TX, sign it, convert through structured@1.0 and +%% back to tx@1.0, and assert nothing was inflated and every commitment +%% still verifies. This exercises the full `hint-device' plumbing across a +%% mixed tree, mirroring the production scenario that motivated the fix. +bundle_convert_mixed_tree_verify_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => hb_test_utils:test_store() + }, + try + %% Build three items. The first carries a child signed bundle=false, + %% the second a child signed bundle=true, the third has no nested + %% child at all. The L1 bundle TX therefore contains items that + %% would individually each round-trip with a different bundle state. + InnerFalse = hb_message:commit( + #{ <<"leaf-tag">> => <<"leaf-false">>, + <<"leaf-list">> => [1, 2, 3] }, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false }), + ?assert(hb_message:verify(InnerFalse, all, TestOpts)), + InnerTrue = hb_message:commit( + #{ <<"leaf-tag">> => <<"leaf-true">>, + <<"leaf-list">> => [4, 5, 6] }, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }), + ?assert(hb_message:verify(InnerTrue, all, TestOpts)), + ItemA = hb_message:commit( + #{ <<"item-tag">> => <<"a">>, <<"inner">> => InnerFalse }, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }), + ?assert(hb_message:verify(ItemA, all, TestOpts)), + ItemB = hb_message:commit( + #{ <<"item-tag">> => <<"b">>, <<"inner">> => InnerTrue }, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false }), + ?assert(hb_message:verify(ItemB, all, TestOpts)), + ItemC = hb_message:commit( + #{ <<"item-tag">> => <<"c">> }, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false }), + ?assert(hb_message:verify(ItemC, all, TestOpts)), + {ok, SignedTX} = build_signed_tx([ItemA, ItemB, ItemC], TestOpts), + ?assert(ar_tx:verify(SignedTX)), + Committed = hb_message:convert( + SignedTX, <<"structured@1.0">>, <<"tx@1.0">>, TestOpts), + ?event(debug_test, {committed, {explicit, Committed}}), + ?assert(hb_message:verify(Committed, all, TestOpts)), + %% Convert back to TX (same path build_proofs uses) and check that + %% the data did not inflate. + TX = hb_message:convert( + Committed, <<"tx@1.0">>, <<"structured@1.0">>, TestOpts), + ?assert(ar_tx:verify(TX)) + after + hb_mock_server:stop(ServerHandle) + end. + +%% Hardcoded item, structurally identical to one observed in a broken +%% production bundle (TXID -BTiilFCWd2kB3oOdCpPDJLGXhjeNxIeMH3kerPXKCM). +%% AO "Assignment" message with `body`, two commitments (HMAC + RSA-PSS), +%% per-event commitments inside the body. All public key / signature +%% bytes are real (from production) since the structured form encodes +%% them. +inlined_broken_item() -> + #{<<"base-hashpath">> => + <<"w_l6KLmO8OeEM6vmdwX1HwdCDmHiOlhUyAeNdjwpspU/p4CQHPCo629uDl8seMpWN5Z4EZpRK6bUNPbGAoOIkrs">>, + <<"block-hash">> => <<"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA">>, + <<"block-height">> => 0, + <<"block-timestamp">> => 0, + <<"body">> => + #{<<"commitments">> => + #{<<"HkUAI3fWd3uHltdfyHzLU5IreUtmIIqv45ZxsC12psI">> => + #{<<"commitment-device">> => <<"httpsig@1.0">>, + <<"committed">> => + [<<"event">>, <<"reference">>, <<"status-class">>], + <<"committer">> => + <<"mfj2T6f_3stKQk7fctrbpZKUfu4V6MQKCH-YHLtFnOY">>, + <<"keyid">> => + <<"publickey:kvgGBmHtEOxZTuJuJuHVqBe51aLpSDvZrzj5RjNEbw8NrYm3GB9+BEKdYD+fHZ0H775PJf8mosGapkP6pB8h8ZEEc6AuOo+lTJ9SKEnYit1Q6YG5Dg306EDfm0dpMU3zKe9pE4CIHf3ffCDqa1Xh4c1zdcFqKyofeT8PWIGQZCScA8rYG+aG2Z/y6QduyxBgzFfITdzeXbnJONmZEwPEA29LeDWmCxA7CSfE6W8+2aDW75qQjETRVXzxou0I0tsc3uXzd9E0/yU6NbDi93sIBiO8z2pNbGrMGIfpH+4dirH/YZNBW8PBgOLjnpe5yoPrT+cI8OoEX27u/al/rkPG4u0wBlxnollSr/lXc5HIn6AWvpSF7nuXcmdtG8Y8RK95h4YZ9d6CG3tSOTvSk7wK8IH97hScB16EpiuT6Xi/TPYh3PwVC/VDxLMox19v1eP1riHC/nkIroerCmaIwGfxI2XNUgQzaTcygjT0DFbbLZFakCZpJ+0+u2/S1I4EbdpdcChWqrA8psUlyR3sbhhPDpEP1ldNO+08OyW/PMfMwXEkVR+WHM2t5m9cyZwjpes6epCQQWjMIkwqeIRZWwo607iJKsXgd5n+73FWytVWNS1mOH1nDDkfDXOq4R60B6C+2k0As14b2Dv4eXZstbr8KbtVIHit/IytBohieLpMcF0=">>, + <<"signature">> => + <<"fZgshLexhVcpiQ1sBhwa_eoDn97vvc6IoJtwntc8VJTSDAokQ0RyThcjqhcgtF04kl4T986lZrVptAXkiKwog-gH1vnJX1T2yGAM-ZlTNTTmdLE7OIQhvs26-0_L3poPUSEjHsZ1vU2RpUUvKLIEQdCwlgTXGx54ZGB6feYXMn9e01tZPEdTVD0AcALa5G55aqyH3Lde5KXx4vOgdvWaCr772dXZ6C8249UG02SIHy3xvp1UdkLtzIbvSY9n5UzC1Bt-b5JftIijmVuIv3oI0_y9rRxGYLm2m7VusHwYjRdFAjN5X_NvYpWx25b62CNNLwprJfDqhllZsDz6PjnhRh9ZocOP3OLrarW0owFt0dfDRt3VBYaksUYTem-9YWtzS3Qa7kZSB754xtOW62wvu3kVH2sNB5C9SoXmheoPUjNLa4qXQv4-NJPF4wVdj8QxM0mYO0KQZfCUZtXhYYaqwRmS2aMyUrca1xjPOkD0nr7B1IS805O08fTkN6YcMluUH93myL4VbPPa2v1V2k-B-OlP4AzOn9F1uzk5ek--K_-2QdC63vgm4EKv8XqBoipUJ0Fe0jKUsE9iLZJddoMrYrsQCp8WMWX7iGaP6zJU2tbMpkAl-rr_Hc8xUkJ3eBd6pQcw-1MQ8EK7trPnjQD0EQZAG2HYj87HG-qCX3l9o8w">>, + <<"type">> => <<"rsa-pss-sha512">>}, + <<"asDiK4CqvjJf2d9FFf3r3-xCrs1jA8ee9tWUp43BuWk">> => + #{<<"commitment-device">> => <<"httpsig@1.0">>, + <<"committed">> => + [<<"event">>, <<"reference">>, <<"status-class">>], + <<"keyid">> => <<"constant:ao">>, + <<"signature">> => + <<"asDiK4CqvjJf2d9FFf3r3-xCrs1jA8ee9tWUp43BuWk">>, + <<"type">> => <<"hmac-sha256">>}}, + <<"event">> => <<"is_admissible">>, + <<"reference">> => + <<"HnbIWJdkG4CCwHCiycMKMmv2posdcTJ5xFcZ9lpTQQs">>, + <<"status-class">> => <<"success">>}, + <<"commitments">> => + #{<<"KT4ZXa_nhnWTfNJdVwOPDwHNN3eqYs_o3JoYv_odNvE">> => + #{<<"commitment-device">> => <<"httpsig@1.0">>, + <<"committed">> => + [<<"ao-types">>, <<"base-hashpath">>, <<"block-hash">>, + <<"block-height">>, <<"block-timestamp">>, <<"body">>, + <<"data-protocol">>, <<"epoch">>, <<"path">>, + <<"process">>, <<"slot">>, <<"timestamp">>, <<"type">>, + <<"variant">>], + <<"keyid">> => <<"constant:ao">>, + <<"signature">> => + <<"KT4ZXa_nhnWTfNJdVwOPDwHNN3eqYs_o3JoYv_odNvE">>, + <<"type">> => <<"hmac-sha256">>}, + <<"j1KSZD2tQOXpYvbPaqmLyRN6OOXxGa20bkgfeCj4a30">> => + #{<<"bundle">> => <<"false">>, + <<"commitment-device">> => <<"ans104@1.0">>, + <<"committed">> => + [<<"ao-types">>, <<"base-hashpath">>, <<"block-hash">>, + <<"block-height">>, <<"block-timestamp">>, <<"body">>, + <<"data-protocol">>, <<"epoch">>, <<"path">>, + <<"process">>, <<"slot">>, <<"timestamp">>, <<"type">>, + <<"variant">>], + <<"committer">> => + <<"n_XZJhUnmldNFo4dhajoPZWhBXuJk-OcQr5JQ49c4Zo">>, + <<"keyid">> => + <<"publickey:9BXuilimqVo7fpnoToPHZwqL7w_C0Qn4N3egeJRy05-nSpUv1vyp9xHbVLKVMPnJsie5Awt_xxob_jDvXSmE1fDsUpNnFurxG88UWN4zSNi87EfOorDQjHPRUqKPIYvg6xqPCpXPpOccJbFuack3ltQKtF5XLoaKWbsPdUtMquRXrbJgnGeOvXhQhbKa4xJKwGmjVC_LpY5FQ8j-cOlBOVVe_B7KF4eWG3sJf-z59MJQOaAozyU2iZpsuhslkTNVj8sM9CqkSfyD8EjEZdfF088IM_dJgk6ehIDHbx3FcGVnxpUHkXEnJFAlXRzdqmNb84QXsTNOHqwQPZ3q5wPRWS6iUaNxfeS_SsR6otIJgrYq04LYJLcpHuKGp53-b8tTeIvDFcmS2_kPijPqPINbf9c5uH0mxMNomB-8rVDIkIZ6Ojc_M0JnaQSk2rYPq8qRy2PuvAFyo1zeGM-2Bo4GNl9dMnfIr_Q6MlxRUwAwLHdOt0BJkxEBfOIw3MkB2d-SiVWtxG1Uqib7Iu_yn3j9DwzUOHjRQTse07giNDXRMsr1ml_sCK3bIetUFVnjjnoTNDEItDSck7lTFgvCdyXKkvXtSiNHkW8TCbdTDY0hBJzLVheKDb_cCyfmcTKo5ql2sWsZYCC7XybKdRMxU2HNNIUSpcDvhnTwv5-oq42Lmqc">>, + <<"signature">> => + <<"sE9TuQTsMCHhaSOmHF-Wqu8QBbSNMSeSztiE1b0tJfmSNOe1nKPmMcCZN1rHD8L9xQWJw4hSVUbChwt4QReTz2IoXFz1NT80F1qCY2x3uFMFxgUHb2abTQW_-VNjFGWFe-sguwYLAIZGYoJ9a2g1EJCRfksk9iOWXRt7j_yIBixKATq-QsEWdcwfBsEUYWq-IRI1RdPAr9ToZeQ13TtWWYxcRbKHwxJ1M58p2CuLCi1OXVmENLjacAawuhBjGV4oTQ1-QBap-JOjB6kRTXtWjNGnMTPF01edFJIxgRncnODrTO_ehz6qkFH6iMhI9oV4w5VcRCKnNM7fxTXKj6DeiuAb1KrirpzohzsTLautMqRhst8gSViBlftd4XoVCDVscawuz8yPDyJoDxhIIup7mO51QSmNVTM6JpSEsG-CbXa64aECBOq7_x-ld9xHyNvCCSHetSJ3EBiJDWHE8XCurePGJ6GLeggugQ85LxgsRaLDm9UIlbMhopkK4X-SyXz5_pGwUSegLa1QHWWxnIaS5zTm0f4yi_YiBmgmS27v28T-nTzOHuBGTl8yUWVG_CKAELjFVREm5I7h4UuDQuFoXlkkFW22-Gyx5tZh1eSxRpl1NOwhyGc9O-6TIR46t1BhlItitOoi6JEf26JjTmwJWF7kR8xyahCYWtHFEkzpob4">>, + <<"type">> => <<"rsa-pss-sha256">>}}, + <<"data-protocol">> => <<"ao">>, + <<"epoch">> => <<"0">>, + <<"path">> => <<"compute">>, + <<"process">> => <<"1V65_gzlifHH_surfFzL6HGfRlLJuEX_y0VbPHwIKec">>, + <<"slot">> => 180901, + <<"timestamp">> => 1778975170441, + <<"type">> => <<"Assignment">>, + <<"variant">> => <<"ao.N.1">>}. diff --git a/src/preloaded/codec/dev_ans104.erl b/src/preloaded/codec/dev_ans104.erl index 2b4b5facc..58584d345 100644 --- a/src/preloaded/codec/dev_ans104.erl +++ b/src/preloaded/codec/dev_ans104.erl @@ -2,7 +2,7 @@ %%% records to and from TABMs. -module(dev_ans104). -device_libraries([lib_arweave_common]). --export([to/3, from/3, commit/3, verify/3, content_type/1]). +-export([to/3, to_hint/3, from/3, commit/3, verify/3, content_type/1]). -export([serialize/3, deserialize/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -128,6 +128,14 @@ do_from(RawTX, Req, Opts) -> ?event({from, {parsed_message, WithCommitments}}), {ok, WithCommitments}. +%% @doc Inspect a message's signed ans104 commitment and, if it carries an +%% explicit `bundle' field, mirror that value onto the request `Req'. +to_hint(Msg, Req, Opts) -> + case lib_arweave_common:bundle_hint(<<"ans104@1.0">>, Msg, Req, Opts) of + not_found -> {ok, Req}; + Hint -> Hint + end. + %% @doc Internal helper to translate a message to its #tx record representation, %% which can then be used by ar_bundles to serialize the message. We call the %% message's device in order to get the keys that we will be checkpointing. We @@ -144,55 +152,19 @@ to(Binary, _Req, _Opts) when is_binary(Binary) -> } }; to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; -to(RawTABM, Req, Opts) when is_map(RawTABM) -> +to(TABM, Req, Opts) when is_map(TABM) -> % Ensure that the TABM is fully loaded if the `bundle` key is set to true. - ?event(ans104_to, {to, {inbound, RawTABM}, {req, Req}}, + ?event(ans104_to, {to, {inbound, TABM}, {req, Req}}, #{debug_print_verify => false}), - MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, - RawTABM, + TX = lib_arweave_common:to( + <<"ans104@1.0">>, TABM, Req, + fun lib_arweave_common:fields_to_tx/4, + fun lib_arweave_common:excluded_tags/3, Opts ), - IsBundle = lib_arweave_common:is_bundle(MaybeCommitment, Req, Opts), - MaybeBundle = lib_arweave_common:maybe_load(RawTABM, IsBundle, Opts), - ?event(ans104_to, {to, {maybe_bundle, MaybeBundle}}, - #{debug_print_verify => false}), - - % Calculate and normalize the `data', if applicable. - Data = lib_arweave_common:data( - MaybeBundle, Req, fun dev_ans104:to/3, Opts), - ?event(ans104_to, {to, {calculated_data, Data}}, - #{debug_print_verify => false}), - TX0 = lib_arweave_common:siginfo( - MaybeBundle, MaybeCommitment, - fun lib_arweave_common:fields_to_tx/4, Opts - ), - ?event(ans104_to, {to, {found_siginfo, TX0}}, - #{debug_print_verify => false}), - TX1 = TX0#tx { data = Data }, - % Calculate the tags for the TX. - Tags = lib_arweave_common:tags( - TX1, MaybeCommitment, MaybeBundle, - lib_arweave_common:excluded_tags(TX1, MaybeBundle, Opts), Opts), - ?event(ans104_to, {to, {calculated_tags, Tags}}, + ?event(ans104_to, {to, {result, TX}}, #{debug_print_verify => false}), - TX2 = TX1#tx { tags = Tags }, - Res = - try ar_tx:normalize(TX2) - catch - Type:Error:Stacktrace -> - ?event({ - {reset_ids_error, Error}, - {tx_without_data, {explicit, TX2}}}), - ?event({prepared_tx_before_ids, - {tags, {explicit, TX2#tx.tags}}, - {data, TX2#tx.data} - }), - erlang:raise(Type, Error, Stacktrace) - end, - ?event(ans104_to, {to, {result, Res}}, - #{debug_print_verify => false}), - {ok, Res}; + {ok, TX}; to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). @@ -840,17 +812,18 @@ test_bundle_commitment(Commit, Encode, Decode) -> hb_util:atom(hb_ao:get(<<"bundle">>, CommittedCommitment, false, Opts)), Label), - Encoded = hb_message:convert(Committed, + Encoded = hb_message:convert(Committed, #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Encode) }, - <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Opts), ?event(debug_test, {encoded, Label, {explicit, Encoded}}), ?assert(ar_bundles:verify_item(Encoded), Label), %% IF the input message is unbundled, #tx.data should be empty. ?assertEqual(ToBool(Commit), Encoded#tx.data /= <<>>, Label), - Decoded = hb_message:convert(Encoded, + Decoded = hb_message:convert(Encoded, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => ToBool(Decode) }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Encode) }, + <<"ans104@1.0">>, Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), @@ -883,16 +856,17 @@ test_bundle_uncommitted(Encode, Decode) -> ToBool = fun(unbundled) -> false; (bundled) -> true end, Label = lists:flatten(io_lib:format("~p -> ~p", [Encode, Decode])), - Encoded = hb_message:convert(Structured, + Encoded = hb_message:convert(Structured, #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Encode) }, - <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Opts), ?event(debug_test, {encoded, Label, {explicit, Encoded}}), %% IF the input message is unbundled, #tx.data should be empty. ?assertEqual(ToBool(Encode), Encoded#tx.data /= <<>>, Label), - Decoded = hb_message:convert(Encoded, + Decoded = hb_message:convert(Encoded, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => ToBool(Decode) }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => ToBool(Encode) }, + <<"ans104@1.0">>, Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), case Encode of @@ -902,3 +876,4 @@ test_bundle_uncommitted(Encode, Decode) -> ?assertEqual([1, 2, 3], maps:get(<<"list">>, Decoded, Opts), Label) end, ok. + diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index f93339031..8c20797bb 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -80,20 +80,30 @@ from(List, Req, Opts) when is_list(List) -> {ok, hb_util:numbered_keys_to_list(DecodedAsMap, Opts)} end; from(Msg, Req, Opts) when is_map(Msg) -> - % Normalize the message, offloading links to the cache. - NormLinks = hb_link:normalize(Msg, linkify_mode(Req, Opts), Opts), + HintedReq = apply_bundle_hint(Msg, Req, Opts), + NormLinks = hb_link:normalize(Msg, linkify_mode(HintedReq, Opts), Opts), NormKeysMap = hb_ao:normalize_keys(NormLinks, Opts), - EncodeTypes = find_encode_types(Req, Opts), + EncodeTypes = find_encode_types(HintedReq, Opts), {Types, Values} = lists:foldl( fun (Key, {Types, Values}) -> case hb_maps:find(Key, NormKeysMap, Opts) of {ok, Value} when is_binary(Value) -> {Types, [{Key, Value} | Values]}; - {ok, Nested} when is_map(Nested) or is_list(Nested) -> + {ok, Nested} when is_map(Nested) orelse is_list(Nested) -> ?event({from_recursing, {nested, Nested}}), - {Types, [{Key, hb_util:ok(from(Nested, Req, Opts))} | Values]}; + % We pass the HintedReq to the recursive call rather than + % Req so that this message's bundle status serves as the + % default for any children that don't explicitly set the + % `bundle' flag on the hinted commitment. + {Types, + [{ + Key, + hb_util:ok(from(Nested, HintedReq, Opts)) + } | Values]}; {ok, Value} when - is_atom(Value) or is_integer(Value) or is_float(Value) -> + is_atom(Value) + orelse is_integer(Value) + orelse is_float(Value) -> BinKey = hb_ao:normalize_key(Key), ?event({encode_value, Value}), case maybe_encode_value(Value, EncodeTypes) of @@ -136,7 +146,7 @@ from(Msg, Req, Opts) when is_map(Msg) -> % Encode the AoTypes as a structured dictionary % And include as a field on the produced TABM WithTypes = - hb_maps:from_list(case Types of + hb_maps:from_list(case Types of [] -> Values; T -> AoTypes = iolist_to_binary(hb_structured_fields:dictionary( @@ -174,16 +184,33 @@ type(Atom) when is_atom(Atom) -> <<"atom">>; type(List) when is_list(List) -> <<"list">>; type(Other) -> Other. +%% @doc If a `hint-device` key is present it indicates the desired +%% terminal format (after being converted via an intermediate `tabm` +%% format). In that case dev_structured defers to the target codec +%% to determine whether child messages should be loaded or unloaded. +apply_bundle_hint(Msg, Req, Opts) -> + case hb_maps:get(<<"hint-device">>, Req, undefined, Opts) of + undefined -> Req; + DeviceBin -> + % May add a `bundle` key to the request + try hb_util:ok( + hb_ao:raw(DeviceBin, <<"to-hint">>, Msg, Req, Opts) + ) + catch _:_ -> + Req + end + end. + %% @doc Discern the linkify mode from the request and the options. linkify_mode(Req, Opts) -> case hb_maps:get(<<"bundle">>, Req, not_found, Opts) of - not_found -> hb_opts:get(linkify_mode, offload, Opts); true -> % The request is asking for a bundle, so we should _not_ linkify. false; - false -> - % The request is asking for a flat message, so we should linkify. - true + _ -> + % The request is either asking for a flat message or has not + % specified. In both cases we should linkify. + hb_opts:get(linkify_mode, offload, Opts) end. %% @doc Convert a TABM into a native HyperBEAM message. diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index b11a0c9a3..7e9ab9d0f 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -2,7 +2,7 @@ %%% records to and from TABMs. -module(dev_tx). -device_libraries([lib_arweave_common]). --export([from/3, to/3, commit/3, verify/3]). +-export([from/3, to/3, to_hint/3, commit/3, verify/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -102,6 +102,13 @@ do_from(RawTX, Req, Opts) -> ?event({from, {parsed_message, hb_util:human_id(TX#tx.id)}}), {ok, WithCommitments}. +%% @doc Inspect a message's signed tx@1.0 commitment and, if the commitment +%% carries an explicit `bundle' field, mirror that value onto the request `Req'. +to_hint(Msg, Req, Opts) -> + case lib_arweave_common:bundle_hint(<<"tx@1.0">>, Msg, Req, Opts) of + not_found -> hb_ao:raw(<<"ans104@1.0">>, <<"to-hint">>, Msg, Req, Opts); + Hint -> Hint + end. %% @doc Internal helper to translate a message to its #tx record representation, %% which can then be used by ar_tx to serialize the message. We call the %% message's device in order to get the keys that we will be checkpointing. We @@ -119,39 +126,17 @@ to(Binary, _Req, _Opts) when is_binary(Binary) -> }) }; to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; -to(RawTABM, Req, Opts) when is_map(RawTABM) -> - % Ensure that the TABM is fully loaded if the `bundle` key is set to true. - ?event({to, {inbound, RawTABM}, {req, Req}}), - MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, - RawTABM, +to(TABM, Req, Opts) when is_map(TABM) -> + ?event({to, {inbound, TABM}, {req, Req}}), + TX = lib_arweave_common:to( + <<"tx@1.0">>, TABM, Req, + fun dev_tx_to:fields_to_tx/4, + fun dev_tx_to:excluded_tags/3, Opts ), - IsBundle = lib_arweave_common:is_bundle(MaybeCommitment, Req, Opts), - MaybeBundle = lib_arweave_common:maybe_load(RawTABM, IsBundle, Opts), - ?event({to, {raw_tabm, RawTABM}, {is_bundle, IsBundle}, {maybe_bundle, MaybeBundle}, {req, Req}, {opts, Opts}}), - % Calculate and normalize the `data', if applicable. - Data = - lib_arweave_common:data( - MaybeBundle, Req, fun lib_arweave_common:to/3, Opts), - ?event({calculated_data, Data}), - TX0 = lib_arweave_common:siginfo( - MaybeBundle, MaybeCommitment, - fun dev_tx_to:fields_to_tx/4, Opts), - ?event({found_siginfo, TX0}), - TX1 = TX0#tx { data = Data }, - % Calculate the tags for the TX. - Tags = lib_arweave_common:tags( - TX1, MaybeCommitment, MaybeBundle, - dev_tx_to:excluded_tags(TX1, MaybeBundle, Opts), - Opts), - ?event({calculated_tags, Tags}), - TX2 = TX1#tx { tags = Tags }, - ?event({tx_before_id_gen, TX2}), - FinalTX = ar_tx:normalize(TX2), - enforce_valid_tx(FinalTX), - ?event({to_result, FinalTX}), - {ok, FinalTX}; + enforce_valid_tx(TX), + ?event({to_result, TX}), + {ok, TX}; to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). @@ -1452,17 +1437,18 @@ test_bundle_commitment(Commit, Encode, Decode) -> hb_util:atom(hb_ao:get(<<"bundle">>, CommittedCommitment, false, Opts)), Label), - Encoded = hb_message:convert(Committed, + Encoded = hb_message:convert(Committed, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Encode) }, - <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Opts), ?event(debug_test, {encoded, Label, {explicit, Encoded}}), ?assert(ar_tx:verify(Encoded), Label), %% IF the input message is unbundled, #tx.data should be empty. ?assertEqual(ToBool(Commit), Encoded#tx.data /= <<>>, Label), - Decoded = hb_message:convert(Encoded, + Decoded = hb_message:convert(Encoded, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => ToBool(Decode) }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Encode) }, + <<"tx@1.0">>, Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), ?assert(hb_message:verify(Decoded, all, Opts), Label), @@ -1494,16 +1480,17 @@ test_bundle_uncommitted(Encode, Decode) -> ToBool = fun(unbundled) -> false; (bundled) -> true end, Label = lists:flatten(io_lib:format("~p -> ~p", [Encode, Decode])), - Encoded = hb_message:convert(Structured, + Encoded = hb_message:convert(Structured, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Encode) }, - <<"structured@1.0">>, Opts), + <<"structured@1.0">>, + Opts), ?event(debug_test, {encoded, Label, {explicit, Encoded}}), - %% IF the input message is unbundled, #tx.data should be empty. + %% If the input message is unbundled, #tx.data should be empty. ?assertEqual(ToBool(Encode), Encoded#tx.data /= <<>>, Label), - Decoded = hb_message:convert(Encoded, + Decoded = hb_message:convert(Encoded, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => ToBool(Decode) }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => ToBool(Encode) }, + <<"tx@1.0">>, Opts), ?event(debug_test, {decoded, Label, {explicit, Decoded}}), case Encode of diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index aa0951aea..755b9492c 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -1,10 +1,10 @@ %%% @doc Shared Arweave codec helpers. -module(lib_arweave_common). --export([from/3, to/3]). +-export([from/3]). -export([fields/3, tags/2, data/5, committed/6, base/5]). -export([with_commitments/8]). --export([is_bundle/3, maybe_load/3, data/4, tags/5, excluded_tags/3]). --export([siginfo/4, fields_to_tx/4]). +-export([bundle_hint/4, data/3, tags/5, excluded_tags/3]). +-export([to/3, to/6, siginfo/4, fields_to_tx/4]). -export([bundle_header/2, bundle_header/3]). -include("include/hb.hrl"). @@ -34,35 +34,56 @@ from_item(RawTX, Req, Opts) -> ) }. -%% @doc Convert a message into its ANS-104 item form. +%% @doc Recursively encode a nested message as an `ans104@1.0' #tx record. to(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, #tx{ tags = [{<<"ao-type">>, <<"binary">>}], data = Binary }}; to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; -to(RawTABM, Req, Opts) when is_map(RawTABM) -> +to(TABM, Req, Opts) when is_map(TABM) -> + {ok, + to( + <<"ans104@1.0">>, TABM, Req, + fun ?MODULE:fields_to_tx/4, + fun ?MODULE:excluded_tags/3, + Opts + )}; +to(Other, _Req, _Opts) -> + throw({invalid_tx, Other}). + +to(Device, TABM, Req, FieldsFun, ExcludedTagsFun, Opts) -> MaybeCommitment = hb_message:commitment( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, - RawTABM, + #{ <<"commitment-device">> => Device }, + TABM, Opts ), - IsBundle = is_bundle(MaybeCommitment, Req, Opts), - MaybeBundle = maybe_load(RawTABM, IsBundle, Opts), - Data = data(MaybeBundle, Req, fun lib_arweave_common:to/3, Opts), - TX0 = - siginfo( - MaybeBundle, MaybeCommitment, - fun lib_arweave_common:fields_to_tx/4, Opts - ), + Data = data(TABM, Req, Opts), + ?event({calculated_data, Data}), + TX0 = siginfo(TABM, MaybeCommitment, FieldsFun, Opts), + ?event({found_siginfo, TX0}), TX1 = TX0#tx{ data = Data }, - Tags = - tags( - TX1, MaybeCommitment, MaybeBundle, - excluded_tags(TX1, MaybeBundle, Opts), Opts - ), - {ok, ar_tx:normalize(TX1#tx{ tags = Tags })}; -to(Other, _Req, _Opts) -> - throw({invalid_tx, Other}). + Tags = tags( + TX1, + MaybeCommitment, + TABM, + ExcludedTagsFun(TX1, TABM, Opts), + Opts + ), + ?event({calculated_tags, Tags}), + TX = TX1#tx{ tags = Tags }, + ?event({tx_before_id_gen, TX}), + try ar_tx:normalize(TX) + catch + Type:Error:Stacktrace -> + ?event({ + {reset_ids_error, Error}, + {tx_without_data, {explicit, TX}}}), + ?event({prepared_tx_before_ids, + {tags, {explicit, TX#tx.tags}}, + {data, TX#tx.data} + }), + erlang:raise(Type, Error, Stacktrace) + end. %% @doc Return a TABM message containing the fields of the given decoded %% ANS-104 data item that should be included in the base message. @@ -429,65 +450,24 @@ deduplicating_from_list(Tags, Opts) -> %%% Encoding helpers. -is_bundle({ok, _, Commitment}, _Req, Opts) -> - hb_util:atom(hb_ao:get(<<"bundle">>, Commitment, false, Opts)); -is_bundle(_, Req, Opts) -> - case hb_maps:is_key(<<"bundle">>, Req, Opts) of - true -> hb_util:atom(hb_ao:get(<<"bundle">>, Req, false, Opts)); - false -> hb_util:atom(hb_ao:get(<<"bundle">>, Opts, false, Opts)) - end. - -%% @doc Determine if the message should be loaded from the cache and re-converted -%% to the TABM format. We do this if the `bundle' key is set to true. -maybe_load(RawTABM, true, Opts) -> - % Convert back to the fully loaded structured@1.0 message, then - % convert to TABM with bundling enabled. - Structured = hb_message:convert(RawTABM, <<"structured@1.0">>, Opts), - Loaded = hb_cache:ensure_all_loaded(Structured, Opts), - % Convert to TABM with bundling enabled. - LoadedTABM = - hb_message:convert( - Loaded, - tabm, +%% @doc Apply the `bundle' hint from a signed commitment for `Device'. +%% Returns `not_found' when no signed commitment for `Device' exists. +bundle_hint(Device, Msg, Req, Opts) -> + case hb_message:commitment( #{ - <<"device">> => <<"structured@1.0">>, - <<"bundle">> => true + <<"commitment-device">> => Device, + <<"committer">> => '_' }, - Opts - ), - % Ensure the commitments from the original message are the only - % ones in the fully loaded message, recursively for nested maps. - replace_commitments_recursive(LoadedTABM, RawTABM); -maybe_load(RawTABM, false, _Opts) -> - RawTABM. - -%% @doc Recursively replace commitments from RawTABM into LoadedTABM. -replace_commitments_recursive(LoadedTABM, RawTABM) - when is_map(LoadedTABM), is_map(RawTABM) -> - LoadedTABM2 = - case maps:find(<<"commitments">>, RawTABM) of - {ok, RawCommitments} -> - LoadedTABM#{ <<"commitments">> => RawCommitments }; - error -> - maps:remove(<<"commitments">>, LoadedTABM) - end, - maps:map( - fun(<<"commitments">>, Value) -> - Value; - (Key, Value) when is_map(Value) -> - case maps:get(Key, RawTABM, undefined) of - RawValue when is_map(RawValue) -> - replace_commitments_recursive(Value, RawValue); - _ -> - Value + Msg, + Opts) of + {ok, _, Commitment} -> + case hb_util:atom( + hb_maps:get(<<"bundle">>, Commitment, not_found, Opts)) of + not_found -> {ok, Req}; + Value -> {ok, Req#{ <<"bundle">> => Value }} end; - (_Key, Value) -> - Value - end, - LoadedTABM2 - ); -replace_commitments_recursive(LoadedTABM, _RawTABM) -> - LoadedTABM. + _ -> not_found + end. %% @doc Calculate the fields for a message, returning an initial TX record. siginfo(_Message, {ok, _, Commitment}, FieldsFun, Opts) -> @@ -568,13 +548,13 @@ fields_to_tx(TX, Prefix, Map, Opts) -> }. %% @doc Calculate the data field for a message. -data(TABM, Req, ToFun, Opts) -> +data(TABM, Req, Opts) -> DataKey = inline_key(TABM), UnencodedNestedMsgs = data_messages(TABM, Opts), NestedMsgs = hb_maps:map( fun(_, Msg) -> - hb_util:ok(ToFun(Msg, Req, Opts)) + hb_util:ok(to(Msg, Req, Opts)) end, UnencodedNestedMsgs, Opts @@ -587,7 +567,7 @@ data(TABM, Req, ToFun, Opts) -> {?DEFAULT_DATA, _} -> NestedMsgs; {DataVal, _} -> - NestedMsgs#{ DataKey => hb_util:ok(ToFun(DataVal, Req, Opts)) } + NestedMsgs#{ DataKey => hb_util:ok(to(DataVal, Req, Opts)) } end. %% @doc Calculate data messages for large tag values or nested messages. diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 837e974a9..f158e9bb9 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -135,21 +135,36 @@ id(RawBase, Req, NodeOpts) -> end. calculate_id(RawBase, Req, NodeOpts) -> - % Find the ID device for the message. - Base = hb_message:convert(RawBase, tabm, NodeOpts), - ?event(debug_id, {calculate_ids, {base, Base}}), - IDMod = - case id_device(Base, NodeOpts) of - {ok, IDDev} -> IDDev; + % Resolve the ID device up-front so we can plumb it as `hint-device' into + % the structured->tabm conversion below. This keeps the children's load + % state consistent with what `commit/3' and `verify/3' would produce. + IDDev = + case id_device(RawBase, NodeOpts) of + {ok, Device} -> Device; {error, Error} -> throw({id, Error}) end, - ?event(debug_id, {generating_id, {idmod, IDMod}, {base, Base}}), - % If the ID device resolves to this device, use the default commitment - % device instead to avoid recursing through `message@1.0/commit'. + % Encode to a TABM. The `bundle' flag (when set on the request) is the + % caller's intent for the top-level message and applies only to the root; + % `hint-device' lets the structured codec reproduce each nested + % commitment's own bundle state per-node, so the id is computed over the + % same shape `commit/3' and `verify/3' would produce. + SourceSpec = + hb_message:add_bundle_hint( + #{ <<"device">> => <<"structured@1.0">> }, + Req#{ <<"device">> => IDDev }, + NodeOpts + ), + Base = hb_message:convert(RawBase, tabm, SourceSpec, NodeOpts), + ?event(debug_id, {calculate_ids, {base, Base}}), + ?event(debug_id, {generating_id, {id_device, IDDev}, {base, Base}}), + % Get the commitment device name from the message, or use the default if + % it is not set. We can tell if the device is not set (or is the default) + % by checking whether the resolved device module is this module itself. + % `hb_ao:raw/5' expects a device name, not a resolved module. CommitDev = - case hb_device:message_to_device(#{ <<"device">> => IDMod }, NodeOpts) of + case hb_device:message_to_device(#{ <<"device">> => IDDev }, NodeOpts) of ?MODULE -> ?DEFAULT_ID_DEVICE; - _ -> IDMod + _ -> IDDev end, ?event(debug_id, {called_id_device, CommitDev}, NodeOpts), {ok, #{ <<"commitments">> := Comms} } = @@ -246,10 +261,19 @@ commit(Self, Req, Opts) -> _ -> Opts#{ <<"linkify-mode">> => offload } end, - % Encode to a TABM + % Encode to a TABM. The `bundle' flag (when set on the request) is the + % caller's intent for the top-level commit and applies only to the root + % message; `hint-device' lets the structured codec preserve each nested + % commitment's own bundle state per-node. + SourceSpec = + hb_message:add_bundle_hint( + #{ <<"device">> => <<"structured@1.0">> }, + Req#{ <<"device">> => AttDev }, + CommitOpts + ), Loaded = ensure_commitments_loaded( - hb_message:convert(Base, tabm, CommitOpts), + hb_message:convert(Base, tabm, SourceSpec, CommitOpts), Opts ), {ok, Committed} = @@ -269,22 +293,13 @@ commit(Self, Req, Opts) -> verify(Self, Req, Opts) -> % Get the target message of the verification request. {ok, RawBase} = hb_message:find_target(Self, Req, Opts), - Base = - hb_message:convert( - ensure_commitments_loaded( - RawBase, - Opts - ), - tabm, - Opts - ), - ?event(verify, {verify, {base_found, Base}}), - Commitments = maps:get(<<"commitments">>, Base, #{}), - IDsToVerify = commitment_ids_from_request(Base, Req, Opts), + CommitmentBase = ensure_commitments_loaded(RawBase, Opts), + Commitments = maps:get(<<"commitments">>, CommitmentBase, #{}), + IDsToVerify = commitment_ids_from_request(CommitmentBase, Req, Opts), % Generate the new commitment request base messsage by removing the keys % used by this function (path, committers, commitments) and returning the % remaining keys. This message will then be merged with each commitment - % message to generate the final request, allowing the caller to pass + % message to generate the final request, allowing the caller to pass % additional keys to the commitment device. ReqBase = maps:without( @@ -300,13 +315,36 @@ verify(Self, Req, Opts) -> Res = lists:all( fun(CommitmentID) -> + Commitment = maps:merge( + ReqBase, + maps:get(CommitmentID, Commitments) + ), + % Build the source spec from the commitment device alone: a + % `hint-device' lets the structured codec reproduce each + % subtree in the bundle state it was committed in. The verify + % request's `bundle' is deliberately *not* propagated -- a + % commitment is always verified in the state it was signed + % in, so any `bundle' passed by the caller is irrelevant. + SourceSpec = + hb_message:add_bundle_hint( + #{ <<"device">> => <<"structured@1.0">> }, + #{ + <<"device">> => + maps:get( + <<"commitment-device">>, + Commitment, + undefined + ) + }, + Opts + ), + Base = hb_message:convert( + CommitmentBase, tabm, SourceSpec, Opts), + ?event(verify, {verify, {base_found, Base}}), {ok, Res} = verify_commitment( Base, - maps:merge( - ReqBase, - maps:get(CommitmentID, Commitments) - ), + Commitment, Opts ), ?event(verify, diff --git a/test/arbundles.js/upload-dataitem.js b/test/arbundles.js/upload-dataitem.js index 7f153d334..1175a5faf 100644 --- a/test/arbundles.js/upload-dataitem.js +++ b/test/arbundles.js/upload-dataitem.js @@ -45,7 +45,7 @@ async function uploadDataItem(itemPath, gatewayUrl = "https://up.arweave.net") { // Upload to the gateway console.log("Uploading to gateway..."); - const uploadUrl = `${gatewayUrl}/tx`; + const uploadUrl = `${gatewayUrl}/~bundler@1.0/tx`; const response = await axios.post(uploadUrl, itemBuffer, { headers: { diff --git a/test/arbundles.js/upload-items.js b/test/arbundles.js/upload-items.js index 9bc63d475..67579202f 100644 --- a/test/arbundles.js/upload-items.js +++ b/test/arbundles.js/upload-items.js @@ -3,19 +3,20 @@ const path = require("path"); const { ArweaveSigner, createData } = require("@dha-team/arbundles"); // Configuration -const BUNDLER_URL = "http://localhost:8734"; +const BUNDLER_URL = process.env.BUNDLER_URL || "http://localhost:8734"; +const ENDPOINT_PATH = process.env.ENDPOINT_PATH || "/~bundler@1.0/item?codec-device=ans104@1.0"; const DEFAULT_WALLET = "../../hyperbeam-key.json"; const CONCURRENT_UPLOADS = 100; // Number of parallel uploads async function performanceTest(walletPath, itemCount, bytesPerItem = 0) { const wallet = require(path.resolve(walletPath)); const signer = new ArweaveSigner(wallet); - const endpoint = `${BUNDLER_URL}/~bundler@1.0/item?codec-device=ans104@1.0`; + const endpoint = `${BUNDLER_URL}${ENDPOINT_PATH}`; console.log("\n" + "=".repeat(70)); console.log("ANS-104 Bundle Upload Performance Test"); console.log("=".repeat(70)); - console.log(`Target: ${BUNDLER_URL}`); + console.log(`Target: ${endpoint}`); console.log(`Items: ${itemCount}`); console.log(`Item Size: ${bytesPerItem > 0 ? `~${bytesPerItem} bytes` : 'default'}`); console.log(`Concurrent: ${CONCURRENT_UPLOADS}`); @@ -147,10 +148,20 @@ if (require.main === module) { console.error(" number_of_items - Number of data items to create and upload"); console.error(" bytes_per_item - Minimum size of each item in bytes (optional)"); console.error(""); + console.error("Environment variables:"); + console.error(" BUNDLER_URL - Gateway base URL (default: http://localhost:8734)"); + console.error(" ENDPOINT_PATH - Path appended to gateway (default: /~bundler@1.0/item?codec-device=ans104@1.0)"); + console.error(""); console.error("Examples:"); console.error(" node upload-items.js 100"); console.error(" node upload-items.js 100 1024"); console.error(" node upload-items.js /path/to/wallet.json 100 1024"); + console.error(" BUNDLER_URL=https://forward.computer node upload-items.js 100"); + console.error(" BUNDLER_URL=https://forward.computer ENDPOINT_PATH='/~bundler@1.0/tx?codec-device=ans104@1.0' node upload-items.js 1"); + console.error(""); + console.error("Note: when posting raw ANS-104 bytes, ENDPOINT_PATH must include"); + console.error(" ?codec-device=ans104@1.0 — otherwise the server will reject"); + console.error(" the body as 'unsigned-item' (no signers visible)."); process.exit(1); }