From 420db9988a505d8497236bc7ad2fcac79a885834 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 13 Apr 2026 17:37:53 -0400 Subject: [PATCH 01/20] impr: `is-admissible` takes `http-reference` and preserves `device` + `path` --- src/dev_hook.erl | 6 ++---- src/dev_query.erl | 13 +++---------- src/hb_http_multi.erl | 22 ++++++++++++++-------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/dev_hook.erl b/src/dev_hook.erl index 1696745ec..29e85a156 100644 --- a/src/dev_hook.erl +++ b/src/dev_hook.erl @@ -164,10 +164,8 @@ execute_handler(HookName, Handler, Req, Opts) -> % committed before execution. BaseReq = Req#{ - <<"path">> => - hb_maps:get(<<"path">>, Handler, HookName, Opts), - <<"method">> => - hb_maps:get(<<"method">>, Handler, <<"GET">>, Opts) + <<"path">> => hb_maps:get(<<"path">>, Handler, HookName, Opts), + <<"method">> => hb_maps:get(<<"method">>, Handler, <<"GET">>, Opts) }, CommitReqBin = hb_util:bin( diff --git a/src/dev_query.erl b/src/dev_query.erl index 4dc43a09d..776acc19b 100644 --- a/src/dev_query.erl +++ b/src/dev_query.erl @@ -50,16 +50,9 @@ graphql(Req, Base, Opts) -> %% @doc Return whether a GraphQL esponse in a message has transaction results. %% This key is used in HB's gateway client multirequest configuration to %% determine if the response from the node should be considered admissible. -has_results(Base, Req, Opts) -> - JSON = - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, Base}, <<"body">>}, - {{as, <<"message@1.0">>, Req}, <<"body">>} - ], - <<"{}">>, - Opts - ), +has_results(_Base, RawReq, Opts) -> + Req = hb_maps:get(<<"body">>, RawReq, #{}, Opts), + JSON = hb_maps:get(<<"body">>, Req, <<>>, Opts), Decoded = hb_json:decode(JSON), ?event(debug_multi, {has_results, {decoded_json, Decoded}}), case Decoded of diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index c428c6b1d..ebdbc8732 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -237,14 +237,20 @@ admissible_status(Status, Statuses) when is_list(Statuses) -> %% @doc If an `admissable` message is set for the request, check if the response %% adheres to it. Else, return `true'. admissible_response(_Response, undefined, _Opts) -> true; -admissible_response(Response, Msg, Opts) -> - Path = hb_maps:get(<<"path">>, Msg, <<"is-admissible">>, Opts), - Req = Response#{ <<"path">> => Path }, - Base = hb_message:without_unless_signed([<<"path">>], Msg, Opts), - ?event(debug_multi, - {executing_admissible_message, {message, Base}, {req, Req}} - ), - try hb_ao:resolve(Base, Req, Opts) of +admissible_response(Response, IsAdmissible, Opts) -> + Req = + IsAdmissible#{ + <<"path">> => + hb_maps:get( + <<"path">>, + IsAdmissible, + <<"is-admissible">>, + Opts + ), + <<"body">> => Response, + <<"http-reference">> => hb_opts:get(http_reference, not_found, Opts) + }, + try hb_ao:resolve(Req, Opts#{ force_message => false }) of {ok, Res} when is_atom(Res) or is_binary(Res) -> ?event(debug_multi, {admissible_result, {result, Res}}), hb_util:atom(Res) == true; From 58488ccf3cffc5fd8bfe6cbaa39de13147229965 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 13 Apr 2026 17:55:10 -0400 Subject: [PATCH 02/20] fix: tests; trace print --- src/dev_scheduler.erl | 29 +++++++++++++++-------------- src/hb_http_multi.erl | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index 3d165342a..c7d343fc9 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -1533,12 +1533,14 @@ redirect_from_graphql_test_() -> {timeout, 60, fun redirect_from_graphql/0}. redirect_from_graphql() -> start(), + TestStore = hb_test_utils:test_store(), Opts = - #{ store => - [ - #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, - #{ <<"store-module">> => hb_store_gateway, <<"store">> => [] } - ] + #{ + store => + [ + TestStore, + #{ <<"store-module">> => hb_store_gateway } + ] }, {ok, Msg} = hb_cache:read(<<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, Opts), ?assertMatch( @@ -1603,14 +1605,12 @@ http_init() -> http_init(#{}). http_init(Opts) -> start(), Wallet = ar_wallet:new(), + TestStore = hb_test_utils:test_store(), ExtendedOpts = Opts#{ priv_wallet => Wallet, store => [ - #{ - <<"store-module">> => hb_store_volatile, - <<"name">> => <<"cache-TEST/volatile">> - }, - #{ <<"store-module">> => hb_store_gateway, <<"store">> => [] } + TestStore, + #{ <<"store-module">> => hb_store_gateway } ] }, Node = hb_http_server:start_node(ExtendedOpts), @@ -1663,17 +1663,18 @@ http_get_schedule(N, PMsg, From, To, Format) -> http_get_schedule_redirect_test_() -> {timeout, 60, fun http_get_schedule_redirect/0}. http_get_schedule_redirect() -> + start(), + TestStore = hb_test_utils:test_store(), Opts = #{ store => [ - #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, - #{ <<"store-module">> => hb_store_gateway, <<"opts">> => #{} } + TestStore, + #{ <<"store-module">> => hb_store_gateway } ], - scheduler_follow_redirects => false + scheduler_follow_redirects => false }, {N, _Wallet} = http_init(Opts), - start(), ProcID = <<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, Res = hb_http:get(N, <<"/", ProcID/binary, "/schedule">>, Opts), ?assertMatch({ok, #{ <<"location">> := Location }} when is_binary(Location), Res). diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index ebdbc8732..7ac95530e 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -263,7 +263,7 @@ admissible_response(Response, IsAdmissible, Opts) -> {admissible_response, {class, Class}, {reason, Reason}, - {stacktrace, Stacktrace} + {stacktrace, {trace, Stacktrace}} } ), false From 39ba1ad0148e0eec35aa5c681f163b2fc815c94d Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 13 Apr 2026 18:17:18 -0400 Subject: [PATCH 03/20] wip: multi-node support in `hb_store_remote_node` --- src/dev_cache.erl | 16 +++++++++- src/hb_store_remote_node.erl | 59 ++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 7f49736b6..aa38cbd1e 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -3,10 +3,24 @@ %%% supports writing messages to the store, if the node message has the %%% writer's address in its `cache_writers' key. -module(dev_cache). --export([read/3, write/3, link/3, read_from_cache/2]). +-export([read/3, write/3, link/3, read_from_cache/2, expected_response/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). +%% @doc An `is-admissible`-compliant key that verifies a `~cache@1.0/read` +%% response contains the message we expect, and that it is valid. Additionally, +%% if the `http-reference` key is set, we execute the `on/client/valid-response` +%% hook. +expected_response(Base, Req, Opts) -> + maybe + {ok, Response} ?= hb_maps:get(<<"body">>, Req, Opts), + {ok, Expected} ?= hb_maps:get(<<"expected">>, Base, Opts), + {ok, Commitments} ?= hb_maps:get(<<"commitments">>, Response, #{}, Opts), + true ?= lists:member(Expected, Commitments), + hb_message:verify(Response, #{ <<"commitment-ids">> => [Expected] }, Opts) + else _ -> false + end. + %% @doc Read data from the cache. %% Retrieves data corresponding to a key from a local store. %% The key is extracted from the incoming message under <<"target">>. diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 90890e9c3..b7ab17686 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -62,6 +62,36 @@ read(Opts = #{ <<"node">> := Node }, Key) -> #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, Opts ), + handle_read_response(Key, HTTPRes, Opts); +read(Opts = #{ <<"nodes">> := Nodes }, Key) -> + ?event(store_remote_node, {executing_read, {nodes, Nodes}, {key, Key}}), + HTTPRes = + hb_http:request( + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"/~cache@1.0/read">>, + <<"target">> => Key, + <<"multirequest-responses">> => 1, + <<"multirequest-stop-after">> => true, + <<"multirequest-admissible">> => #{ + <<"device">> => <<"cache@1.0">>, + <<"path">> => <<"expected-response">> + } + }, + Opts#{ + routes => + [ + #{ + <<"template">> => <<"/~cache@1.0/read">>, + <<"nodes">> => Nodes, + <<"opts">> => Opts + } + ] + } + ), + handle_read_response(Key, HTTPRes, Opts). + +handle_read_response(Key, HTTPRes, Opts) -> case HTTPRes of {ok, Res} -> % returning the whole response to get the test-key @@ -72,8 +102,7 @@ read(Opts = #{ <<"node">> := Node }, Key) -> {error, _Err} -> ?event(store_remote_node, {read_not_found, {key, Key}}), not_found - end; -read(_, _) -> not_found. + end. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional @@ -252,3 +281,29 @@ read_only_ids_test() -> <<"only-ids">> => true } ], ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). + +multiread_test() -> + LocalStore1 = hb_test_utils:test_store(), + {ok, ID} = + hb_cache:write( + <<"message">>, + #{ store => LocalStore1 } + ), + Node1 = + hb_http_server:start_node( + #{ store => [LocalStore1] } + ), + Node2 = + hb_http_server:start_node( + #{ store => [hb_test_utils:test_store()] } + ), + RemoteStore = + [#{ + <<"store-module">> => hb_store_remote_node, + <<"nodes">> => [Node1, Node2] + }], + {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), + ?assertMatch( + #{ <<"message">> := <<"message">> }, + hb_cache:ensure_all_loaded(RetrievedMsg) + ). \ No newline at end of file From 668865425cc8faffc33790d94cc536d328c985ec Mon Sep 17 00:00:00 2001 From: Ayush Agrawal Date: Mon, 13 Apr 2026 20:59:56 -0400 Subject: [PATCH 04/20] fix: multi request test --- src/dev_cache.erl | 8 ++++---- src/hb_store_remote_node.erl | 29 +++++++++++++++++------------ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index aa38cbd1e..6b7a78cb7 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -13,10 +13,10 @@ %% hook. expected_response(Base, Req, Opts) -> maybe - {ok, Response} ?= hb_maps:get(<<"body">>, Req, Opts), - {ok, Expected} ?= hb_maps:get(<<"expected">>, Base, Opts), - {ok, Commitments} ?= hb_maps:get(<<"commitments">>, Response, #{}, Opts), - true ?= lists:member(Expected, Commitments), + {ok, Response} ?= hb_maps:find(<<"body">>, Req, Opts), + {ok, Expected} ?= hb_maps:find(<<"expected">>, Base, Opts), + {ok, Commitments} ?= hb_maps:find(<<"commitments">>, Response, Opts), + true ?= lists:member(Expected, maps:keys(Commitments)), hb_message:verify(Response, #{ <<"commitment-ids">> => [Expected] }, Opts) else _ -> false end. diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index b7ab17686..158e055bd 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -28,6 +28,9 @@ scope(_StoreOpts) -> %% @returns The resolved key. resolve(#{ <<"node">> := Node }, Key) -> ?event({remote_resolve, {node, Node}, {key, Key}}), + Key; +resolve(#{ <<"nodes">> := Nodes }, Key) -> + ?event({remote_resolve, {nodes, Nodes}, {key, Key}}), Key. %% @doc Determine the type of value at a given key. @@ -37,8 +40,8 @@ resolve(#{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key whose value type is determined. %% @returns simple if found, or not_found otherwise. -type(Opts = #{ <<"node">> := Node }, Key) -> - ?event({remote_type, {node, Node}, {key, Key}}), +type(Opts, Key) when is_map_key(<<"node">>, Opts); is_map_key(<<"nodes">>, Opts) -> + ?event({remote_type, {opts, Opts}, {key, Key}}), case read(Opts, Key) of not_found -> not_found; _ -> simple @@ -75,7 +78,8 @@ read(Opts = #{ <<"nodes">> := Nodes }, Key) -> <<"multirequest-stop-after">> => true, <<"multirequest-admissible">> => #{ <<"device">> => <<"cache@1.0">>, - <<"path">> => <<"expected-response">> + <<"path">> => <<"expected-response">>, + <<"expected">> => Key } }, Opts#{ @@ -283,12 +287,10 @@ read_only_ids_test() -> ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). multiread_test() -> - LocalStore1 = hb_test_utils:test_store(), - {ok, ID} = - hb_cache:write( - <<"message">>, - #{ store => LocalStore1 } - ), + LocalStore1 = hb_test_utils:test_store(), + Wallet = ar_wallet:new(), + Msg = hb_message:commit(#{ <<"key">> => <<"message">> }, #{priv_wallet => Wallet}), + {ok, ID} = hb_cache:write(Msg, #{ store => LocalStore1 }), Node1 = hb_http_server:start_node( #{ store => [LocalStore1] } @@ -300,10 +302,13 @@ multiread_test() -> RemoteStore = [#{ <<"store-module">> => hb_store_remote_node, - <<"nodes">> => [Node1, Node2] + <<"nodes">> => [ + #{ <<"prefix">> => Node1 }, + #{ <<"prefix">> => Node2 } + ] }], {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), ?assertMatch( - #{ <<"message">> := <<"message">> }, - hb_cache:ensure_all_loaded(RetrievedMsg) + #{ <<"key">> := <<"message">>}, + RetrievedMsg ). \ No newline at end of file From 0fa16395f0d86e572af98b540483bd65b8ed829d Mon Sep 17 00:00:00 2001 From: Ayush Agrawal Date: Mon, 13 Apr 2026 21:01:46 -0400 Subject: [PATCH 05/20] chore: no need to turn off force message --- src/hb_http_multi.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index 7ac95530e..c72a81537 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -250,7 +250,7 @@ admissible_response(Response, IsAdmissible, Opts) -> <<"body">> => Response, <<"http-reference">> => hb_opts:get(http_reference, not_found, Opts) }, - try hb_ao:resolve(Req, Opts#{ force_message => false }) of + try hb_ao:resolve(Req, Opts) of {ok, Res} when is_atom(Res) or is_binary(Res) -> ?event(debug_multi, {admissible_result, {result, Res}}), hb_util:atom(Res) == true; From 5715cb65e01ff68db84978d730e6ff2558c3f1ca Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 08:12:57 -0400 Subject: [PATCH 06/20] impr: accept multirequest directives; tidy logic --- src/hb_store_remote_node.erl | 45 ++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 158e055bd..4a90afa53 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -58,18 +58,23 @@ type(Opts, Key) when is_map_key(<<"node">>, Opts); is_map_key(<<"nodes">>, Opts) read(#{ <<"only-ids">> := true }, Key) when not ?IS_ID(Key) -> not_found; read(Opts = #{ <<"node">> := Node }, Key) -> - ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), - HTTPRes = - hb_http:get( - Node, - #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, - Opts + OptsWithoutNode = maps:remove(<<"node">>, Opts), + read(OptsWithoutNode#{ <<"nodes">> => [#{ <<"prefix">> => Node }] }, Key); +read(StoreOpts = #{ <<"nodes">> := Nodes }, Key) -> + MultirequestDirectives = + maps:filter( + fun(<<"multirequest-", _/binary>>, _) -> true; (_, _) -> false end, + StoreOpts ), - handle_read_response(Key, HTTPRes, Opts); -read(Opts = #{ <<"nodes">> := Nodes }, Key) -> - ?event(store_remote_node, {executing_read, {nodes, Nodes}, {key, Key}}), - HTTPRes = - hb_http:request( + ?event( + {read, + {nodes, Nodes}, + {key, Key}, + {multirequest_directives, MultirequestDirectives} + } + ), + HTTPReq = + maps:merge( #{ <<"method">> => <<"GET">>, <<"path">> => <<"/~cache@1.0/read">>, @@ -82,26 +87,32 @@ read(Opts = #{ <<"nodes">> := Nodes }, Key) -> <<"expected">> => Key } }, - Opts#{ + MultirequestDirectives + ), + ?event(store_remote_node, {http_request, HTTPReq}), + HTTPRes = + hb_http:request( + HTTPReq, + #{ routes => [ #{ <<"template">> => <<"/~cache@1.0/read">>, <<"nodes">> => Nodes, - <<"opts">> => Opts + <<"opts">> => StoreOpts } ] } ), - handle_read_response(Key, HTTPRes, Opts). + handle_read_response(Key, HTTPRes, StoreOpts). -handle_read_response(Key, HTTPRes, Opts) -> +handle_read_response(Key, HTTPRes, StoreOpts) -> case HTTPRes of {ok, Res} -> % returning the whole response to get the test-key - {ok, Msg} = hb_message:with_only_committed(Res, Opts), + {ok, Msg} = hb_message:with_only_committed(Res, StoreOpts), ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(Opts, Msg, [Key]), + maybe_cache(StoreOpts, Msg, [Key]), {ok, Msg}; {error, _Err} -> ?event(store_remote_node, {read_not_found, {key, Key}}), From 425d4bcdec4c306d06ad6e6a62f42539549b9a98 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 08:14:44 -0400 Subject: [PATCH 07/20] impr: remote store validity enforcement tests --- src/hb_store_remote_node.erl | 145 ++++++++++++++++++++++++++--------- 1 file changed, 110 insertions(+), 35 deletions(-) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 4a90afa53..2d31a6490 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -247,6 +247,44 @@ make_group(_StoreOpts, _Path) -> not_found. %%% Tests %%%-------------------------------------------------------------------- +multinode_env() -> + Node1Store = [hb_test_utils:test_store()], + Node2Store = [hb_test_utils:test_store()], + Wallet1 = ar_wallet:new(), + Wallet2 = ar_wallet:new(), + Opts1 = #{ priv_wallet => Wallet1, store => Node1Store }, + Opts2 = #{ priv_wallet => Wallet2, store => Node2Store }, + Msg1 = hb_message:commit(#{ <<"key1">> => <<"message1">>, <<"key2">> => 2 }, Opts1), + Msg2 = hb_message:commit(#{ <<"key2">> => <<"message2">> }, Opts2), + BothMsg = + hb_message:commit( + #{ <<"key-both">> => <<"value-both">> }, + Opts1 + ), + {ok, ID1} = hb_cache:write(Msg1, Opts1), + {ok, ID2} = hb_cache:write(Msg2, Opts2), + {ok, IDBoth} = hb_cache:write(BothMsg, Opts1), + {ok, IDBoth} = hb_cache:write(BothMsg, Opts2), + Node1 = hb_http_server:start_node(Opts1), + Node2 = hb_http_server:start_node(Opts2), + RemoteStore = + #{ + <<"store-module">> => hb_store_remote_node, + <<"nodes">> => [ + #{ <<"prefix">> => Node1, <<"http-reference">> => <<"node1">> }, + #{ <<"prefix">> => Node2, <<"http-reference">> => <<"node2">> } + ], + <<"parallel">> => 1 + }, + #{ + ids_single => [ID1, ID2], + id_both => [IDBoth], + nodes => [Node1, Node2], + stores => [Node1Store, Node2Store], + opts => [Opts1, Opts2], + remote_store => RemoteStore + }. + %% @doc Test that we can create a store, write a random message to it, then %% start a remote node with that store, and read the message from it. read_test() -> @@ -284,42 +322,79 @@ read_only_ids_test() -> <<"message">>, #{ store => LocalStore } ), - Node = - hb_http_server:start_node( - #{ - store => LocalStore - } - ), - RemoteStore = [ - #{ <<"store-module">> => hb_store_remote_node, - <<"node">> => Node, - <<"only-ids">> => true } - ], - ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). - -multiread_test() -> - LocalStore1 = hb_test_utils:test_store(), - Wallet = ar_wallet:new(), - Msg = hb_message:commit(#{ <<"key">> => <<"message">> }, #{priv_wallet => Wallet}), - {ok, ID} = hb_cache:write(Msg, #{ store => LocalStore1 }), - Node1 = - hb_http_server:start_node( - #{ store => [LocalStore1] } - ), - Node2 = - hb_http_server:start_node( - #{ store => [hb_test_utils:test_store()] } - ), + Node = hb_http_server:start_node(#{ store => LocalStore }), RemoteStore = - [#{ + #{ <<"store-module">> => hb_store_remote_node, - <<"nodes">> => [ - #{ <<"prefix">> => Node1 }, - #{ <<"prefix">> => Node2 } - ] - }], - {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), + <<"node">> => Node, + <<"only-ids">> => true + }, + ?assertEqual(not_found, hb_cache:read(ID, #{ store => [RemoteStore] })). + +multiread_test() -> + #{ ids_single := [ID1, ID2], remote_store := RemoteStore } = multinode_env(), + ?assertMatch( + {ok, #{ <<"key1">> := <<"message1">>}}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + ?assertMatch( + {ok, #{ <<"key2">> := <<"message2">>}}, + hb_cache:read(ID2, #{ store => RemoteStore }) + ). + +multiread_enforces_valid_response_test() -> + #{ + ids_single := [ID1|_], + stores := [Store1|_], + remote_store := RemoteStore + } = multinode_env(), + % Overwrite the message in the first nodes with an invalid value, such that + % `hb_message:verify` should fail. Start by reading the message back and + % checking that it is accessible (and valid) to start with. + ?assertMatch( + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + {ok, Msg} = hb_cache:read(ID1, #{ store => Store1 }), + CorruptMsg = Msg#{ <<"key1">> => <<"corrupt-value">> }, + {ok, CorruptID} = hb_cache:write(CorruptMsg, #{ store => Store1 }), + ?assertMatch(not_found, hb_cache:read(CorruptID, #{ store => RemoteStore })). + +multiread_enforces_expected_id_response_test() -> + #{ + id_both := IDBoth, + ids_single := [ID1, ID2], + stores := [Store1, Store2], + remote_store := RemoteStore + } = multinode_env(), + % Force an invalid link on one node to the nessage stored in both nodes. + FakeID = hb_util:human_id(<<0:256>>), + ok = hb_store:make_link(Store1, IDBoth, FakeID), + % Check we can read the message back from the store locally. This would be + % a local node store integrity failure if it were to happen in the wild, but + % our security model assumes that the local store is trustworthy for local + % computation. + {ok, RetrievedMsg} = hb_cache:read(FakeID, #{ store => Store1 }), + ?assertMatch( + #{ <<"key-both">> := <<"value-both">> }, + hb_cache:ensure_all_loaded(RetrievedMsg) + ), + % Ensure that we _cannot_ read the message back from the remote node. This + % should fail despite the remote peer returning a valid message (with the + % wrong message) because the `multirequest-admissible' directive will fail. + ?assertMatch( + not_found, + hb_cache:read(FakeID, #{ store => RemoteStore }) + ), + % Now try linking ID2 to ID1 on the first node+store. The first node will + % return first but with the wrong message. It should fail and trigger a + % call to the second node, which should return it correctly. + ok = hb_store:make_link(Store1, ID1, ID2), + ?assert( + hb_cache:read(ID2, #{ store => Store1 }) =/= + hb_cache:read(ID2, #{ store => Store2 }) + ), ?assertMatch( - #{ <<"key">> := <<"message">>}, - RetrievedMsg + {ok, #{ <<"key2">> := <<"message2">>}}, + hb_cache:read(ID2, #{ store => RemoteStore }) ). \ No newline at end of file From 092bdf6800f9a3129b632e28c254fb91f66efe5b Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 08:55:18 -0400 Subject: [PATCH 08/20] impr: `~hook@1.0/on` can trigger on deeply nested paths --- src/dev_hook.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_hook.erl b/src/dev_hook.erl index 29e85a156..1307b2e0a 100644 --- a/src/dev_hook.erl +++ b/src/dev_hook.erl @@ -92,7 +92,7 @@ find(HookName, Opts) -> find(#{}, #{ <<"target">> => <<"body">>, <<"body">> => HookName }, Opts). find(_Base, Req, Opts) -> HookName = maps:get(maps:get(<<"target">>, Req, <<"body">>), Req), - case maps:get(HookName, hb_opts:get(on, #{}, Opts), []) of + case hb_util:deep_get(HookName, hb_opts:get(on, #{}, Opts), [], Opts) of Handler when is_map(Handler) -> case hb_util:is_ordered_list(Handler, Opts) of true -> From c8f9b74673856f29c8a78af784b8cdb036fcc87a Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 12:53:37 -0400 Subject: [PATCH 09/20] wip: avoid `ao-types` encoding `status` as integer --- src/dev_meta.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_meta.erl b/src/dev_meta.erl index 0dda4185e..d48d77b0f 100644 --- a/src/dev_meta.erl +++ b/src/dev_meta.erl @@ -315,7 +315,7 @@ embed_status({ErlStatus, Res}, NodeMsg) when is_map(Res) -> case lists:member(<<"status">>, hb_message:committed(Res, all, NodeMsg)) of false -> HTTPCode = status_code({ErlStatus, Res}, NodeMsg), - {ok, Res#{ <<"status">> => HTTPCode }}; + {ok, Res#{ <<"status">> => hb_util:bin(HTTPCode) }}; true -> {ok, Res} end; From 24cf6ad8e8d38e09d2d52b6508cd4535e787291b Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 12:54:26 -0400 Subject: [PATCH 10/20] wip: test setup --- src/dev_cache.erl | 30 +++++++- src/dev_test.erl | 47 ++++++++++++ src/hb_http_multi.erl | 9 ++- src/hb_store_remote_node.erl | 143 +++++++++++++++++++++++++++-------- 4 files changed, 189 insertions(+), 40 deletions(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 6b7a78cb7..26eda6ed7 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -9,16 +9,38 @@ %% @doc An `is-admissible`-compliant key that verifies a `~cache@1.0/read` %% response contains the message we expect, and that it is valid. Additionally, -%% if the `http-reference` key is set, we execute the `on/client/valid-response` +%% if the `http-reference` key is set, we execute the `on/cache-valid-response` %% hook. expected_response(Base, Req, Opts) -> maybe {ok, Response} ?= hb_maps:find(<<"body">>, Req, Opts), {ok, Expected} ?= hb_maps:find(<<"expected">>, Base, Opts), {ok, Commitments} ?= hb_maps:find(<<"commitments">>, Response, Opts), - true ?= lists:member(Expected, maps:keys(Commitments)), - hb_message:verify(Response, #{ <<"commitment-ids">> => [Expected] }, Opts) - else _ -> false + CommIDs = maps:keys(Commitments), + ?event(debug_admissible, + {expected_response, + {response, {explicit, Response}}, + {expected, Expected}, + {commitments, CommIDs} + } + ), + true ?= lists:member(Expected, CommIDs) orelse expected_id_not_found, + {ok, OnlyCommitted} = hb_message:with_only_committed(Response, Opts), + true ?= + hb_message:verify( + OnlyCommitted, + #{ <<"commitment-ids">> => [Expected] }, + Opts + ) orelse invalid_commitment, + dev_hook:on( + [<<"~cache@1.0">>, <<"admissible-response">>], + Response, + Opts + ), + {ok, true} + else Reason -> + ?event(debug_admissible, {expected_response_error, Reason}), + {ok, false} end. %% @doc Read data from the cache. diff --git a/src/dev_test.erl b/src/dev_test.erl index 4a3632632..6d692d015 100644 --- a/src/dev_test.erl +++ b/src/dev_test.erl @@ -2,6 +2,7 @@ -export([info/3]). -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). -export([mangle/3, update_state/3, increment_counter/3, delay/3]). +-export([log_request/3, logs/3]). -export([index/3, postprocess/3, load/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). @@ -209,6 +210,52 @@ mangle(Base, _Req, Opts) -> end end. +%%% Logged messages functionality + +-define(LOG_PREFIX, "~test-device@1.0/request-"). + +%% @doc Determines the store to use for logging requests. +determine_log_store(Base, _Req, Opts) -> + hb_maps:get( + <<"store">>, + Base, + hb_opts:get(store, no_viable_store, Opts), + Opts + ). + +%% @doc Write a pseudo-path to the store linking the received message to the +%% millisecond timestamp that the key was invoked. +log_request(Base, Req, Opts) -> + Timestamp = hb_util:bin(erlang:system_time(millisecond)), + Store = determine_log_store(Base, Req, Opts), + {ok, ReqID} = hb_cache:write(Req, Opts#{ store => Store }), + hb_store:make_link( + Store, + ReqID, + <>) + , + {ok, ReqID}. + +%% @doc Return all logs of requests to the device. +logs(Base, Req, Opts) -> + Store = determine_log_store(Base, Req, Opts), + LogOpts = Opts#{ store => Store }, + case hb_store:list(Store, <>) of + {ok, LogKeys} -> + Logs = + maps:from_list( + lists:map( + fun(K = <>) -> + {ok, Request} = hb_cache:read(K, LogOpts), + {hb_util:int(TimeBin), Request} + end, + LogKeys + ) + ), + {ok, Logs}; + _ -> {ok, #{}} + end. + %%% Tests %% @doc Tests the resolution of a default function. diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index c72a81537..5795acfa2 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -248,14 +248,15 @@ admissible_response(Response, IsAdmissible, Opts) -> Opts ), <<"body">> => Response, - <<"http-reference">> => hb_opts:get(http_reference, not_found, Opts) + <<"http-reference">> => hb_opts:get(http_reference, undefined, Opts) }, - try hb_ao:resolve(Req, Opts) of + ?event(debug_admissible, {admissible_response, {request, Req}, {opts, Opts}}), + try hb_ao:resolve(Req, Opts#{ hashpath => ignore }) of {ok, Res} when is_atom(Res) or is_binary(Res) -> - ?event(debug_multi, {admissible_result, {result, Res}}), + ?event(debug_admissible, {admissible_result, {result, Res}}), hb_util:atom(Res) == true; {error, Reason} -> - ?event(debug_multi, {admissible_error, {reason, Reason}}), + ?event(debug_admissible, {admissible_error, {reason, Reason}}), false catch Class:Reason:Stacktrace -> diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 2d31a6490..314ec634a 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -89,35 +89,48 @@ read(StoreOpts = #{ <<"nodes">> := Nodes }, Key) -> }, MultirequestDirectives ), - ?event(store_remote_node, {http_request, HTTPReq}), + MaybeHooks = + case maps:find(<<"on">>, StoreOpts) of + {ok, Hooks} -> #{ on => Hooks }; + error -> #{} + end, + ?event( + store_remote_node, + {remote_read, {request, HTTPReq}, {hooks, MaybeHooks}} + ), HTTPRes = hb_http:request( HTTPReq, - #{ + MaybeHooks#{ + store => [#{ <<"store-module">> => hb_store_lmdb, <<"name">> => <<"cache-sadness">> }], + match_index => false, routes => [ #{ <<"template">> => <<"/~cache@1.0/read">>, - <<"nodes">> => Nodes, - <<"opts">> => StoreOpts + <<"nodes">> => Nodes } ] } ), handle_read_response(Key, HTTPRes, StoreOpts). -handle_read_response(Key, HTTPRes, StoreOpts) -> - case HTTPRes of - {ok, Res} -> - % returning the whole response to get the test-key - {ok, Msg} = hb_message:with_only_committed(Res, StoreOpts), - ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(StoreOpts, Msg, [Key]), - {ok, Msg}; - {error, _Err} -> - ?event(store_remote_node, {read_not_found, {key, Key}}), - not_found - end. +%% @doc Handle a read response from the remote node, filtering the raw response +%% and invoking a possible local cache write operation. +handle_read_response(Key, {ok, Res}, StoreOpts) -> + {ok, Msg} = hb_message:with_only_committed(Res, StoreOpts), + ?event( + debug_admissible, + {remote_read, {only_committed, {explicit, Msg}}, {raw_response, Res}} + ), + maybe_cache(StoreOpts, Msg, [Key]), + {ok, Msg}; +handle_read_response(Key, UnexpectedRes, _StoreOpts) -> + ?event( + debug_admissible, + {read_failed, {key, Key}, {unexpected_response, UnexpectedRes}} + ), + not_found. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional @@ -254,7 +267,7 @@ multinode_env() -> Wallet2 = ar_wallet:new(), Opts1 = #{ priv_wallet => Wallet1, store => Node1Store }, Opts2 = #{ priv_wallet => Wallet2, store => Node2Store }, - Msg1 = hb_message:commit(#{ <<"key1">> => <<"message1">>, <<"key2">> => 2 }, Opts1), + Msg1 = hb_message:commit(#{ <<"key1">> => <<"message1">>, <<"num1">> => 1 }, Opts1), Msg2 = hb_message:commit(#{ <<"key2">> => <<"message2">> }, Opts2), BothMsg = hb_message:commit( @@ -270,6 +283,7 @@ multinode_env() -> RemoteStore = #{ <<"store-module">> => hb_store_remote_node, + <<"max-retries">> => 0, <<"nodes">> => [ #{ <<"prefix">> => Node1, <<"http-reference">> => <<"node1">> }, #{ <<"prefix">> => Node2, <<"http-reference">> => <<"node2">> } @@ -342,28 +356,33 @@ multiread_test() -> hb_cache:read(ID2, #{ store => RemoteStore }) ). -multiread_enforces_valid_response_test() -> +corrupted_id_test() -> #{ ids_single := [ID1|_], stores := [Store1|_], remote_store := RemoteStore } = multinode_env(), - % Overwrite the message in the first nodes with an invalid value, such that - % `hb_message:verify` should fail. Start by reading the message back and - % checking that it is accessible (and valid) to start with. + % Start by reading the message back and checking that it is accessible + % (and valid) to start with. ?assertMatch( {ok, #{ <<"key1">> := _ }}, hb_cache:read(ID1, #{ store => RemoteStore }) ), {ok, Msg} = hb_cache:read(ID1, #{ store => Store1 }), - CorruptMsg = Msg#{ <<"key1">> => <<"corrupt-value">> }, - {ok, CorruptID} = hb_cache:write(CorruptMsg, #{ store => Store1 }), - ?assertMatch(not_found, hb_cache:read(CorruptID, #{ store => RemoteStore })). + % Corrupt the value of `key1`, but keep the commitments. These commitments + % will now be invalid. A local store will return this invalid value, but + % a remote store will not. + hb_cache:write(Msg#{ <<"key1">> => <<"corrupt-value">> }, #{ store => Store1 }), + {ok, ReadCorruptMsg} = hb_cache:read(ID1, #{ store => Store1 }), + ?assertMatch( + #{ <<"key1">> := <<"corrupt-value">> }, + hb_cache:ensure_all_loaded(ReadCorruptMsg, #{ store => Store1 }) + ), + ?assertMatch(not_found, hb_cache:read(ID1, #{ store => RemoteStore })). -multiread_enforces_expected_id_response_test() -> +multiread_corrupted_id_test() -> #{ id_both := IDBoth, - ids_single := [ID1, ID2], stores := [Store1, Store2], remote_store := RemoteStore } = multinode_env(), @@ -385,16 +404,76 @@ multiread_enforces_expected_id_response_test() -> ?assertMatch( not_found, hb_cache:read(FakeID, #{ store => RemoteStore }) - ), - % Now try linking ID2 to ID1 on the first node+store. The first node will - % return first but with the wrong message. It should fail and trigger a + ). + +multiread_swapped_id_test() -> + #{ + ids_single := [ID1, ID2], + nodes := [Node1, Node2], + stores := [Store1, Store2], + remote_store := RemoteStore + } = multinode_env(), + % Link ID2 to ID1 on the first node and store. The first node will + % return ID2 but with the wrong message. It should fail and trigger a % call to the second node, which should return it correctly. ok = hb_store:make_link(Store1, ID1, ID2), - ?assert( - hb_cache:read(ID2, #{ store => Store1 }) =/= + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, hb_cache:read(ID2, #{ store => Store2 }) ), ?assertMatch( - {ok, #{ <<"key2">> := <<"message2">>}}, + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID2, #{ store => Store1 }) + ), + % Verify that a remote store with only the corrupt node will not return ID2, + % but a store with both the corrupt and correct nodes will. + ?assertMatch( + not_found, + hb_cache:read( + ID2, + #{ store => RemoteStore#{ <<"nodes">> => [#{ <<"prefix">> => Node1 }] } } + ) + ), + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, + hb_cache:read(ID2, #{ store => Store2 }) + ), + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, hb_cache:read(ID2, #{ store => RemoteStore }) + ). + +multiread_admissible_responsehook_test() -> + #{ + ids_single := [ID1|_], + stores := [Store1|_], + remote_store := BaseRemoteStore + } = multinode_env(), + % Ensure that we can execute a hook on valid read responses. + LogStore = hb_test_utils:test_store(), + RemoteStore = + BaseRemoteStore#{ + <<"on">> => #{ + <<"~cache@1.0">> => + #{ + <<"valid-response">> => #{ + <<"device">> => <<"test-device@1.0">>, + <<"store">> => LogStore, + <<"path">> => <<"log-request">> + } + } + } + }, + Opts = #{ store => RemoteStore }, + ?assertMatch( + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + ?assertMatch( + {ok, Logs} when is_map(Logs) andalso map_size(Logs) > 1, + hb_ao:resolve( + #{ <<"device">> => <<"test-device@1.0">> }, + <<"logs">>, + Opts#{ store => LogStore } + ) ). \ No newline at end of file From 6ced7f87f039745f4023369e941c80ef10c84864 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 17:40:58 -0400 Subject: [PATCH 11/20] fix: always strip to only committed before commit/verify --- src/dev_message.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dev_message.erl b/src/dev_message.erl index 857845e8e..24886c26e 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -271,9 +271,10 @@ commit(Self, Req, Opts) -> CommitOpts ), % Encode to a TABM + {ok, OnlyCommitted} = hb_message:with_only_committed(Base, Opts), Loaded = ensure_commitments_loaded( - hb_message:convert(Base, tabm, CommitOpts), + hb_message:convert(OnlyCommitted, tabm, CommitOpts), Opts ), {ok, Committed} = @@ -297,10 +298,11 @@ commit(Self, Req, Opts) -> verify(Self, Req, Opts) -> % Get the target message of the verification request. {ok, RawBase} = hb_message:find_target(Self, Req, Opts), + {ok, OnlyCommitted} = hb_message:with_only_committed(RawBase, Opts), Base = hb_message:convert( ensure_commitments_loaded( - RawBase, + OnlyCommitted, Opts ), tabm, From e601c0f647444d76167951cca0b50e9f5f68678f Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 14 Apr 2026 18:13:59 -0400 Subject: [PATCH 12/20] fix: `hb_store_remote_node` admissible response hook + test --- src/dev_cache.erl | 2 +- src/dev_test.erl | 17 +++++++++++------ src/hb_store_remote_node.erl | 18 +++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 26eda6ed7..2714e6a70 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -19,7 +19,7 @@ expected_response(Base, Req, Opts) -> CommIDs = maps:keys(Commitments), ?event(debug_admissible, {expected_response, - {response, {explicit, Response}}, + {response, Response}, {expected, Expected}, {commitments, CommIDs} } diff --git a/src/dev_test.erl b/src/dev_test.erl index 6d692d015..0dc2a7619 100644 --- a/src/dev_test.erl +++ b/src/dev_test.erl @@ -212,7 +212,7 @@ mangle(Base, _Req, Opts) -> %%% Logged messages functionality --define(LOG_PREFIX, "~test-device@1.0/request-"). +-define(LOG_PREFIX, "~test-device@1.0"). %% @doc Determines the store to use for logging requests. determine_log_store(Base, _Req, Opts) -> @@ -232,8 +232,8 @@ log_request(Base, Req, Opts) -> hb_store:make_link( Store, ReqID, - <>) - , + <> + ), {ok, ReqID}. %% @doc Return all logs of requests to the device. @@ -245,15 +245,20 @@ logs(Base, Req, Opts) -> Logs = maps:from_list( lists:map( - fun(K = <>) -> - {ok, Request} = hb_cache:read(K, LogOpts), + fun(K = <<"request-", TimeBin/binary>>) -> + {ok, Request} = + hb_cache:read( + <<"~test-device@1.0/", K/binary>>, + LogOpts + ), {hb_util:int(TimeBin), Request} end, LogKeys ) ), {ok, Logs}; - _ -> {ok, #{}} + _ -> + {error, <<"No logs found.">>} end. %%% Tests diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 314ec634a..f2cb7341b 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -89,21 +89,18 @@ read(StoreOpts = #{ <<"nodes">> := Nodes }, Key) -> }, MultirequestDirectives ), + % TODO: When `opts` key normalization lands, we should re-work this. MaybeHooks = case maps:find(<<"on">>, StoreOpts) of {ok, Hooks} -> #{ on => Hooks }; error -> #{} end, - ?event( - store_remote_node, - {remote_read, {request, HTTPReq}, {hooks, MaybeHooks}} - ), + ?event({remote_read, {request, HTTPReq}, {hooks, MaybeHooks}}), HTTPRes = hb_http:request( HTTPReq, MaybeHooks#{ - store => [#{ <<"store-module">> => hb_store_lmdb, <<"name">> => <<"cache-sadness">> }], - match_index => false, + cache_control => [<<"no-cache">>, <<"no-store">>], routes => [ #{ @@ -121,7 +118,7 @@ handle_read_response(Key, {ok, Res}, StoreOpts) -> {ok, Msg} = hb_message:with_only_committed(Res, StoreOpts), ?event( debug_admissible, - {remote_read, {only_committed, {explicit, Msg}}, {raw_response, Res}} + {remote_read, {only_committed, Msg}, {raw_response, Res}} ), maybe_cache(StoreOpts, Msg, [Key]), {ok, Msg}; @@ -443,20 +440,19 @@ multiread_swapped_id_test() -> hb_cache:read(ID2, #{ store => RemoteStore }) ). -multiread_admissible_responsehook_test() -> +multiread_admissible_response_hook_test() -> #{ ids_single := [ID1|_], - stores := [Store1|_], remote_store := BaseRemoteStore } = multinode_env(), % Ensure that we can execute a hook on valid read responses. - LogStore = hb_test_utils:test_store(), + LogStore = [hb_test_utils:test_store()], RemoteStore = BaseRemoteStore#{ <<"on">> => #{ <<"~cache@1.0">> => #{ - <<"valid-response">> => #{ + <<"admissible-response">> => #{ <<"device">> => <<"test-device@1.0">>, <<"store">> => LogStore, <<"path">> => <<"log-request">> From fa296d81d933e339cd9e2aeab4529ca664a840fa Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Apr 2026 15:13:44 -0400 Subject: [PATCH 13/20] fix: ensure that read target is part of default HTTP `vary`ing scheme --- src/hb_store_remote_node.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index f2cb7341b..cc290c9b1 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -77,8 +77,7 @@ read(StoreOpts = #{ <<"nodes">> := Nodes }, Key) -> maps:merge( #{ <<"method">> => <<"GET">>, - <<"path">> => <<"/~cache@1.0/read">>, - <<"target">> => Key, + <<"path">> => <<"/~cache@1.0/read?target=", Key/binary>>, <<"multirequest-responses">> => 1, <<"multirequest-stop-after">> => true, <<"multirequest-admissible">> => #{ From f3e0b111b594b2eea39deea4ae942f7b49d1dca0 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Apr 2026 15:55:10 -0400 Subject: [PATCH 14/20] fix: only enforce membership of a read path from a remote when it is an ID --- src/dev_cache.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 2714e6a70..8048fbea2 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -24,7 +24,10 @@ expected_response(Base, Req, Opts) -> {commitments, CommIDs} } ), - true ?= lists:member(Expected, CommIDs) orelse expected_id_not_found, + true ?= + (not ?IS_ID(Expected) + orelse lists:member(Expected, CommIDs)) + orelse expected_id_not_found, {ok, OnlyCommitted} = hb_message:with_only_committed(Response, Opts), true ?= hb_message:verify( From f48dca1cdd74eb81214427650f4b857c80096f7f Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Apr 2026 15:55:40 -0400 Subject: [PATCH 15/20] chore: add real-world Arweave.net `hb_store_remote_node` tests --- src/hb_store_remote_node.erl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index cc290c9b1..7845c0e54 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -471,4 +471,30 @@ multiread_admissible_response_hook_test() -> <<"logs">>, Opts#{ store => LogStore } ) + ). + +arweave_dot_net_as_remote_node_test() -> + TestIDs = + [ + <<"93Ui7nOLDNVCVMLeFkVeeOCVkm5Jy-kf6FNatW3q2TI">>, + <<"VuhnX2G8qVAb6kwHOiCQKl2c-42uoMKSIpHgKc0Pnzg">> + ], + Opts = + #{ + store => + [ + #{ + <<"store-module">> => hb_store_remote_node, + <<"name">> => <<"cache-arweave">>, + <<"node">> => <<"https://arweave.net">> + } + ] + }, + % Recent bundled AO messages -- no `signature` tag collision. + lists:foreach( + fun(ID) -> + {ok, M} = hb_cache:read(ID, Opts), + ?assert(hb_message:verify(M, all, Opts)) + end, + TestIDs ). \ No newline at end of file From 4099a5074fd00a1eafefe4990a025c60d5ef9f43 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 16 Apr 2026 18:27:22 -0400 Subject: [PATCH 16/20] fix: writing `ID` and `data/ID` paths; verify all commitments on non-ID read --- src/dev_cache.erl | 66 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 8048fbea2..5dde59030 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -15,6 +15,46 @@ expected_response(Base, Req, Opts) -> maybe {ok, Response} ?= hb_maps:find(<<"body">>, Req, Opts), {ok, Expected} ?= hb_maps:find(<<"expected">>, Base, Opts), + true ?= check_response_matches_expected(Response, Expected, Opts), + dev_hook:on( + [<<"~cache@1.0">>, <<"admissible-response">>], + Response, + Opts + ), + {ok, true} + else Reason -> + ?event(debug_admissible, {expected_response_error, Reason}), + {ok, false} + end. + +%% @doc Verify that a response from a remote cache matches the expected ID. +%% There are three cases: +%% 1. The response is a raw binary (e.g., a direct content-addressed read +%% of a data blob written with `hb_cache:write/2'): the binary's SHA-256 +%% hash must match `Expected'. +%% 2. The response is a structured message with commitments, and `Expected' +%% is a commitment ID (the common case for data items): `Expected' must +%% be among the commitment IDs and the corresponding commitment must +%% verify. +%% 3. The response is a structured message but `Expected' is a path rather +%% than a commitment ID (e.g. `~scheduler@1.0/assignments//'): +%% we verify all committer-attributed commitments on the response. +check_response_matches_expected(Response, Expected, _Opts) when is_binary(Response) -> + % Accept either a bare ID or a `data/' path (the convention used by + % `hb_cache:write/2' for content-addressed binary blobs). In both cases, + % the blob's SHA-256 hash must match the expected ID. + BinID = + case binary:split(Expected, <<"/">>) of + [<<"data">>, ID] -> ID; + _ -> Expected + end, + case ?IS_ID(BinID) + andalso hb_util:human_id(hb_crypto:sha256(Response)) =:= BinID of + true -> true; + false -> binary_hash_mismatch + end; +check_response_matches_expected(Response, Expected, Opts) -> + maybe {ok, Commitments} ?= hb_maps:find(<<"commitments">>, Response, Opts), CommIDs = maps:keys(Commitments), ?event(debug_admissible, @@ -29,21 +69,19 @@ expected_response(Base, Req, Opts) -> orelse lists:member(Expected, CommIDs)) orelse expected_id_not_found, {ok, OnlyCommitted} = hb_message:with_only_committed(Response, Opts), + % For ID-based reads, verify the specific commitment that claims to be + % the requested ID. For non-ID reads (path-based reads), the `Expected' + % value is not itself a commitment ID, so we instead verify all + % committer-attributed commitments on the response. + VerifyReq = + case ?IS_ID(Expected) of + true -> #{ <<"commitment-ids">> => [Expected] }; + false -> #{ <<"committers">> => <<"all">> } + end, true ?= - hb_message:verify( - OnlyCommitted, - #{ <<"commitment-ids">> => [Expected] }, - Opts - ) orelse invalid_commitment, - dev_hook:on( - [<<"~cache@1.0">>, <<"admissible-response">>], - Response, - Opts - ), - {ok, true} - else Reason -> - ?event(debug_admissible, {expected_response_error, Reason}), - {ok, false} + hb_message:verify(OnlyCommitted, VerifyReq, Opts) + orelse invalid_commitment, + true end. %% @doc Read data from the cache. From bb7521eb8db2e44670cf273a05db4974a28f1695 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 17 Apr 2026 01:01:01 -0400 Subject: [PATCH 17/20] fix: symmetrical handling of `status` encoding Co-Authored-By: Claude Opus 4.7 (1M context) --- src/dev_codec_httpsig.erl | 26 ++++++++++++++++++++++++++ src/dev_meta.erl | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/dev_codec_httpsig.erl b/src/dev_codec_httpsig.erl index 8e9635a13..4f98e72bf 100644 --- a/src/dev_codec_httpsig.erl +++ b/src/dev_codec_httpsig.erl @@ -575,6 +575,32 @@ validate_large_message_from_http_test() -> ?assert(hb_message:verify(OnlyCommitted, all, Opts)), ?event({msg_with_only_committed_verifies_hmac, <<"hmac-sha256">>}). +%% @doc Ensure that a signed response that contains both `status' and another +%% top-level locally-typed key (e.g. an integer) round-trips through HTTP and +%% verifies on the receiving side. This exercises the case where `ao-types' +%% must agree between signer and verifier for multiple locally-typed fields. +validate_sibling_typed_key_over_http_test() -> + Node = hb_http_server:start_node(Opts = #{ + force_signed => true, + commitment_device => <<"httpsig@1.0">>, + % Top-level locally-typed siblings to `status' in the signed response: + % one integer that sorts before `status' alphabetically, one that sorts + % after, and an atom. All become entries in `ao-types' alongside + % `status'. + <<"alpha-count">> => 7, + <<"test-count">> => 42, + <<"zebra-flag">> => true + }), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({received_with_sibling_typed, Res}), + ?assertEqual(7, hb_ao:get(<<"alpha-count">>, Res, undefined, Opts)), + ?assertEqual(42, hb_ao:get(<<"test-count">>, Res, undefined, Opts)), + ?assertEqual(true, hb_ao:get(<<"zebra-flag">>, Res, undefined, Opts)), + Signers = hb_message:signers(Res, Opts), + ?assert(length(Signers) == 1), + ?assert(hb_message:verify(Res, Signers, Opts)), + ?assert(hb_message:verify(Res, all, Opts)). + committed_id_test() -> Msg = #{ <<"basic">> => <<"value">> }, Opts = #{ priv_wallet => hb:wallet() }, diff --git a/src/dev_meta.erl b/src/dev_meta.erl index d48d77b0f..0dda4185e 100644 --- a/src/dev_meta.erl +++ b/src/dev_meta.erl @@ -315,7 +315,7 @@ embed_status({ErlStatus, Res}, NodeMsg) when is_map(Res) -> case lists:member(<<"status">>, hb_message:committed(Res, all, NodeMsg)) of false -> HTTPCode = status_code({ErlStatus, Res}, NodeMsg), - {ok, Res#{ <<"status">> => hb_util:bin(HTTPCode) }}; + {ok, Res#{ <<"status">> => HTTPCode }}; true -> {ok, Res} end; From 29c1908f78dd116ae6446ff816649ad31c9942f9 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 17 Apr 2026 02:14:15 -0400 Subject: [PATCH 18/20] fix: skip SNP NIF call when the mocked report path is enabled `dev_snp_nif:verify_signature/1` panics on hosts without AMD SEV-SNP hardware. When the test harness installs a mocked report via `mock_snp_nif/1', the integrity check short-circuits with `{ok, true}' rather than entering the NIF. Also handle the NIF's `{error, _}' return as a signature-invalid outcome instead of letting the tuple bleed out. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/dev_snp.erl | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/dev_snp.erl b/src/dev_snp.erl index 48ce38a2b..a74b17ee1 100644 --- a/src/dev_snp.erl +++ b/src/dev_snp.erl @@ -439,11 +439,23 @@ extract_measurement_args(Msg, NodeOpts) -> -spec verify_report_integrity(ReportJSON :: binary()) -> {ok, true} | {error, report_signature_invalid}. verify_report_integrity(ReportJSON) -> - {ok, ReportIsValid} = dev_snp_nif:verify_signature(ReportJSON), - ?event({report_is_valid, ReportIsValid}), - case ReportIsValid of - true -> {ok, true}; - false -> {error, report_signature_invalid} + case get(mock_snp_nif_enabled) of + true -> + % The test harness has installed a mock report; skip the NIF call + % (which will panic on hosts without AMD SEV-SNP hardware). + {ok, true}; + _ -> + case dev_snp_nif:verify_signature(ReportJSON) of + {ok, ReportIsValid} -> + ?event({report_is_valid, ReportIsValid}), + case ReportIsValid of + true -> {ok, true}; + false -> {error, report_signature_invalid} + end; + {error, Reason} -> + ?event({report_integrity_check_failed, Reason}), + {error, report_signature_invalid} + end end. %% @doc Check if the node's debug policy is enabled. From f973947dbd9e89bfc0978074af27d40465c2567b Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 17 Apr 2026 02:14:26 -0400 Subject: [PATCH 19/20] fix: bail from `ensure_started/1' cleanly when the sidecar dir is absent `dev_genesis_wasm:ensure_started/1' opens a port on the `genesis-wasm-server' sidecar's `launch-monitored.sh' script. On a development host that hasn't fetched the Node.js subproject (or a CI image that excludes it), `open_port' exits the spawned process with `enoent' and the caller proceeds into `hb_util:until/1', polling a socket that will never come up. The loop eventually resolves to a `failed_connect' error out of the polling loop, masking the true cause. Detect the missing sidecar directory up-front, log a warning, and return `false' so the caller reports the clean "not compiled" error rather than hanging or surfacing a misleading connect failure. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/dev_genesis_wasm.erl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/dev_genesis_wasm.erl b/src/dev_genesis_wasm.erl index 9e1a8e1cd..a520eb5fc 100644 --- a/src/dev_genesis_wasm.erl +++ b/src/dev_genesis_wasm.erl @@ -191,10 +191,22 @@ ensure_started(Opts) -> IsRunning = is_genesis_wasm_server_running(Opts), IsCompiled = hb_features:genesis_wasm(), GenWASMProc = is_pid(hb_name:lookup(<<"genesis-wasm@1.0">>)), + ServerDirExists = filelib:is_dir(GenesisWasmServerDir), case IsRunning orelse (IsCompiled andalso GenWASMProc) of true -> % If it is, do nothing. true; + false when not ServerDirExists -> + % The sidecar isn't present on disk (e.g. a development host that + % hasn't fetched the `genesis-wasm-server' Node subproject). Report + % as unavailable rather than spawning an open_port that will die + % with `enoent' and leaving the caller hanging in the start poll. + ?event( + warning, + {ensure_started, genesis_wasm_server_missing, + GenesisWasmServerDir} + ), + false; false -> % The device is not running, so we need to start it. PID = From 333a94bc622e66133b7d35de43b3801f69779c1e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 17 Apr 2026 02:39:41 -0400 Subject: [PATCH 20/20] fix: uncommit `test_process/1' base before merging the execution stack `test_process/1' passed the already-committed `base_process/1' message straight into `hb_maps:merge/3' before re-committing. When `dev_codec_httpsig:keys_to_commit/3' sees a message that already carries a commitment, it replicates that commitment's `committed' key list verbatim (the "stacked commitments" branch) rather than re-deriving the list from the top-level keys. The execution-device/device-stack keys merged in on top were therefore never added to the signed set, so `with_only_committed' stripped them out of the view seen by `dev_process_lib:run_as/4'. That in turn fell back to the execution device default (`genesis-wasm@1.0'), produced a `not compiled with genesis-wasm' error from `delegate_request/3', and left `test_device_compute_test/0' red on hosts that don't carry the sidecar. Strip the base commitment before the merge, matching the pattern already used by `wasm_process/2' and `aos_process/2' in the same module. The committed set now includes the merged keys and the test passes without ever touching the `genesis-wasm@1.0' path. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/dev_process_test_vectors.erl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dev_process_test_vectors.erl b/src/dev_process_test_vectors.erl index 98f90dac7..d1f9afa10 100644 --- a/src/dev_process_test_vectors.erl +++ b/src/dev_process_test_vectors.erl @@ -97,13 +97,21 @@ test_process() -> test_process(#{}). test_process(Opts) -> Wallet = hb:wallet(), + % Strip the base commitment before merging in the execution stack. A + % re-commit over a message that still carries a prior commitment will + % replicate that commitment's `committed' key list verbatim (see the + % "stacked commitments" branch of `keys_to_commit/3' in + % `dev_codec_httpsig'), leaving the newly-added `execution-device' + % and `device-stack' keys out of the signed set — and therefore out + % of the `with_only_committed' view used in compute. The + % `wasm_process'/`aos_process' helpers already follow this pattern. hb_message:commit( hb_maps:merge( - base_process(Opts), + hb_message:uncommitted(base_process(Opts), Opts), #{ <<"execution-device">> => <<"stack@1.0">>, <<"device-stack">> => [<<"test-device@1.0">>, <<"test-device@1.0">>] - }, + }, Opts ), Opts#{ priv_wallet => Wallet }