diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 000000000..d04c80095 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,105 @@ +**Updated Model** + +Yes: `#{}` is “vary on nothing except implicit keys”, and `#{ _ => _ }` is “vary on everything, preserving unmatched keys unloaded/uncoerced.” That gives us the simple all/nothing vocabulary we need. + +I also agree that `map()` should collapse to `_`: it means “no AO-specific shape constraint”, so the value is accepted as-is rather than treated as an empty closed message. AO message specs should use `#{...}` when they want projection semantics. + +**Spec Syntax** + +```erlang +-spec compute( + #{ already_seen => [integer()] }, + #{ slot := integer() }, + _ +) -> {ok, #{ results := _, _ => base }}. +``` + +Input rules: + +- `_` or `map()` means unchanged/any. +- `#{}` means no explicit dependency. +- `#{ key := Type }` means required, load and coerce. +- `#{ key => Type }` means optional, load and coerce if present. +- `#{ _ => _ }` means preserve all unmatched keys as part of the varied message. +- Base implicitly includes `device => _`. +- Request implicitly includes `path := _`. +- For `add_key` handlers, AO-Core overrides request `path` with the resolved key before varying. + +Return rules: + +- `#{ _ => base }` means cache the raw result, then `set` it over the original base. +- `#{ _ => request }` means cache the raw result, then `set` it over the original request. +- The return spec takes precedence. We should skip runtime markers for the first implementation. + +**Hashpath Rule** + +Hashpath should cover exactly the values that influence the final returned result: + +```erlang +none: + HashBase = VariedBase + HashReq = VariedReq + +_ => base: + HashBase = OldBase + HashReq = VariedReq + +_ => request: + HashBase = VariedBase + HashReq = OldReq +``` + +So your correction is right: if the result extends the original request, the full original request participates in the final hashpath. + +**AO-Core Flow** + +1. Stage 1 normalizes `OldBase/OldReq`. +2. Resolve key/device/function once, before cache lookup. +3. Vary using the resolved function spec. +4. Always run `hb_message:normalize_commitments(..., fast)` on `VariedBase/VariedReq`. +5. Cache lookup uses `VariedBase/VariedReq`. +6. Persistent grouping uses `VariedBase/VariedReq`. +7. Execute on `VariedBase/VariedReq`. +8. Normalize raw result, set its generic hashpath from `VariedBase/VariedReq`, and cache only this generic result. +9. Notify persistent waiters with the generic result, not the caller-finalized result. +10. Each caller finalizes locally: + - apply overlay with `hb_ao:set(OldBase | OldReq, Result, Opts#{ hashpath => ignore, ... })` + - normalize commitments fast + - set final hashpath using the rule above +11. Existing spawn-worker and stream continuation run with the finalized result. + +The point behind step 9 is the cache-hit concern I was gesturing at earlier: if two callers share `VariedBase/VariedReq` but have different original bases/requests for overlay, the leader cannot broadcast its finalized overlay result. It must broadcast the generic cached result, and each waiter applies its own overlay/hashpath. + +**Minimal Patch Strategy** + +I agree on avoiding a context map. To keep the diff tight, I’d use the existing `Opts` temp-state pattern already used for `add_key`. + +Minimal shape: + +- Add temp keys to `?TEMP_OPTS`, for example `resolved_func` and `vary`. +- In stage 2, do function lookup plus vary, then continue with `VariedBase/VariedReq`. +- Stage 5 first checks `resolved_func`; if present, it skips lookup. +- Stage 9 remains the generic hashpath step for cacheability. +- Stage 10 still caches, but only the generic varied result. +- Stage 11 notifies waiters with the generic result, then calls one small finalization helper. +- Cache hits and persistent waits also call that same finalization helper before returning. + +This keeps the change localized and avoids threading new arguments through every resolver stage. + +**Trade-Offs** + +Moving function lookup before cache means a missing/unloadable device can no longer be masked by an old exact cache hit. I think that is acceptable because variant caching cannot be sound without knowing the actual function. + +The only new complexity I think is unavoidable is “generic result versus finalized result.” We can minimize it by naming exactly one helper for finalization and by keeping cache, persistent notify, and wait behavior visibly tied to the generic result. + +**Tests I’d Add First** + +- `#{}` varies to implicit-only. +- `map()` and `_` leave message unchanged. +- `#{ _ => _ }` preserves extras without force-loading them. +- Return `#{ _ => base }` caches generic result and overlays current base. +- Return `#{ _ => request }` hashes with full original request. +- Persistent waiters sharing a varied execution finalize against their own original inputs. +- `add_key` handler uses resolved key as request `path`. + +If that matches your intent, I’m ready to start cutting the minimal patch. diff --git a/scripts/build-preloaded-store.escript b/scripts/build-preloaded-store.escript index 83a0ccca2..62ba90a08 100755 --- a/scripts/build-preloaded-store.escript +++ b/scripts/build-preloaded-store.escript @@ -90,6 +90,8 @@ hb_opts_compile_opts(Ebin) -> end. drop_outdir([{outdir, _} | Rest]) -> drop_outdir(Rest); +drop_outdir([{d, Name, Value} | Rest]) when is_list(Name) -> + [{d, list_to_atom(Name), Value} | drop_outdir(Rest)]; drop_outdir([Opt | Rest]) -> [Opt | drop_outdir(Rest)]; drop_outdir([]) -> []. diff --git a/scripts/hyper-token.lua b/scripts/hyper-token.lua index 237542a0a..233421724 100644 --- a/scripts/hyper-token.lua +++ b/scripts/hyper-token.lua @@ -119,11 +119,25 @@ function count_common(a, b) if type(a) ~= "table" then a = { a } end if type(b) ~= "table" then b = { b } end + -- local count = 0 + -- for _, v in ipairs(a) do + -- for _, w in ipairs(b) do + -- if v == w then + -- count = count + 1 + -- end + -- end + -- end + + local seen = {} local count = 0 for _, v in ipairs(a) do - for _, w in ipairs(b) do - if v == w then - count = count + 1 + if not seen[v] then + seen[v] = true + for _, w in ipairs(b) do + if v == w then + count = count + 1 + break + end end end end @@ -895,4 +909,4 @@ function compute(base, assignment) ao.event({ "Process initialized.", { slot = assignment.slot } }) return "ok", base end -end \ No newline at end of file +end diff --git a/src/core/device/hb_device_archive.erl b/src/core/device/hb_device_archive.erl index 350e4d36b..37fdf67b8 100644 --- a/src/core/device/hb_device_archive.erl +++ b/src/core/device/hb_device_archive.erl @@ -1,6 +1,6 @@ %%% @doc Helpers for packaged-device implementation archives. -module(hb_device_archive). --export([create/2, module_metadata/1, contents/1]). +-export([create/2, module_metadata/1, contents/1, object_code/1]). -export([load/1, load/4, load_modules/1, loaded/1]). -export([implementation_dir/1]). -export([modules_match_root/2, write_resources/2]). @@ -206,18 +206,35 @@ unsafe_resource_part(_) -> false. %% on-load devices fall back to ordinary Erlang loading semantics. load_modules(Modules) -> case code:atomic_load(Modules) of - ok -> ok; + ok -> remember_object_code(Modules); {error, Reason} -> case atomic_load_rejected_on_load(Reason) of - true -> load_modules_naturally(Modules); + true -> + case load_modules_naturally(Modules) of + ok -> remember_object_code(Modules); + Error -> Error + end; false -> case loaded(Modules) of - true -> ok; + true -> remember_object_code(Modules); false -> {error, Reason} end end end. +%% @doc Return the BEAM bytes for a module loaded from a device archive. +object_code(Module) -> + persistent_term:get({?MODULE, object_code, Module}, undefined). + +remember_object_code(Modules) -> + lists:foreach( + fun({Mod, _File, Beam}) -> + persistent_term:put({?MODULE, object_code, Mod}, Beam) + end, + Modules + ), + ok. + %% @doc Return true if `code:atomic_load/1' rejected module on-load callbacks. atomic_load_rejected_on_load(Reason) when is_list(Reason) -> lists:any( diff --git a/src/core/http/hb_client_remote.erl b/src/core/http/hb_client_remote.erl index 05c7e5df5..016dffbac 100644 --- a/src/core/http/hb_client_remote.erl +++ b/src/core/http/hb_client_remote.erl @@ -94,11 +94,25 @@ upload(Msg, Opts) -> UploadResults = lists:map( fun(Device) -> - upload(Msg, Opts, Device) + upload( + hb_message:with_commitments( + #{ <<"commitment-device">> => Device }, + Msg, + Opts + ), + Opts, + Device + ) end, hb_message:commitment_devices(Msg, Opts) ), - {ok, UploadResults}. + case lists:filter(fun upload_failed/1, UploadResults) of + [] -> {ok, UploadResults}; + Errors -> {error, Errors} + end. +upload_failed({error, _}) -> true; +upload_failed({failure, _}) -> true; +upload_failed(_) -> false. upload(Msg, Opts, <<"httpsig@1.0">>) -> case hb_opts:get(bundler_httpsig, not_found, Opts) of not_found -> @@ -107,13 +121,17 @@ upload(Msg, Opts, <<"httpsig@1.0">>) -> ?event({uploading_item, Msg}), hb_http:post(Bundler, <<"/tx">>, Msg, Opts) end; -upload(Msg, Opts, _CommitmentDevice) -> +upload(Msg, Opts, CommitmentDevice) -> ?event({uploading_item, Msg}), hb_ao:raw( <<"arweave@2.9">>, <<"tx">>, - #{}, - Msg#{ <<"method">> => <<"POST">> }, + Msg, + #{ + <<"method">> => <<"POST">>, + <<"target">> => <<"base">>, + <<"commitment-device">> => CommitmentDevice + }, Opts ). diff --git a/src/core/http/hb_http.erl b/src/core/http/hb_http.erl index 241d5e63a..82cb31dca 100644 --- a/src/core/http/hb_http.erl +++ b/src/core/http/hb_http.erl @@ -638,9 +638,15 @@ encode_reply(Status, TABMReq, Message, Opts) -> end, Opts ), + DefaultAcceptBundle = + case {Codec, hb_maps:get(<<"require-codec">>, TABMReq, not_found, Opts)} of + {<<"json@1.0">>, not_found} -> false; + {<<"json@1.0">>, _} -> true; + _ -> false + end, AcceptBundle = hb_util:atom( - hb_maps:get(<<"accept-bundle">>, TABMReq, false, Opts) + hb_maps:get(<<"accept-bundle">>, TABMReq, DefaultAcceptBundle, Opts) ), ?event(debug_http, {encoding_reply, @@ -1054,6 +1060,16 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> ), FilterKeys = hb_opts:get(http_inbound_filter_keys, ?DEFAULT_FILTER_KEYS, Opts), FilteredMsg = hb_message:without_unless_signed(FilterKeys, Msg, Opts), + DefaultAcceptBundle = + case maps:get( + <<"require-codec">>, + Msg, + maps:get(<<"require-codec">>, PrimMsg, not_found) + ) of + <<"application/json">> -> true; + <<"json@1.0">> -> true; + _ -> maps:get(<<"accept-bundle">>, RawHeaders, false) + end, BaseMsg = FilteredMsg#{ <<"method">> => Method, @@ -1065,7 +1081,7 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> maps:get( <<"accept-bundle">>, PrimMsg, - maps:get(<<"accept-bundle">>, RawHeaders, false) + DefaultAcceptBundle ) ), <<"accept">> => diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 7150504cf..3cc61355e 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -417,7 +417,46 @@ resolve_stage(1, RawBase, RawReq, Opts) -> Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), resolve_stage(2, Base, Req, Opts); -resolve_stage(2, Base, Req, Opts) -> +resolve_stage(2, RawBase, Req, Opts) -> + ?event(debug_ao_core, {stage, 2, prepare_vary}, Opts), + case maybe_direct_cache_lookup(RawBase, Req, Opts) of + continue -> + Base = anchor_loaded_base(RawBase, ensure_message_loaded(RawBase, Opts)), + case is_map(Base) andalso is_map(Req) of + false -> + legacy_cache_lookup(Base, Req, Opts); + true -> + case resolve_device_func(Base, Req, Opts) of + {ok, Func, AddKey, Device, Key} -> + UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), + {ok, VariedBase0, VariedReq0, Overlay} = + hb_types:vary(Device, Key, Func, AddKey, Base, Req, UserOpts), + VariedBase = normalize_varied(Base, VariedBase0, Opts), + VariedReq = normalize_varied(Req, VariedReq0, Opts), + resolve_stage( + 2, + Base, + Req, + VariedBase, + VariedReq, + Func, + AddKey, + Overlay, + Opts + ); + Other -> + Other + end + end; + Other -> + Other + end; +resolve_stage(3, Base, Req, Opts) when not is_map(Base) or not is_map(Req) -> + ?event(debug_ao_core, {stage, 3, validation_check_type_error}, Opts), + {error, not_found}; +resolve_stage(3, Base, Req, Opts) -> + resolve_stage(2, Base, Req, Opts). +resolve_stage(2, OldBase, OldReq, Base, Req, Func, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. % If we do not find a result, we continue to the next stage, @@ -426,17 +465,18 @@ resolve_stage(2, Base, Req, Opts) -> case hb_cache_control:maybe_lookup(Base, Req, Opts) of {ok, Res} -> ?event(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), - {ok, Res}; + finalize_result(OldBase, OldReq, Base, Req, {ok, Res}, Overlay, Opts); {continue, NewBase, NewReq} -> - resolve_stage(3, NewBase, NewReq, Opts); + resolve_stage(3, OldBase, OldReq, NewBase, NewReq, Func, AddKey, Overlay, Opts); {error, CacheResp} -> {error, CacheResp} end; -resolve_stage(3, Base, Req, Opts) when not is_map(Base) or not is_map(Req) -> +resolve_stage(3, _OldBase, _OldReq, Base, Req, _Func, _AddKey, _Overlay, Opts) + when not is_map(Base) or not is_map(Req) -> % Validation check: If the messages are not maps, we cannot find a key % in them, so return not_found. ?event(debug_ao_core, {stage, 3, validation_check_type_error}, Opts), {error, not_found}; -resolve_stage(3, Base, Req, Opts) -> +resolve_stage(3, OldBase, OldReq, Base, Req, Func, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 3, validation_check}, Opts), % Validation checks: If `paranoid_message_verification' is enabled, we should % verify the base and request messages prior to execution. @@ -449,8 +489,8 @@ resolve_stage(3, Base, Req, Opts) -> }, Opts ), - resolve_stage(4, Base, Req, Opts); -resolve_stage(4, Base, Req, Opts) -> + resolve_stage(4, OldBase, OldReq, Base, Req, Func, AddKey, Overlay, Opts); +resolve_stage(4, OldBase, OldReq, Base, Req, Func, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. @@ -466,7 +506,7 @@ resolve_stage(4, Base, Req, Opts) -> true -> ?event(worker_spawns, {will_become, ExecName}); _ -> ok end, - resolve_stage(5, Base, Req, ExecName, Opts); + resolve_stage(5, OldBase, OldReq, Base, Req, ExecName, Func, AddKey, Overlay, Opts); {wait, Leader} -> % There is another executor of this resolution in-flight. % Bail execution, register to receive the response, then @@ -484,11 +524,11 @@ resolve_stage(4, Base, Req, Opts) -> Opts ), % Re-try again if the group leader has died. - resolve_stage(4, Base, Req, Opts); + resolve_stage(4, OldBase, OldReq, Base, Req, Func, AddKey, Overlay, Opts); Res -> % Now that we have the result, we can skip right to potential % recursion (step 11) in the outer-wrapper. - Res + finalize_result(OldBase, OldReq, Base, Req, Res, Overlay, Opts) end; {infinite_recursion, GroupName} -> % We are the leader for this resolution, but we executing the @@ -507,88 +547,24 @@ resolve_stage(4, Base, Req, Opts) -> case hb_opts:get(allow_infinite, false, Opts) of true -> % We are OK with infinite loops, so we just continue. - resolve_stage(5, Base, Req, GroupName, Opts); + resolve_stage(5, OldBase, OldReq, Base, Req, GroupName, Func, AddKey, Overlay, Opts); false -> % We are not OK with infinite loops, so we raise an error. error_infinite(Base, Req, Opts) end end. -resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event(debug_ao_core, {stage, 5, device_lookup}, Opts), - % Device lookup: Find the Erlang function that should be utilized to - % execute Req on Base. - {ResolvedFunc, NewOpts} = - try - UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), - Key = hb_path:hd(Req, UserOpts), - % Try to load the device and get the function to call. - ?event( - { - resolving_key, - {key, Key}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), - ?event( - {found_func_for_exec, - {key, Key}, - {device, Device}, - {func, Func}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. - { - Func, - Opts#{ - <<"add-key">> => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - ?event( - ao_result, - { - load_device_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {exec_class, Class}, - {exec_exception, Exception}, - {exec_stacktrace, Stacktrace}, - {opts, Opts} - }, - Opts - ), - % If the device cannot be loaded, we alert the caller. - error_execution( - ExecName, - Req, - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). -resolve_stage(6, Func, Base, Req, ExecName, Opts) -> +resolve_stage(5, OldBase, OldReq, Base, Req, ExecName, Func, AddKey, Overlay, Opts) -> + ?event(debug_ao_core, {stage, 5, pre_execution}, Opts), + resolve_stage(6, Func, OldBase, OldReq, Base, Req, ExecName, AddKey, Overlay, Opts); +resolve_stage(6, Func, OldBase, OldReq, Base, Req, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 6, ExecName, execution}, Opts), % Execution. - ExecOpts = execution_opts(Opts), - Args = - case hb_opts:get(add_key, false, Opts) of - false -> [Base, Req, ExecOpts]; - Key -> [Key, Base, Req, ExecOpts] - end, + ExecOpts = execution_opts(Opts), + Args = + case AddKey of + false -> [Base, Req, ExecOpts]; + Key -> [Key, Base, Req, ExecOpts] + end, % Try to execute the function. Res = try @@ -648,13 +624,17 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> }, Opts ), - resolve_stage(7, Base, Req, Res, ExecName, Opts); + resolve_stage(7, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts); resolve_stage( 7, + OldBase, + OldReq, Base, Req, {St, Res}, ExecName, + AddKey, + Overlay, Opts = #{ <<"on">> := On = #{ <<"step">> := _ }} ) -> ?event(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), @@ -670,7 +650,7 @@ resolve_stage( }, case hb_hook:on(<<"step">>, HookReq, Opts) of {ok, #{ <<"status">> := NewStatus, <<"body">> := NewRes }} -> - resolve_stage(8, Base, Req, {NewStatus, NewRes}, ExecName, Opts); + resolve_stage(8, OldBase, OldReq, Base, Req, {NewStatus, NewRes}, ExecName, AddKey, Overlay, Opts); Error -> ?event( ao_core, @@ -682,75 +662,69 @@ resolve_stage( ), Error end; -resolve_stage(7, Base, Req, Res, ExecName, Opts) -> +resolve_stage(7, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), - resolve_stage(8, Base, Req, Res, ExecName, Opts); -resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> + resolve_stage(8, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts); +resolve_stage(8, OldBase, OldReq, Base, Req, {ok, {resolve, Sublist}}, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), % If the result is a `{resolve, Sublist}' tuple, we need to execute it % as a sub-resolution. - resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); -resolve_stage(8, Base, Req, Res, ExecName, Opts) -> + resolve_stage(9, OldBase, OldReq, Base, Req, resolve_many(Sublist, Opts), ExecName, AddKey, Overlay, Opts); +resolve_stage(8, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), - resolve_stage(9, Base, Req, Res, ExecName, Opts); -resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> + resolve_stage(9, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts); +resolve_stage(9, OldBase, OldReq, Base, Req, {ok, Res}, ExecName, AddKey, Overlay, Opts) when is_map(Res) -> ?event(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), % Cryptographic linking. Now that we have generated the result, we % need to cryptographically link the output to its input via a hashpath. - resolve_stage(10, Base, Req, - case hb_opts:get(hashpath, update, Opts#{ <<"only">> => local }) of - update -> - NormRes = Res, - Priv = hb_private:from_message(NormRes), - HP = hb_path:hashpath(Base, Req, Opts), - if not is_binary(HP) or not is_map(Priv) -> - throw({invalid_hashpath, {hp, HP}, {res, NormRes}}); - true -> - {ok, NormRes#{ <<"priv">> => Priv#{ <<"hashpath">> => HP } }} - end; - reset -> - Priv = hb_private:from_message(Res), - {ok, Res#{ <<"priv">> => hb_maps:without([<<"hashpath">>], Priv, Opts) }}; - ignore -> - Priv = hb_private:from_message(Res), - if not is_map(Priv) -> - throw({invalid_private_message, {res, Res}}); - true -> - {ok, Res} - end - end, + resolve_stage(10, OldBase, OldReq, Base, Req, + update_hashpath(Base, Req, strip_overlay_marker(Overlay, Res, Opts), Opts), ExecName, + AddKey, + Overlay, Opts ); -resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> +resolve_stage(9, OldBase, OldReq, Base, Req, {Status, Res}, ExecName, AddKey, Overlay, Opts) when is_map(Res) -> ?event(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), % Skip cryptographic linking and reset the hashpath if the result is abnormal. Priv = hb_private:from_message(Res), resolve_stage( - 10, Base, Req, + 10, OldBase, OldReq, Base, Req, {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, - ExecName, Opts); -resolve_stage(9, Base, Req, Res, ExecName, Opts) -> + ExecName, AddKey, Overlay, Opts); +resolve_stage(9, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), % Skip cryptographic linking and continue if we don't have a map that can have % a hashpath at all. - resolve_stage(10, Base, Req, Res, ExecName, Opts); -resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> + resolve_stage(10, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts); +resolve_stage(10, OldBase, OldReq, Base, Req, {ok, Res}, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), % Result caching: Optionally, cache the result of the computation locally. - hb_cache_control:maybe_store(Base, Req, Res, Opts), - resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); -resolve_stage(10, Base, Req, Res, ExecName, Opts) -> + hb_cache_control:maybe_store( + Base, + Req, + Res, + cache_store_opts(OldBase, OldReq, Base, Req, Overlay, Opts) + ), + resolve_stage(11, OldBase, OldReq, Base, Req, {ok, Res}, ExecName, AddKey, Overlay, Opts); +resolve_stage(10, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), % Skip result caching if the result is abnormal. - resolve_stage(11, Base, Req, Res, ExecName, Opts); -resolve_stage(11, Base, Req, Res, ExecName, Opts) -> + resolve_stage(11, OldBase, OldReq, Base, Req, Res, ExecName, AddKey, Overlay, Opts); +resolve_stage(11, OldBase, OldReq, Base, Req, Res, ExecName, _AddKey, Overlay, Opts) -> ?event(debug_ao_core, {stage, 11, ExecName}, Opts), % Notify processes that requested the resolution while we were executing and % unregister ourselves from the group. hb_persistent:unregister_notify(ExecName, Req, Res, Opts), - resolve_stage(12, Base, Req, Res, ExecName, Opts); + resolve_stage( + 12, + Base, + Req, + finalize_result(OldBase, OldReq, Base, Req, Res, Overlay, Opts), + ExecName, + Opts + ). resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> ?event(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution @@ -769,6 +743,144 @@ resolve_stage(12, _Base, _Req, OtherRes, ExecName, Opts) -> ?event(debug_ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), OtherRes. +legacy_cache_lookup(Base, Req, Opts) -> + case hb_cache_control:maybe_lookup(Base, Req, Opts) of + {ok, Res} -> {ok, Res}; + {continue, NewBase, NewReq} -> resolve_stage(3, NewBase, NewReq, Opts); + {error, CacheResp} -> {error, CacheResp} + end. + +maybe_direct_cache_lookup(Base, Req, Opts) when ?IS_ID(Base), is_map(Req) -> + Store = hb_opts:get(store, no_viable_store, Opts), + case hb_device:is_direct_key_access(Base, Req, Opts, Store) of + true -> + case hb_cache_control:maybe_lookup(Base, Req, Opts) of + {ok, Res} -> {ok, Res}; + {error, CacheResp} -> {error, CacheResp}; + {continue, _, _} -> continue + end; + _ -> + continue + end; +maybe_direct_cache_lookup(_Base, _Req, _Opts) -> + continue. + +resolve_device_func(Base, Req, Opts) -> + try + UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), + Key = hb_path:hd(Req, UserOpts), + ?event( + { + resolving_key, + {key, Key}, + {base, Base}, + {req, Req}, + {opts, Opts} + } + ), + {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), + ?event( + {found_func_for_exec, + {key, Key}, + {device, Device}, + {func, Func}, + {base, Base}, + {req, Req}, + {opts, Opts} + } + ), + AddKey = + case Status of + add_key -> Key; + _ -> false + end, + {ok, Func, AddKey, Device, Key} + catch + Class:Exception:Stacktrace -> + ?event( + ao_result, + { + load_device_failed, + {base, Base}, + {req, Req}, + {exec_class, Class}, + {exec_exception, Exception}, + {exec_stacktrace, Stacktrace}, + {opts, Opts} + }, + Opts + ), + error_execution( + ungrouped_exec, + Req, + loading_device, + {Class, Exception, Stacktrace}, + Opts + ) + end. + +update_hashpath(Base, Req, Res, Opts) -> + case hb_opts:get(hashpath, update, Opts#{ <<"only">> => local }) of + update -> + Priv = hb_private:from_message(Res), + HP = hb_path:hashpath(Base, Req, Opts), + if not is_binary(HP) or not is_map(Priv) -> + throw({invalid_hashpath, {hp, HP}, {res, Res}}); + true -> + {ok, Res#{ <<"priv">> => Priv#{ <<"hashpath">> => HP } }} + end; + reset -> + Priv = hb_private:from_message(Res), + {ok, Res#{ <<"priv">> => hb_maps:without([<<"hashpath">>], Priv, Opts) }}; + ignore -> + Priv = hb_private:from_message(Res), + if not is_map(Priv) -> + throw({invalid_private_message, {res, Res}}); + true -> + {ok, Res} + end + end. + +normalize_varied(Original, Original, _Opts) -> + Original; +normalize_varied(_Original, Varied, Opts) -> + hb_message:normalize_commitments(Varied, normalize_opts(Opts), fast). + +normalize_opts(Opts) when is_map(Opts) -> + Opts; +normalize_opts(_Opts) -> + #{}. + +cache_store_opts(OldBase, OldReq, Base, Req, Overlay, Opts) -> + case (OldBase =/= Base) orelse (OldReq =/= Req) orelse (Overlay =/= none) of + true -> Opts#{ cache_hashpath_maps => true }; + false -> Opts + end. + +strip_overlay_marker(none, Res, _Opts) -> + Res; +strip_overlay_marker(_Overlay, Res, Opts) -> + hb_maps:without([<<"_">>, <<"...">>], Res, Opts). + +finalize_result(_OldBase, _OldReq, _Base, _Req, Res, none, _Opts) -> + Res; +finalize_result(OldBase, OldReq, Base, Req, {ok, Res}, Overlay, Opts) when is_map(Res) -> + Patch = strip_overlay_marker(Overlay, Res, Opts), + {OverlayBase, HashBase, HashReq} = + case Overlay of + base -> {OldBase, OldBase, Req}; + request -> {OldReq, Base, OldReq} + end, + Merged = set(OverlayBase, Patch, internal_opts(Opts)), + update_hashpath( + HashBase, + HashReq, + hb_message:normalize_commitments(Merged, Opts, fast), + Opts + ); +finalize_result(_OldBase, _OldReq, _Base, _Req, Res, _Overlay, _Opts) -> + Res. + %% @doc Execute a sub-resolution. subresolve(RawBase, DevID, ReqPath, Opts) when is_binary(ReqPath) -> % If the request is a binary, we assume that it is a path. @@ -918,6 +1030,15 @@ ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> ensure_message_loaded(Msg, _Opts) -> Msg. +anchor_loaded_base(RawBase, Base) when ?IS_ID(RawBase), is_map(Base) -> + Priv = hb_private:from_message(Base), + case maps:is_key(<<"hashpath">>, Priv) of + true -> Base; + false -> Base#{ <<"priv">> => Priv#{ <<"hashpath">> => RawBase }} + end; +anchor_loaded_base(_RawBase, Base) -> + Base. + %% @doc Catch all return if we are in an infinite loop. error_infinite(Base, Req, Opts) -> ?event( diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index 082ae060f..f9a3df5d8 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -40,7 +40,8 @@ -module(hb_cache). -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). --export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). +-export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2]). +-export([write_result/3, write_result/4, link_result/4, link/3]). -export([match/2, list/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). -include("include/hb.hrl"). @@ -512,9 +513,60 @@ write_hashpath(HP, Msg, Opts) when is_binary(HP) or is_list(HP) -> Store = hb_opts:get(store, no_viable_store, Opts), ?event({writing_hashpath, {hashpath, HP}, {msg, Msg}, {store, Store}}), {ok, Path} = write(Msg, Opts), - hb_store:link(Store, #{ hb_path:to_binary(HP) => Path }, Opts), + HPBin = hb_path:to_binary(HP), + LinkReq = + case hb_store:resolve(Store, HPBin, Opts) of + {ok, HPBin} -> #{ HPBin => Path }; + {ok, ResolvedHP} -> #{ HPBin => Path, ResolvedHP => Path }; + _ -> #{ HPBin => Path } + end, + ok = hb_store:link(Store, LinkReq, Opts), + {ok, Path}. + +%% @doc Write a result once, then link one or more `{Base, Req}' edges to it. +write_result(Base, Req, Res, Opts) -> + write_result([{Base, Req}], Res, Opts). +write_result(Edges, Res, Opts) when is_list(Edges) -> + {ok, Path} = write(Res, Opts), + lists:foreach( + fun({EdgeBase, EdgeReq}) -> + maybe_write_edge_part(EdgeBase, Opts), + maybe_write_edge_part(EdgeReq, Opts), + ok = link_result(EdgeBase, EdgeReq, Path, Opts) + end, + Edges + ), {ok, Path}. +maybe_write_edge_part(Msg, Opts) when is_map(Msg); is_list(Msg) -> + {ok, _} = write(Msg, Opts), + ok; +maybe_write_edge_part(_Ref, _Opts) -> + ok. + +%% @doc Link a `{Base, Req}' result edge to an existing stored result path. +link_result(Base, Req, Existing, Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + EdgePath = result_edge_path(Base, Req, Opts), + ExistingPath = hb_path:to_binary(Existing), + hb_store:link(Store, #{ EdgePath => ExistingPath }, Opts). + +result_edge_path(BaseID, ReqID, Opts) when ?IS_ID(BaseID) and ?IS_ID(ReqID) -> + result_edge_path_from_id(BaseID, ReqID, Opts); +result_edge_path(BaseID, Req, Opts) when ?IS_ID(BaseID) and is_map(Req) -> + result_hashpath(BaseID, Req, Opts); +result_edge_path(BaseID, Key, Opts) when ?IS_ID(BaseID) and is_binary(Key) -> + result_edge_path_from_id(BaseID, hb_ao:normalize_key(Key, Opts), Opts); +result_edge_path(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> + hb_path:hashpath(BaseMsg, Req, Opts). + +result_hashpath(BaseID, Req, Opts) when ?IS_ID(BaseID) and is_map(Req) -> + ReqID = hb_message:id(Req, #{ <<"committers">> => <<"all">> }, Opts), + result_edge_path_from_id(BaseID, ReqID, Opts). + +result_edge_path_from_id(BaseID, Suffix, _Opts) -> + hb_path:to_binary([<<"ao-results">>, BaseID, Suffix]). + %% @doc Write a raw binary keys into the store and link it at a given hashpath. write_binary(Hashpath, Bin, Opts) -> write_binary(Hashpath, Bin, hb_opts:get(store, no_viable_store, Opts), Opts). @@ -904,14 +956,17 @@ read_in_memory_key(BaseMsg, NormKey, _Opts) -> %% @doc Read the output of a prior computation, given BaseMsg and Req. read_hashpath(BaseMsgID, ReqID, Opts) when ?IS_ID(BaseMsgID) and ?IS_ID(ReqID) -> ?event({cache_lookup, {base, BaseMsgID}, {req, ReqID}, {opts, Opts}}), - hashpath_read_result(read(<>, Opts)); + read_result_edge(BaseMsgID, ReqID, Opts); read_hashpath(BaseMsgID, Req, Opts) when ?IS_ID(BaseMsgID) and is_map(Req) -> - ReqID = hb_message:id(Req, all, Opts), - hashpath_read_result(read(<>, Opts)); + ReqID = hb_message:id(Req, #{ <<"committers">> => <<"all">> }, Opts), + read_result_edge(BaseMsgID, ReqID, Opts); read_hashpath(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> hashpath_read_result(read(hb_path:hashpath(BaseMsg, Req, Opts), Opts)); read_hashpath(_, _, _) -> miss. +read_result_edge(BaseID, ReqID, Opts) -> + hashpath_read_result(read(result_edge_path_from_id(BaseID, ReqID, Opts), Opts)). + hashpath_read_result({ok, Msg}) -> {hit, {ok, Msg}}; hashpath_read_result({error, not_found}) -> miss; hashpath_read_result(Other) -> {hit, Other}. @@ -1228,6 +1283,30 @@ test_raw_match_read(Store) -> hb_maps:get(<<"body">>, RawMsg, undefined, RawOpts) ). +test_write_result_edges(Store) -> + hb_store:reset(Store), + Opts = #{ <<"store">> => Store }, + Base = #{ <<"device">> => <<"process@1.0">>, <<"kind">> => <<"test">> }, + {ok, BaseID} = write(Base, Opts), + SlotReq = #{ <<"path">> => <<"compute">>, <<"slot">> => 7 }, + LatestReq = #{ <<"path">> => <<"latest">> }, + Res = #{ <<"device">> => <<"process@1.0">>, <<"at-slot">> => 7 }, + {ok, WrittenID} = write_result([{BaseID, SlotReq}, {BaseID, LatestReq}], Res, Opts), + SlotReqID = hb_message:id(SlotReq, #{ <<"committers">> => <<"all">> }, Opts), + {ok, ReadBase} = read(BaseID, Opts), + ?assertEqual(false, maps:is_key(SlotReqID, ReadBase)), + {ok, WrittenRes} = read(WrittenID, Opts), + ?assert(hb_message:match(Res, WrittenRes, strict, Opts)), + {hit, {ok, SlotRes}} = read_resolved(BaseID, SlotReq, Opts), + ?assert(hb_message:match(Res, SlotRes, strict, Opts)), + {hit, {ok, LatestRes}} = read_resolved(BaseID, LatestReq, Opts), + ?assert(hb_message:match(Res, LatestRes, strict, Opts)), + NextReq = #{ <<"path">> => <<"compute">>, <<"slot">> => 8 }, + NextRes = Res#{ <<"at-slot">> := 8 }, + {ok, _} = write_result(BaseID, NextReq, NextRes, Opts), + {hit, {ok, NextSlotRes}} = read_resolved(BaseID, NextReq, Opts), + ?assert(hb_message:match(NextRes, NextSlotRes, strict, Opts)). + cache_suite_test_() -> hb_store:generate_test_suite([ {"store unsigned empty message", @@ -1242,7 +1321,8 @@ cache_suite_test_() -> {"match message", fun test_match_message/1}, {"match linked message", fun test_match_linked_message/1}, {"match typed message", fun test_match_typed_message/1}, - {"raw match read", fun test_raw_match_read/1} + {"raw match read", fun test_raw_match_read/1}, + {"write result edges", fun test_write_result_edges/1} ]). %% @doc Test that message whose device is `#{}' cannot be written. If it were to diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 85746bf0d..882abc643 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -56,7 +56,14 @@ lookup(Base, Req, Opts) -> Opts, hb_opts:get(store_scope_resolved, local, Opts) ), - case hb_cache:read_resolved(Base, Req, OutputScopedOpts) of + CacheRead = + try hb_cache:read_resolved(Base, Req, OutputScopedOpts) of + ReadRes -> ReadRes + catch + throw:{necessary_message_not_found, _, _} -> + miss + end, + case CacheRead of {hit, not_found} -> {error, not_found}; {hit, {ok, Res}} -> @@ -142,7 +149,10 @@ perform_cache_write(Base, Req, Res, Opts) -> Opts ); Map when is_map(Map) -> - hb_cache:write(Res, Opts); + case hb_opts:get(cache_hashpath_maps, false, Opts) of + true -> hb_cache:write_hashpath(Map, Opts); + false -> hb_cache:write(Res, Opts) + end; _ -> ?event({cannot_write_result, Res}), skip_caching @@ -423,4 +433,4 @@ cache_message_result_test() -> {ok, Res3} = hb_ao:resolve(Base, Req, #{ <<"cache-control">> => [<<"only-if-cached">>] }), ?event({res2, Res2}), ?event({res3, Res3}), - ?assertEqual(Res2, Res3). \ No newline at end of file + ?assertEqual(Res2, Res3). diff --git a/src/core/resolver/hb_hook.erl b/src/core/resolver/hb_hook.erl index b9a77009d..f5d7b175a 100644 --- a/src/core/resolver/hb_hook.erl +++ b/src/core/resolver/hb_hook.erl @@ -57,6 +57,8 @@ %% @doc Execute a named hook with the provided request and options %% This function finds all handlers for the hook and evaluates them in sequence. %% The result of each handler is used as input to the next handler. +-spec on(binary() | atom(), #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. on(HookName, Req, Opts) -> ?event(hook, {attempting_execution_for_hook, HookName}), % Get all handlers for this hook from the options @@ -73,9 +75,14 @@ on(HookName, Req, Opts) -> end. %% @doc Get all handlers for a specific hook from the node message options. -%% Handlers are stored in the `on' key of this message. +%% Handlers are stored in the `on' key of this message. The `find/2' variant of +%% this function only takes a hook name and node message, and is not called +%% directly via the device API. Instead it is used by `on/3' and other internal +%% functionality to find handlers when necessary. The `find/3' variant can, +%% however, be called directly via the device API. find(HookName, Opts) -> find(#{}, #{ <<"target">> => <<"body">>, <<"body">> => HookName }, Opts). +-spec find(#{ _ => _ }, #{ _ => _ }, map()) -> term(). find(_Base, Req, Opts) -> HookName = maps:get(maps:get(<<"target">>, Req, <<"body">>), Req), case maps:get(HookName, hb_opts:get(on, #{}, Opts), []) of @@ -193,7 +200,10 @@ execute_handler(HookName, Handler, Req, Opts) -> hb_ao:raw( PreparedBase, PreparedReq, - Opts#{ <<"hashpath">> => ignore } + Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] + } ), ?event(hook, {handler_result, diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl index 4d89c1dda..5c9bf763f 100644 --- a/src/core/resolver/hb_opts.erl +++ b/src/core/resolver/hb_opts.erl @@ -531,7 +531,7 @@ raw_default_message() -> } ] }, - <<"scheduler-default-commitment-spec">> => <<"httpsig@1.0">>, + <<"scheduler-default-commitment-spec">> => <<"ans104@1.0">>, <<"genesis-wasm-import-authorities">> => [ <<"WjnS-s03HWsDSdMnyTdzB1eHZB2QheUWP_FVRVYxkXk">> diff --git a/src/core/util/hb_util.erl b/src/core/util/hb_util.erl index 3e2349945..6b31c848b 100644 --- a/src/core/util/hb_util.erl +++ b/src/core/util/hb_util.erl @@ -5,7 +5,7 @@ -export([ceil_int/2, floor_int/2]). -export([id/1, id/2, native_id/1, human_id/1, human_int/1, to_hex/1]). -export([secret_key_to_committer/1, remove_scheme_prefix/1]). --export([key_to_atom/1, key_to_atom/2, binary_to_strings/1]). +-export([atom_to_key/1, key_to_atom/1, key_to_atom/2, binary_to_strings/1]). -export([encode/1, decode/1, decode/2, safe_encode/1, safe_decode/1]). -export([is_printable_string/1]). -export([find_value/2, find_value/3]). @@ -234,6 +234,10 @@ to_sorted_keys(Msg, Opts) when is_map(Msg) -> to_sorted_keys(Msg, _Opts) when is_list(Msg) -> lists:sort(fun(Key1, Key2) -> Key1 < Key2 end, Msg). +%% @doc Convert an atom to its `binary-dashed-key`-equivalent form. +atom_to_key(Atom) -> + binary:replace(hb_util:bin(Atom), <<"_">>, <<"-">>, [global]). + %% @doc Convert keys in a map to atoms, lowering `-' to `_'. key_to_atom(Key) -> key_to_atom(Key, existing). key_to_atom(Key, _Mode) when is_atom(Key) -> Key; diff --git a/src/hb_types.erl b/src/hb_types.erl new file mode 100644 index 000000000..59ca4d30d --- /dev/null +++ b/src/hb_types.erl @@ -0,0 +1,1159 @@ +%%% @doc Extract Dialyzer-style type information from AO-Core devices and apply +%%% a static `vary` transform to base and request messages. +-module(hb_types). +-export([extract/2, vary/5, vary/7]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-define(EXTRACT_CACHE_TAG, {hb_types, extract, 2}). +-define(EXTRACT_CACHE_MISS, '$hb_types_extract_cache_miss'). + +%% @doc Apply a device's declared base/request schemas to the messages that will +%% participate in one AO-Core key execution. If no schema is provided, we return +%% the messages unchanged. +vary(Device, Key, Base, Request, Opts) -> + case extract(Device, Opts) of + {ok, #{ <<"keys">> := KeySchemas }} -> + case maps:get(normalize_name(Key), KeySchemas, undefined) of + undefined -> + {ok, Base, Request}; + Schema -> + ?event({apply_schema, {schema, Schema}, {base, Base}, {request, Request}}), + {ok, + apply_schema( + maps:get(<<"base">>, Schema, any_type()), + Base, + Opts + ), + apply_schema( + maps:get(<<"request">>, Schema, any_type()), + Request, + Opts + ) + } + end; + {error, _Reason} -> + {ok, Base, Request} + end. + +%% @doc Apply the schema for a resolved device function. This is the AO-Core +%% entrypoint: the resolver has already mapped a key to its Erlang function. +vary(Device, Key, Func, AddKey, Base, Request, Opts) -> + case function_schema(Device, Func, Key, Opts) of + undefined -> + {ok, Base, Request, none}; + Schema -> + {BaseSchema, ReqSchema, ReturnSchema} = + execution_schemas(Schema, AddKey), + Req = + case AddKey of + false -> Request; + _ -> Request#{ <<"path">> => Key } + end, + {ok, + apply_schema(implicit_base(BaseSchema), Base, Opts), + apply_schema(implicit_request(ReqSchema), Req, Opts), + overlay(ReturnSchema) + } + end. + +%% @doc Extract the public type schema for a device. +extract(Device, _Opts) when is_map(Device) -> + {error, {unsupported_device_type, Device}}; +extract(Module, Opts) when is_atom(Module) -> + case code:ensure_loaded(Module) of + {module, Module} -> + cached_extract(Module, Opts); + {error, Reason} -> + {error, {module_not_loaded, Module, Reason}} + end; +extract(Device, Opts) when is_binary(Device) -> + case hb_device_load:reference(Device, Opts) of + {ok, Module} -> extract(Module, Opts); + Error -> Error + end; +extract(Device, _Opts) -> + {error, {unsupported_device_type, Device}}. + +cached_extract(Module, Opts) -> + CacheKey = {?EXTRACT_CACHE_TAG, Module, Module:module_info(md5)}, + case persistent_term:get(CacheKey, ?EXTRACT_CACHE_MISS) of + ?EXTRACT_CACHE_MISS -> + Path = extract_cache_path(Module), + Res = + case read_cached_extract(Path, Opts) of + {ok, Cached} -> Cached; + miss -> + Extracted = do_extract(Module), + write_cached_extract(Path, Extracted, Opts), + Extracted + end, + persistent_term:put(CacheKey, Res), + Res; + Res -> + Res + end. + +extract_cache_path(Module) -> + ModuleBin = atom_to_binary(Module, utf8), + BeamHash = + case code:get_object_code(Module) of + {Module, Beam, _Filename} -> hb_util:encode(hb_crypto:sha256(Beam)); + _ -> hb_util:encode(Module:module_info(md5)) + end, + hb_path:to_binary([<<"ao-core">>, <<"device-", ModuleBin/binary>>, BeamHash]). + +read_cached_extract(Path, Opts) -> + try hb_store:read(Path, hb_store:scope(Opts, local)) of + {ok, Bin} -> + case binary_to_term(Bin, [safe]) of + {?EXTRACT_CACHE_TAG, Res} -> {ok, Res}; + _ -> miss + end; + _ -> + miss + catch _:_ -> + miss + end. + +write_cached_extract(Path, Res = {ok, _}, Opts) -> + try hb_store:write(#{ Path => term_to_binary({?EXTRACT_CACHE_TAG, Res}) }, + hb_store:scope(Opts, local)) of + _ -> ok + catch _:_ -> ok + end; +write_cached_extract(_Path, _Res, _Opts) -> + ok. + +do_extract(Module) -> + case beam_lib:chunks(module_beam(Module), [abstract_code]) of + {ok, {_, [{abstract_code, {_, Forms}}]}} -> + TypeEnv = build_type_env(Forms), + Specs = [ Attr || Attr = {attribute, _, spec, _} <- Forms ], + KeySchemas = + lists:foldl( + fun(Spec, Acc) -> + case spec_to_schema(Spec, TypeEnv) of + false -> Acc; + {Key, Schema} -> store_schema(Key, Schema, Acc) + end + end, + #{}, + Specs + ), + {ok, + #{ + <<"module">> => hb_util:bin(atom_to_binary(Module, utf8)), + <<"keys">> => KeySchemas, + <<"types">> => export_type_env(TypeEnv) + } + }; + Error -> + {error, {abstract_code_unavailable, Module, Error}} + end. + +module_beam(Module) -> + case code:get_object_code(Module) of + {Module, Binary, _Filename} -> + Binary; + _ -> + case hb_device_archive:object_code(Module) of + undefined -> code:which(Module); + Binary -> Binary + end + end. + +build_type_env(Forms) -> + lists:foldl( + fun + ({attribute, _, Tag, {Name, Ast, Vars}}, Acc) + when Tag =:= type; Tag =:= opaque -> + Acc#{ + Name => + #{ + vars => [var_name(Var) || Var <- Vars], + ast => Ast + } + }; + (_, Acc) -> + Acc + end, + #{}, + Forms + ). + +export_type_env(TypeEnv) -> + maps:from_list( + lists:map( + fun({Name, #{ ast := Ast, vars := Vars }}) -> + { + normalize_name(Name), + #{ + <<"kind">> => <<"alias">>, + <<"name">> => normalize_name(Name), + <<"vars">> => [normalize_name(Var) || Var <- Vars], + <<"type">> => parse_type(Ast, TypeEnv, #{}, [Name]) + } + } + end, + maps:to_list(TypeEnv) + ) + ). + +spec_to_schema({attribute, _, spec, {{Name, Arity}, [Spec]}}, TypeEnv) -> + {Args, Return} = parse_fun_spec(Spec, TypeEnv), + { + normalize_name(Name), + #{ + <<"arity">> => Arity, + <<"args">> => Args, + <<"base">> => maybe_nth(1, Args, any_type()), + <<"request">> => maybe_nth(2, Args, any_type()), + <<"opts">> => maybe_nth(3, Args, any_type()), + <<"return">> => Return + } + }; +spec_to_schema(_, _) -> + false. + +maybe_nth(N, List, Default) -> + case catch lists:nth(N, List) of + {'EXIT', _} -> Default; + Value -> Value + end. + +store_schema(Key, Schema, Acc) -> + case maps:get(Key, Acc, undefined) of + undefined -> + Acc#{ Key => Schema }; + Existing -> + ExistingArity = maps:get(<<"arity">>, Existing), + SchemaArity = maps:get(<<"arity">>, Schema), + Overloads0 = + maps:get( + <<"overloads">>, + Existing, + #{ ExistingArity => maps:without([<<"overloads">>], Existing) } + ), + Acc#{ + Key => + Schema#{ + <<"overloads">> => + Overloads0#{ + SchemaArity => maps:without([<<"overloads">>], Schema) + } + } + } + end. + +function_schema(Device, Func, Key, Opts) -> + case extract(Device, Opts) of + {ok, #{ <<"keys">> := KeySchemas }} -> + case function_schema(Func, Key, KeySchemas) of + undefined -> function_module_schema(Device, Func, Key, Opts); + Schema -> Schema + end; + {error, _Reason} -> + function_module_schema(Device, Func, Key, Opts) + end. + +function_module_schema(Device, Func, Key, Opts) -> + case erlang:fun_info(Func, module) of + {module, Device} -> + undefined; + {module, Module} -> + case extract(Module, Opts) of + {ok, #{ <<"keys">> := KeySchemas }} -> + function_schema(Func, Key, KeySchemas); + {error, _Reason} -> + undefined + end; + _ -> + undefined + end. + +function_schema(Func, Key, KeySchemas) -> + {arity, Arity} = erlang:fun_info(Func, arity), + FuncSchema = + case erlang:fun_info(Func, name) of + {name, Name} -> named_schema(Name, Arity, KeySchemas); + _ -> undefined + end, + case FuncSchema of + undefined -> named_schema(Key, Arity, KeySchemas); + Schema -> Schema + end. + +named_schema(Name, Arity, KeySchemas) -> + case maps:get(normalize_name(Name), KeySchemas, undefined) of + undefined -> + undefined; + #{ <<"overloads">> := Overloads } -> + maps:get(Arity, Overloads, undefined); + #{ <<"arity">> := Arity } = Schema -> + Schema; + _ -> + undefined + end. + +execution_schemas(Schema, AddKey) -> + Args = maps:get(<<"args">>, Schema, []), + Offset = + case AddKey of + false -> 0; + _ -> 1 + end, + { + maybe_nth(1 + Offset, Args, any_type()), + maybe_nth(2 + Offset, Args, any_type()), + maps:get(<<"return">>, Schema, any_type()) + }. + +implicit_base(Schema) -> + implicit_key(Schema, <<"device">>, optional). + +implicit_request(Schema) -> + implicit_key(Schema, <<"path">>, required). + +implicit_key(Schema = #{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, Key, Presence) -> + case maps:is_key(Key, Keys) of + true -> Schema; + false -> + Schema#{ + <<"keys">> => + Keys#{ + Key => + #{ + <<"presence">> => Presence, + <<"type">> => any_type() + } + } + } + end; +implicit_key(Schema, _Key, _Presence) -> + Schema. + +overlay(ReturnSchema) -> + case overlay_type(ReturnSchema) of + base -> base; + request -> request; + _ -> none + end. + +overlay_type(#{ <<"kind">> := <<"message">> } = Schema) -> + Wildcard = + case maps:get(<<"wildcard">>, Schema, undefined) of + #{ <<"type">> := WildcardType } -> overlay_marker(WildcardType); + _ -> none + end, + case Wildcard of + none -> + overlay_marker( + maps:get( + <<"type">>, + maps:get(<<"...">>, maps:get(<<"keys">>, Schema, #{}), #{}), + #{} + ) + ); + Overlay -> Overlay + end; +overlay_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }) -> + first_overlay(Items); +overlay_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }) -> + first_overlay(Members); +overlay_type(_) -> + none. + +first_overlay([]) -> + none; +first_overlay([Schema | Rest]) -> + case overlay_type(Schema) of + none -> first_overlay(Rest); + Overlay -> Overlay + end. + +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"request">> }) -> request; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"request">> }) -> request; +overlay_marker(_) -> none. + +parse_fun_spec({type, _, bounded_fun, [FunSpec, _Constraints]}, TypeEnv) -> + parse_fun_spec(FunSpec, TypeEnv); +parse_fun_spec({type, _, 'fun', [{type, _, product, Args}, Ret]}, TypeEnv) -> + { + lists:map(fun(Arg) -> parse_type(Arg, TypeEnv, #{}, []) end, Args), + parse_type(Ret, TypeEnv, #{}, []) + }; +parse_fun_spec(Other, _TypeEnv) -> + {[unknown_type(Other)], any_type()}. + +parse_type({ann_type, _, [_Var, Type]}, TypeEnv, VarEnv, Seen) -> + parse_type(Type, TypeEnv, VarEnv, Seen); +parse_type({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + any_type(); +parse_type({var, _, Name}, TypeEnv, VarEnv, Seen) -> + case maps:get(Name, VarEnv, undefined) of + undefined -> variable_type(Name); + Bound -> parse_type(Bound, TypeEnv, VarEnv, Seen) + end; +parse_type({user_type, _, Name, Args}, TypeEnv, VarEnv, Seen) -> + case lists:member(Name, Seen) of + true -> + alias_type(Name); + false -> + case maps:get(Name, TypeEnv, undefined) of + undefined -> + alias_type(Name); + #{ vars := Vars, ast := Ast } -> + BoundEnv = + maps:merge( + VarEnv, + maps:from_list(lists:zip(Vars, Args)) + ), + parse_type(Ast, TypeEnv, BoundEnv, [Name | Seen]) + end + end; +parse_type({remote_type, _, [{atom, _, Mod}, {atom, _, Name}, Args]}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"remote">>, + <<"module">> => normalize_name(Mod), + <<"name">> => normalize_name(Name), + <<"args">> => lists:map(fun(Arg) -> parse_type(Arg, TypeEnv, VarEnv, Seen) end, Args) + }; +parse_type({type, _, map, any}, _TypeEnv, _VarEnv, _Seen) -> + any_type(); +parse_type({type, _, map, Fields}, TypeEnv, VarEnv, Seen) -> + message_type( + maps:from_list( + lists:map( + fun({type, _, Assoc, [KeyAst, ValueAst]}) -> + { + key_name(KeyAst, TypeEnv, VarEnv, Seen), + #{ + <<"presence">> => field_presence(Assoc), + <<"type">> => parse_type(ValueAst, TypeEnv, VarEnv, Seen) + } + } + end, + Fields + ) + ) + ); +parse_type({type, _, ListType, [Item]}, TypeEnv, VarEnv, Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ + <<"kind">> => <<"list">>, + <<"item">> => parse_type(Item, TypeEnv, VarEnv, Seen) + }; +parse_type({type, _, ListType, []}, _TypeEnv, _VarEnv, _Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ + <<"kind">> => <<"list">>, + <<"item">> => any_type() + }; +parse_type({type, _, ListType, Item}, TypeEnv, VarEnv, Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ + <<"kind">> => <<"list">>, + <<"item">> => parse_type(Item, TypeEnv, VarEnv, Seen) + }; +parse_type({type, _, tuple, Items}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"tuple">>, + <<"items">> => lists:map(fun(Item) -> parse_type(Item, TypeEnv, VarEnv, Seen) end, Items) + }; +parse_type({type, _, union, Members}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => + lists:map( + fun(Member) -> parse_type(Member, TypeEnv, VarEnv, Seen) end, + Members + ) + }; +parse_type({type, _, range, [Min, Max]}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"range">>, + <<"min">> => literal_value(parse_type(Min, TypeEnv, VarEnv, Seen)), + <<"max">> => literal_value(parse_type(Max, TypeEnv, VarEnv, Seen)) + }; +parse_type({type, _, integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"integer">>); +parse_type({type, _, non_neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"non-neg-integer">>); +parse_type({type, _, pos_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pos-integer">>); +parse_type({type, _, neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"neg-integer">>); +parse_type({type, _, float, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"float">>); +parse_type({type, _, number, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"number">>); +parse_type({type, _, binary, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"binary">>); +parse_type({type, _, bitstring, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"bitstring">>); +parse_type({type, _, boolean, []}, _TypeEnv, _VarEnv, _Seen) -> boolean_type(); +parse_type({type, _, atom, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"atom">>); +parse_type({type, _, pid, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pid">>); +parse_type({type, _, any, []}, _TypeEnv, _VarEnv, _Seen) -> any_type(); +parse_type({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(Atom)); +parse_type({integer, _, Int}, _TypeEnv, _VarEnv, _Seen) -> literal_type(Int); +parse_type({char, _, Char}, _TypeEnv, _VarEnv, _Seen) -> literal_type(<>); +parse_type({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(String)); +parse_type({nil, _}, _TypeEnv, _VarEnv, _Seen) -> literal_type([]); +parse_type(Other, _TypeEnv, _VarEnv, _Seen) -> + ?event({parse_type_other, Other}), + unknown_type(Other). + +field_presence(map_field_exact) -> required; +field_presence(map_field_assoc) -> optional; +field_presence(Other) -> normalize_name(Other). + +key_name({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> + normalize_name(Atom); +key_name({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> + hb_util:bin(String); +key_name({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + <<"_">>; +key_name(Other, TypeEnv, VarEnv, Seen) -> + case parse_type(Other, TypeEnv, VarEnv, Seen) of + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } when is_binary(Value) -> + Value; + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } -> + hb_util:bin(io_lib:format("~tp", [Value])); + _ -> + hb_util:bin(io_lib:format("~tp", [Other])) + end. + +apply_schema(#{ <<"kind">> := <<"message">>, <<"keys">> := Keys, <<"all">> := All }, Message, Opts) + when is_map(Message) -> + ?event(apply_schema, {message, {keys, Keys}, {all, All}, {message, Message}}), + % Apply declared keys first so their coerced values take precedence. + {Explicit, Changed} = + lists:foldl( + fun({Key, #{ <<"presence">> := Presence, <<"type">> := Type }}, {Acc, Changed}) -> + ?event({apply_schema_find, {key, Key}, {message, Message}, {presence, Presence}, {type, Type}}), + % If we find the key in the message, apply the schema to the value. + % If the key is not found and the field is required, throw an error. + % If the key is not found and the field is optional, skip it. + case hb_maps:find(Key, Message, Opts) of + {ok, Value} -> + Applied = apply_schema(Type, Value, Opts), + RawValue = maps:get(Key, Message, '$hb_types_missing'), + { + Acc#{ Key => Applied }, + Changed orelse Applied =/= RawValue + }; + error when Presence =:= required -> + throw({required_key_missing, Key}); + error -> + {Acc, Changed} + end + end, + {#{}, false}, + maps:to_list(Keys) + ), + % If `all` is true, pass through any unmatched keys unchanged. + case All of + true when not Changed -> + Message; + true -> + maps:merge( + maps:without(maps:keys(Keys), Message), + Explicit + ); + false -> + Explicit + end; +apply_schema(Type, Message, _Opts) -> + ?event({apply_schema_check_type, {type, Type}, {message, Message}}), + % If the type matches the message, return the message unchanged. + % If the type does not match the message, coerce the message to the type. + case check_type(Type, Message) of + true -> Message; + false -> + case coerce_type(Type, Message) of + error -> throw({invalid_type, Type, Message}); + Coerced -> Coerced + end + end. + +%% @doc Coerce a value to a type. If the value is not coercible, return error. +%% Otherwise, return the coerced value. +coerce_type(_, undefined) -> error; +coerce_type(#{ <<"kind">> := <<"any">> }, Value) -> Value; +coerce_type(#{ <<"kind">> := <<"integer">> }, Value) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"pos-integer">> }, Value) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"neg-integer">> }, Value) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"float">> }, Value) -> + try_coerce(fun hb_util:float/1, Value); +coerce_type(#{ <<"kind">> := <<"number">> }, Value) -> + coerce_with([fun hb_util:int/1, fun hb_util:float/1], Value); +coerce_type(#{ <<"kind">> := <<"binary">> }, Value) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"bitstring">> }, Value) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"boolean">> }, Value) -> + case is_boolean_coercible(Value) of + true -> try_coerce(fun hb_util:bool/1, Value); + false -> error + end; +coerce_type(#{ <<"kind">> := <<"atom">> }, Value) -> + try_coerce(fun hb_util:atom/1, Value); +coerce_type(#{ <<"kind">> := <<"pid">> }, _Value) -> + error; +coerce_type(#{ <<"kind">> := <<"message">> }, Value) -> + try_coerce(fun hb_util:map/1, Value); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) when is_tuple(Value) -> + coerce_type(#{ <<"kind">> => <<"tuple">>, <<"items">> => Items }, tuple_to_list(Value)); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) when is_list(Value) -> + case length(Value) =:= length(Items) of + false -> error; + true -> + case coerce_sequence(lists:zip(Items, Value)) of + error -> error; + Coerced -> list_to_tuple(Coerced) + end + end; +coerce_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value) -> + case try_coerce(fun hb_util:list/1, Value) of + error -> error; + Coerced -> coerce_list(ItemType, Coerced) + end; +coerce_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value) -> + coerce_union(Members, Value); +coerce_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value) -> + coerce_literal(Expected, Value); +coerce_type(#{ <<"kind">> := <<"range">> }, Value) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(_, _) -> error. + +try_coerce(Fun, Value) -> + try Fun(Value) of + Coerced -> Coerced + catch + _:_ -> error + end. + +%% @doc Coerce a value with a list of functions. +%% This is useful for kind: number, which can be coerced to an integer or a float. +coerce_with([], _Value) -> + error; +coerce_with([Fun | Rest], Value) -> + case try_coerce(Fun, Value) of + error -> coerce_with(Rest, Value); + Coerced -> Coerced + end. + +%% @doc Coerce a sequence of values to a list of types. +%% This is useful for coercing a list to a tuple. +coerce_sequence([]) -> + []; +coerce_sequence([{Type, Value} | Rest]) -> + case coerce_type(Type, Value) of + error -> error; + Coerced -> + case coerce_sequence(Rest) of + error -> error; + CoercedRest -> [Coerced | CoercedRest] + end + end. + +coerce_list(ItemType, Value) when is_list(Value) -> + coerce_sequence([{ItemType, Item} || Item <- Value]); +coerce_list(_ItemType, _Value) -> + error. + +coerce_union([], _Value) -> + error; +coerce_union([Member | Rest], Value) -> + case coerce_type(Member, Value) of + error -> coerce_union(Rest, Value); + Coerced -> Coerced + end. + +coerce_literal(Expected, Value) when is_integer(Expected) -> + try_coerce(fun hb_util:int/1, Value); +coerce_literal(Expected, Value) when is_float(Expected) -> + try_coerce(fun hb_util:float/1, Value); +coerce_literal(Expected, Value) when is_binary(Expected) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_literal(Expected, Value) when is_atom(Expected) -> + case is_boolean(Expected) andalso is_boolean_coercible(Value) of + true -> try_coerce(fun hb_util:bool/1, Value); + false -> try_coerce(fun hb_util:atom/1, Value) + end; +coerce_literal(Expected, Value) when is_list(Expected) -> + case try_coerce(fun hb_util:list/1, Value) of + error -> error; + Coerced when length(Coerced) =:= length(Expected) -> Coerced; + _ -> error + end; +coerce_literal(Expected, Value) when is_map(Expected) -> + try_coerce(fun hb_util:map/1, Value); +coerce_literal(Expected, Value) when Value =:= Expected -> + Value; +coerce_literal(_Expected, _Value) -> + error. + +is_boolean_coercible(Value) -> + Coercible = [true, false, 1, 0, <<"true">>, <<"false">>, <<"1">>, <<"0">>], + lists:member(Value, Coercible). + +check_type(#{ <<"kind">> := <<"any">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"integer">> }, Value) -> is_integer(Value); +check_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value) -> is_integer(Value) andalso Value >= 0; +check_type(#{ <<"kind">> := <<"pos-integer">> }, Value) -> is_integer(Value) andalso Value > 0; +check_type(#{ <<"kind">> := <<"neg-integer">> }, Value) -> is_integer(Value) andalso Value < 0; +check_type(#{ <<"kind">> := <<"float">> }, Value) -> is_float(Value); +check_type(#{ <<"kind">> := <<"number">> }, Value) -> is_number(Value); +check_type(#{ <<"kind">> := <<"binary">> }, Value) -> is_binary(Value); +check_type(#{ <<"kind">> := <<"bitstring">> }, Value) -> is_bitstring(Value); +check_type(#{ <<"kind">> := <<"boolean">> }, Value) -> is_boolean(Value); +check_type(#{ <<"kind">> := <<"atom">> }, Value) -> is_atom(Value); +check_type(#{ <<"kind">> := <<"pid">> }, Value) -> is_pid(Value); +check_type(#{ <<"kind">> := <<"message">> }, Value) -> is_map(Value); +check_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) -> + is_tuple(Value) + andalso tuple_size(Value) =:= length(Items) + andalso lists:all( + fun({Index, ItemType}) -> check_type(ItemType, element(Index, Value)) end, + lists:zip(lists:seq(1, length(Items)), Items) + ); +check_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value) -> + is_list(Value) andalso lists:all(fun(Item) -> check_type(ItemType, Item) end, Value); +check_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value) -> + lists:any(fun(Member) -> check_type(Member, Value) end, Members); +check_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value) -> + Value =:= Expected; +check_type(#{ <<"kind">> := <<"range">>, <<"min">> := Min, <<"max">> := Max }, Value) -> + is_integer(Value) andalso Value >= Min andalso Value =< Max; +check_type(#{ <<"kind">> := <<"remote">> }, _Value) -> + true; +check_type(#{ <<"kind">> := <<"alias">> }, _Value) -> + true; +check_type(#{ <<"kind">> := <<"variable">> }, _Value) -> + true; +check_type(_, _) -> + true. + +%% @doc Ensure that a name is a `dash-separated-binary` form, rather than +%% an atom, list, etc. +normalize_name('_') -> <<"_">>; +normalize_name(Name) when is_atom(Name) -> hb_util:atom_to_key(Name); +normalize_name(Name) -> hb_util:bin(Name). + +%% @doc Extract the value from a literal form. +literal_value(#{ <<"kind">> := <<"literal">>, <<"value">> := Value }) -> Value. + +%% @doc Extract the name from a variable form. +var_name({var, _, Name}) -> Name; +var_name(Name) -> Name. + +any_type() -> #{ <<"kind">> => <<"any">> }. +scalar_type(Name) -> #{ <<"kind">> => Name }. +literal_type(Value) -> #{ <<"kind">> => <<"literal">>, <<"value">> => Value }. +alias_type(Name) -> #{ <<"kind">> => <<"alias">>, <<"name">> => normalize_name(Name) }. +variable_type(Name) -> #{ <<"kind">> => <<"variable">>, <<"name">> => normalize_name(Name) }. +message_type(AllKeys) -> + Wildcard = maps:get(<<"_">>, AllKeys, undefined), + ?event(apply_schema, {message_type, {all_keys, AllKeys}, {wildcard, Wildcard}}), + % If the `_` key is exactly the literal `_`, pass through unmatched keys. + % Otherwise, only maintain explicitly declared keys. + #{ + <<"kind">> => <<"message">>, + <<"keys">> => maps:without([<<"_">>], AllKeys), + <<"wildcard">> => Wildcard, + <<"all">> => + case Wildcard of + #{ <<"type">> := #{ <<"kind">> := <<"any">> } } -> true; + #{ <<"type">> := #{ <<"value">> := <<"_">> } } -> true; + _ -> false + end + }. +unknown_type(Other) -> #{ <<"kind">> => <<"unknown">>, <<"ast">> => hb_util:bin(io_lib:format("~tp", [Other])) }. +boolean_type() -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => [literal_type(true), literal_type(false)] + }. + +%%% Tests + +test_opts() -> + #{ + store => [hb_test_utils:test_store()], + priv_wallet => hb:wallet() + }. + +extract_test() -> + Res = extract(<<"test-device@1.0">>, #{}), + ?event({extraction_result, Res}), + ?assertMatch( + {ok, #{ <<"keys">> := #{}, <<"types">> := #{}}}, + Res + ). + +successful_vary_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{ <<"unused">> => 1 }, + #{ <<"slot">> => 1 }, + Opts + ), + ?assertEqual(#{ <<"unused">> => 1 }, VariedBase), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq), + ?event( + debug_types, + {vary_result, + {varied_base, {explicit, VariedBase}}, + {varied_req, {explicit, VariedReq}} + } + ). + +function_vary_adds_implicit_keys_and_overlay_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq, Overlay} = + vary( + <<"test-device@1.0">>, + <<"varied">>, + fun dev_test:varied/3, + false, + #{ + <<"device">> => <<"test-device@1.0">>, + <<"x">> => <<"1">>, + <<"extra">> => <<"base">> + }, + #{ <<"path">> => <<"varied">>, <<"extra">> => <<"req">> }, + Opts + ), + ?assertEqual( + #{ <<"device">> => <<"test-device@1.0">>, <<"x">> => 1 }, + VariedBase + ), + ?assertEqual(#{ <<"path">> => <<"varied">> }, VariedReq), + ?assertEqual(base, Overlay). + +vary_throw_required_key_missing_test() -> + Opts = test_opts(), + ?assertThrow( + {required_key_missing, _}, + vary(<<"test-device@1.0">>, <<"compute">>, #{}, #{}, Opts) + ). + +vary_required_key_wrong_type_test() -> + Opts = test_opts(), + ?assertMatch( + { + ok, + #{}, + #{ <<"slot">> := 1 } + }, + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{}, + #{ <<"slot">> => <<"1">> }, + Opts + ) + ). + +vary_optional_key_wrong_type_test() -> + Opts = test_opts(), + ?assertMatch( + { + ok, + #{ <<"already-seen">> := [1] }, + #{ <<"slot">> := 1 } + }, + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{ <<"already-seen">> => [<<"1">>] }, + #{ <<"slot">> => <<"1">> }, + Opts + ) + ). + +successful_nested_vary_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-nested">>, + #{}, + #{ + <<"outer">> => + #{ + <<"slot">> => 1, + <<"unused">> => + #{ <<"unused-key">> => <<"unused-value">> } + } + }, + Opts + ), + ?event(debug_types, {vary_result, {varied_base, VariedBase}, {varied_req, VariedReq}}), + ?assertEqual(#{}, VariedBase), + ?assertEqual( + #{ <<"outer">> => #{ <<"slot">> => 1 }}, + VariedReq + ). + +vary_throw_nested_key_missing_test() -> + Opts = test_opts(), + ?assertThrow( + {required_key_missing, _}, + vary( + <<"test-device@1.0">>, + <<"compute-nested">>, + #{}, + #{ <<"outer">> => #{ <<"not-slot">> => 1 }}, + Opts + ) + ). + +vary_nested_key_wrong_type_test() -> + Opts = test_opts(), + ?assertMatch( + {ok, #{}, #{ <<"outer">> := #{ <<"slot">> := 1 }}}, + vary( + <<"test-device@1.0">>, + <<"compute-nested">>, + #{}, + #{ <<"outer">> => #{ <<"slot">> => <<"1">> }}, + Opts + ) + ). + +vary_coerces_required_key_from_binary_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{}, + #{ <<"slot">> => <<"1">> }, + Opts + ), + ?assertEqual(#{}, VariedBase), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq). + +vary_coerces_required_key_from_list_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{}, + #{ <<"slot">> => "1" }, + Opts + ), + ?assertEqual(#{}, VariedBase), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq). + +vary_throw_required_key_noncoercible_test() -> + Opts = test_opts(), + ?assertThrow( + {invalid_type, _, _}, + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{}, + #{ <<"slot">> => <<"not-an-int">> }, + Opts + ) + ). + +vary_coerces_optional_base_key_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{ <<"already-seen">> => [<<"2">>] }, + #{ <<"slot">> => <<"1">> }, + Opts + ), + ?assertEqual(#{ <<"already-seen">> => [2] }, VariedBase), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq). + +vary_throw_optional_base_key_noncoercible_test() -> + Opts = test_opts(), + ?assertThrow( + {invalid_type, _, _}, + vary( + <<"test-device@1.0">>, + <<"compute">>, + #{ <<"already-seen">> => [<<"not-an-int">>] }, + #{ <<"slot">> => 1 }, + Opts + ) + ). + +unschematized_key_returns_messages_unchanged_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"nonexistent-func">>, + #{ <<"base">> => <<"value">> }, + #{ + <<"outer">> => + #{ + <<"slot">> => 1, + <<"unused">> => + #{ <<"unused-key">> => <<"unused-value">> } + } + }, + Opts + ), + ?event(debug_types, {vary_result, {varied_base, VariedBase}, {varied_req, VariedReq}}), + ?assertEqual(#{ <<"base">> => <<"value">> }, VariedBase), + ?assertEqual( + #{ + <<"outer">> => + #{ + <<"slot">> => 1, + <<"unused">> => + #{ <<"unused-key">> => <<"unused-value">> } + } + }, + VariedReq + ). + +unschematized_key_missing_req_is_unchanged_test() -> + Opts = test_opts(), + ?assertEqual( + {ok, #{}, #{ <<"outer">> => #{ <<"not-slot">> => 1 }}}, + vary( + <<"test-device@1.0">>, + <<"nonexistent-func">>, + #{}, + #{ <<"outer">> => #{ <<"not-slot">> => 1 }}, + Opts + ) + ). + +unschematized_key_wrong_type_is_unchanged_test() -> + Opts = test_opts(), + ?assertEqual( + {ok, #{}, #{ <<"outer">> => #{ <<"slot">> => <<"1">> }}}, + vary( + <<"test-device@1.0">>, + <<"nonexistent-func">>, + #{}, + #{ <<"outer">> => #{ <<"slot">> => <<"1">> }}, + Opts + ) + ). + +vary_on_all_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-all">>, + #{ <<"a">> => <<"1">>, <<"b">> => 2 }, + #{ <<"slot">> => 1 }, + Opts + ), + ?event(debug_types, {vary_result, {varied_base, VariedBase}, {varied_req, VariedReq}}), + ?assertEqual(#{ <<"a">> => 1, <<"b">> => 2 }, VariedBase), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq). + +vary_on_all_nested_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-all">>, + #{ + <<"a">> => <<"1">>, + <<"b">> => 2, + <<"outer">> => #{ + <<"c">> => <<"3">>, + <<"d">> => <<"4">> + } + }, + #{ <<"slot">> => 1 }, + Opts + ), + ?event(debug_types, {vary_result, {varied_base, VariedBase}, {varied_req, VariedReq}}), + ?assertEqual( + #{ + <<"a">> => 1, + <<"b">> => 2, + <<"outer">> => #{ <<"c">> => <<"3">>, <<"d">> => <<"4">> } + }, + VariedBase + ), + ?assertEqual(#{ <<"slot">> => 1 }, VariedReq). + +vary_on_all_preserves_extra_request_keys_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-all">>, + #{ <<"a">> => 1 }, + #{ <<"slot">> => 1, <<"extra">> => <<"x">> }, + Opts + ), + ?assertEqual(#{ <<"a">> => 1 }, VariedBase), + ?assertEqual( + #{ <<"slot">> => 1, <<"extra">> => <<"x">> }, + VariedReq + ). + +vary_on_all_preserves_nested_request_keys_test() -> + Opts = test_opts(), + {ok, _VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-all">>, + #{}, + #{ + <<"slot">> => 1, + <<"outer">> => #{ <<"c">> => 3, <<"d">> => 4 } + }, + Opts + ), + ?assertEqual( + #{ + <<"slot">> => 1, + <<"outer">> => #{ <<"c">> => 3, <<"d">> => 4 } + }, + VariedReq + ). + +vary_on_all_removes_schematized_nested_keys_test() -> + Opts = test_opts(), + {ok, VariedBase, VariedReq} = + vary( + <<"test-device@1.0">>, + <<"compute-all-nested">>, + #{ + <<"nested">> => #{ <<"a">> => <<"1">>, <<"b">> => <<"2">> }, + <<"other">> => <<"3">> + }, + #{ + <<"slot">> => 1, + <<"nested">> => #{ <<"c">> => 3, <<"d">> => 4 } + }, + Opts + ), + ?assertEqual( + #{ + <<"nested">> => #{ <<"a">> => 1 }, + <<"other">> => <<"3">> + }, + VariedBase + ), + ?assertEqual( + #{ + <<"slot">> => 1, + <<"nested">> => #{ <<"c">> => 3, <<"d">> => 4 } + }, + VariedReq + ). diff --git a/src/preloaded/arweave/dev_arweave.erl b/src/preloaded/arweave/dev_arweave.erl index 88ca6523d..cde9b0812 100644 --- a/src/preloaded/arweave/dev_arweave.erl +++ b/src/preloaded/arweave/dev_arweave.erl @@ -26,12 +26,15 @@ info() -> }. %% @doc Proxy the `/info' endpoint from the Arweave node. +-spec status(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. status(_Base, _Request, Opts) -> request(<<"GET">>, <<"/info">>, Opts). %% @doc Returns the given transaction as an AO-Core message. By default, this %% embeds the `/raw` payload. Set `exclude-data` to true to return just the %% header. +-spec tx(#{ _ => _ }, #{ method => binary(), tx => binary(), target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. tx(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"POST">> -> post_tx(Base, Request, Opts); @@ -45,13 +48,20 @@ tx(Base, Request, Opts) -> %% Note: When uploading ans104 transactions, this function will use the %% node's default bundler. If instead you want to use this node as a bundler %% you should use the ~bundler@1.0 device. +-spec post_tx(#{ _ => _ }, #{ target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. post_tx(Base, RawRequest, Opts) -> {ok, Request} = extract_target(Base, RawRequest, Opts), - case hb_maps:find(<<"commitment-device">>, Request, Opts) of + case hb_maps:find(<<"commitment-device">>, RawRequest, Opts) of {ok, Device} -> post_tx(Base, Request, Opts, Device); error -> - post_tx_detect_device(Base, Request, Opts) + case hb_maps:find(<<"commitment-device">>, Request, Opts) of + {ok, Device} -> + post_tx(Base, Request, Opts, Device); + error -> + post_tx_detect_device(Base, Request, Opts) + end end. %% @doc Detect the commitment device to use when posting a transaction. @@ -157,6 +167,8 @@ get_tx(Base, Request, Opts) -> %% @doc A router for range requests by method. Both `HEAD` and `GET` requests %% are supported. +-spec raw(#{ raw => binary(), _ => _ }, #{ method => binary(), raw => binary(), range => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _}. raw(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"HEAD">> -> head_raw(Base, Request, Opts); @@ -372,6 +384,11 @@ list_find(Key, [{XKey, Value} | Rest], Default) -> %% offset and length. %% - `GET` with `txid`: `GET`s a chunk or range of bytes from the given offset, %% relative to the given transaction's data root. +-spec chunk( + #{ _ => _ }, + #{ method => binary(), offset => integer(), length => integer(), pending => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, binary() | #{ _ => _ }} | {error, _}. chunk(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"POST">> -> post_chunk(Base, Request, Opts); @@ -662,11 +679,76 @@ get_chunk(Offset, Opts) -> Path = <<"/chunk/", (hb_util:bin(Offset))/binary>>, request(<<"GET">>, Path, #{ <<"route-by">> => Offset }, Opts). +%% @doc Read and decode the bundle header index at the given global start +%% offset, returning the header size alongside the decoded index entries. +-spec bundle_header(non_neg_integer(), non_neg_integer() | infinity, #{ _ => _ }) -> + {ok, non_neg_integer(), [_]} | {error, _}. +bundle_header(BundleStartOffset, Opts) -> + bundle_header(BundleStartOffset, infinity, Opts). +bundle_header(BundleStartOffset, MaxSize, Opts) -> + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + 1 + }, + Opts + ) of + {ok, FirstChunk} -> + case ar_bundles:bundle_header_size(FirstChunk) of + invalid_bundle_header -> + {error, invalid_bundle_header}; + HeaderSize when HeaderSize > MaxSize -> + {error, invalid_bundle_header}; + HeaderSize -> + case read_bundle_header( + BundleStartOffset, HeaderSize, + FirstChunk, Opts + ) of + {ok, HeaderBin} -> + case ar_bundles:decode_bundle_header( + HeaderBin + ) of + {_Items, BundleIndex} -> + {ok, HeaderSize, BundleIndex}; + invalid_bundle_header -> + {error, invalid_bundle_header} + end; + Error -> + Error + end + end; + Error -> + Error + end. + +%% @doc Read exactly the bytes needed to decode a bundle header. +read_bundle_header(_BundleStartOffset, HeaderSize, FirstChunk, _Opts) + when HeaderSize =< byte_size(FirstChunk) -> + {ok, binary:part(FirstChunk, 0, HeaderSize)}; +read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) -> + RemainingSize = HeaderSize - byte_size(FirstChunk), + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + byte_size(FirstChunk) + 1, + <<"length">> => RemainingSize + }, + Opts + ) of + {ok, RemainingChunk} -> + {ok, <>}; + Error -> + Error + end. + %% @doc Retrieve (and cache) block information from Arweave. If the `block' key %% is present, it is used to look up the associated block. If it is of Arweave %% block hash length (43 characters), it is used as an ID. If it is parsable as %% an integer, it is used as a block height. If it is not present, the current %% block is used. +-spec block(#{ _ => _ }, #{ _ => _ }, map()) -> term(). block(Base, Request, Opts) when is_map(Base) -> Block = hb_ao:get_first( @@ -744,9 +826,12 @@ only_if_cached(Req, Opts) -> ). %% @doc Retrieve the current block information from Arweave. +-spec current(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. current(_Base, _Request, Opts) -> request(<<"GET">>, <<"/block/current">>, Opts). +-spec price(#{ size => integer(), _ => _ }, #{ size => integer(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _}. price(Base, Request, Opts) -> Size = hb_ao:get_first( @@ -764,11 +849,14 @@ price(Base, Request, Opts) -> request(<<"GET">>, <<"/price/", (hb_util:bin(Size))/binary>>, Opts) end. +-spec tx_anchor(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }} | {error, _}. tx_anchor(_Base, _Request, Opts) -> request(<<"GET">>, <<"/tx_anchor">>, Opts). %% @doc Retrieve either a list of the pending TXIDs on the configured Arweave %% nodes, or a specific unconfirmed transaction header by its TXID. +-spec pending(#{ pending => binary(), _ => _ }, #{ pending => binary(), offset => integer(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | [binary()] | #{ _ => _ }} | {error, _}. pending(Base, Request, Opts) -> case find_key(<<"pending">>, Base, Request, Opts) of not_found -> request(<<"GET">>, <<"/tx/pending">>, Opts); diff --git a/src/preloaded/arweave/dev_arweave_offset.erl b/src/preloaded/arweave/dev_arweave_offset.erl index 66e6923a6..b5cd4dcd8 100644 --- a/src/preloaded/arweave/dev_arweave_offset.erl +++ b/src/preloaded/arweave/dev_arweave_offset.erl @@ -8,6 +8,7 @@ %% @doc Resolve either a message at an Arweave offset, or a direct key from the %% base message if the key is not an integer. +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). get(Key, Base, _Request, Opts) -> case parse(Key) of {ok, StartOffset, Length} -> diff --git a/src/preloaded/arweave/dev_bundler.erl b/src/preloaded/arweave/dev_bundler.erl index 98112fdd9..82a6ead00 100644 --- a/src/preloaded/arweave/dev_bundler.erl +++ b/src/preloaded/arweave/dev_bundler.erl @@ -31,11 +31,15 @@ %%% Public interface. %% @doc An alias for `item/3'. +-spec tx(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ id := binary(), timestamp := integer(), _ => _ }} | {error, #{ _ => _ }}. tx(Base, Req, Opts) -> item(Base, Req, Opts). %% @doc Implements an `up.arweave.net'-compatible endpoint for %% bundling messages. +-spec item(#{ _ => _ }, #{ 'bundler-subject' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ id := binary(), timestamp := integer(), _ => _ }} | {error, #{ _ => _ }}. item(_Base, Req, Opts) -> ServerPID = ensure_server(Opts), ItemToProcess = @@ -1542,10 +1546,10 @@ assert_bundle(Node, ExpectedItems, Anchor, Price, TXRequest, Proofs, ClientOpts) fun(ChunkRequest) -> ProofBinary = maps:get(<<"body">>, ChunkRequest), ProofJSON = hb_json:decode(ProofBinary), - Offset = binary_to_integer(maps:get(<<"offset">>, ProofJSON)), + Offset = hb_util:int(maps:get(<<"offset">>, ProofJSON)), Chunk = hb_util:decode(maps:get(<<"chunk">>, ProofJSON)), DataRoot = hb_util:decode(maps:get(<<"data_root">>, ProofJSON)), - DataSize = binary_to_integer(maps:get(<<"data_size">>, ProofJSON)), + DataSize = hb_util:int(maps:get(<<"data_size">>, ProofJSON)), DataPath = hb_util:decode(maps:get(<<"data_path">>, ProofJSON)), Valid = ar_merkle:validate_path(DataRoot, Offset, DataSize, DataPath), ?assertNotEqual(false, Valid), diff --git a/src/preloaded/arweave/dev_manifest.erl b/src/preloaded/arweave/dev_manifest.erl index 55f43a7e2..b0871826f 100644 --- a/src/preloaded/arweave/dev_manifest.erl +++ b/src/preloaded/arweave/dev_manifest.erl @@ -14,6 +14,10 @@ info() -> }. %% @doc Return the fallback index page when the manifest itself is requested. +-spec index(#{ index => #{ path => binary(), _ => _ }, paths => #{ _ => _ }, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, _} | {error, not_found}. index(M1, M2, Opts) -> ?event(debug_manifest, {index_request, {base, M1}, {request, M2}}, Opts), case route(<<"index">>, M1, M2, Opts) of @@ -25,6 +29,8 @@ index(M1, M2, Opts) -> end. %% @doc Route a request to the associated data via its manifest. +-spec route(binary(), #{ paths => #{ _ => _ }, index => #{ path => binary(), _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, not_found}. route(<<"index">>, M1, M2, Opts) -> ?event({manifest_index, M1, M2}), case manifest(M1, M2, Opts) of @@ -86,6 +92,8 @@ route(Key, M1, M2, Opts) -> %% @doc Implement the `on/request' hook for the `manifest@1.0' device, finding %% requests for legacy (non-device-tagged) manifests and casting them to %% `manifest@1.0' before execution. Allowing `/ID/path` style access for old data. +-spec request(#{ _ => _ }, #{ body := [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ body := [_], _ => _ }} | {error, #{ status := integer(), body := binary() }}. request(Base, Req, Opts) -> ?event({on_req_manifest_detector, {base, Base}, {req, Req}}), maybe @@ -111,7 +119,7 @@ request(Base, Req, Opts) -> {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; {_, {ok, Casted}} -> ?event(debug_manifest, {manifest_returning_subpath, {req, Req}}), - {ok, Req#{ <<"body">> => [Casted|Rest] }} + {ok, Req#{ <<"body">> => [Casted|maybe_no_cache_404(Rest, Opts)] }} end else {error, not_found} -> @@ -129,6 +137,19 @@ request(Base, Req, Opts) -> {ok, Req} end. +maybe_no_cache_404(Rest, Opts) -> + case hb_opts:get(manifest_404, fallback, Opts) of + error -> + lists:map(fun no_cache/1, Rest); + _ -> + Rest + end. + +no_cache(Msg) when is_map(Msg) -> + Msg#{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }; +no_cache(Msg) -> + Msg. + %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; diff --git a/src/preloaded/auth/dev_auth_hook.erl b/src/preloaded/auth/dev_auth_hook.erl index 0cc6e83d6..a8dbc2ffe 100644 --- a/src/preloaded/auth/dev_auth_hook.erl +++ b/src/preloaded/auth/dev_auth_hook.erl @@ -104,6 +104,11 @@ %% by the user request). %% %% +-spec request( + #{ 'secret-provider' => _, 'generate-path' => binary(), 'finalize-path' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | {skip, _, _} | error. request(Base, HookReq, Opts) -> ?event({auth_hook_request, {base, Base}, {hook_req, HookReq}}), maybe @@ -112,8 +117,8 @@ request(Base, HookReq, Opts) -> {ok, Provider} ?= find_provider(Base, Opts), % Check if the request already has signatures, or the hook base enforces % that we should always attempt to sign the request. - {ok, Request} ?= hb_maps:find(<<"request">>, HookReq, Opts), - {ok, OrigMessages} ?= hb_maps:find(<<"body">>, HookReq, Opts), + {ok, Request} ?= maps:find(<<"request">>, HookReq), + {ok, OrigMessages} ?= maps:find(<<"body">>, HookReq), true ?= is_relevant(Base, Request, OrigMessages, Opts), ?event(auth_hook_is_relevant), % Call the key provider to normalize authentication (generate if needed) @@ -258,7 +263,7 @@ generate_secret(Provider, Request, Opts) -> % If there is a `wallet' field in the request, we move it to the % provider, else continue with the existing provider. ?event({normalized_req, NormalizedReq}), - case hb_maps:find(<<"secret">>, NormalizedReq, Opts) of + case maps:find(<<"secret">>, NormalizedReq) of {ok, Key} -> ?event({key_found_in_normalized_req, Key}), { @@ -273,8 +278,8 @@ generate_secret(Provider, Request, Opts) -> end. %% @doc Strip the `secret' field from a request. -strip_sensitive(Request, Opts) -> - hb_maps:without([<<"secret">>], Request, Opts). +strip_sensitive(Request, _Opts) -> + maps:remove(<<"secret">>, Request). %% @doc Generate a wallet with the key if the `wallet' field is not present in %% the provider after normalization. @@ -299,7 +304,7 @@ sign_request(Provider, Msg, Opts) -> true -> % Wallet signs without ignored keys IgnoredKeys = ignored_keys(Msg, Opts), - WithoutIgnored = hb_maps:without(IgnoredKeys, Msg, Opts), + WithoutIgnored = maps:without(IgnoredKeys, Msg), % Call the wallet to sign the request. case hb_ao:raw( <<"secret@1.0">>, @@ -310,10 +315,9 @@ sign_request(Provider, Msg, Opts) -> {ok, Signed} -> ?event({auth_hook_signed, Signed}), SignedWithIgnored = - hb_maps:merge( + maps:merge( Signed, - hb_maps:with(IgnoredKeys, Msg, Opts), - Opts + maps:with(IgnoredKeys, Msg) ), {ok, SignedWithIgnored}; {error, Err} -> @@ -413,7 +417,7 @@ call_provider(Key, Provider, Request, Opts) -> case hb_ao:resolve(Provider, Request#{ <<"path">> => ExecKey }, Opts) of {ok, Msg} when is_map(Msg) -> % The result is a message. We revert the path to its original value. - case hb_maps:find(<<"path">>, Request, Opts) of + case maps:find(<<"path">>, Request) of {ok, Path} -> {ok, Msg#{ <<"path">> => Path }}; _ -> {ok, Msg} end; diff --git a/src/preloaded/auth/dev_cookie.erl b/src/preloaded/auth/dev_cookie.erl index e5fdbedae..aad4c3212 100644 --- a/src/preloaded/auth/dev_cookie.erl +++ b/src/preloaded/auth/dev_cookie.erl @@ -50,17 +50,22 @@ opts(Opts) -> hb_private:opts(Opts). %%% ~message@1.0 Commitments API keys. +-spec commit(#{ _ => _ }, #{ secret => binary(), _ => _ }, map()) -> term(). commit(Base, Req, RawOpts) -> dev_cookie_auth:commit(Base, Req, RawOpts). +-spec verify(#{ _ => _ }, #{ secret => binary(), _ => _ }, map()) -> term(). verify(Base, Req, RawOpts) -> dev_cookie_auth:verify(Base, Req, RawOpts). %% @doc Preprocessor keys that utilize cookies and the `~secret@1.0' device to %% sign inbound HTTP requests from users if they are not already signed. We use %% the hook authentication framework to implement this. +-spec generate(#{ _ => _ }, #{ committer => binary(), generator => _, _ => _ }, map()) -> term(). generate(Base, Req, Opts) -> dev_cookie_auth:generate(Base, Req, Opts). %% @doc Finalize an `on-request' hook by adding the `set-cookie' header to the %% end of the message sequence. +-spec finalize(#{ _ => _ }, #{ request := #{ _ => _ }, body := _, _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, no_request}. finalize(Base, Request, Opts) -> dev_cookie_auth:finalize(Base, Request, Opts). @@ -76,14 +81,16 @@ finalize(Base, Request, Opts) -> %% %% The `format' may be specified in the request message as the `req:format' key. %% If no `format' is specified, the default is `default'. +-spec get_cookie(#{ _ => _ }, #{ key := binary(), format => binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, not_found}. get_cookie(Base, Req, RawOpts) -> Opts = opts(RawOpts), {ok, Cookies} = extract(Base, Req, Opts), - Key = hb_maps:get(<<"key">>, Req, undefined, Opts), - case hb_maps:get(Key, Cookies, undefined, Opts) of + Key = maps:get(<<"key">>, Req), + case maps:get(Key, Cookies, undefined) of undefined -> {error, not_found}; Cookie -> - Format = hb_maps:get(<<"format">>, Req, <<"default">>, Opts), + Format = maps:get(<<"format">>, Req, <<"default">>), case Format of <<"default">> -> {ok, Cookie}; <<"set-cookie">> -> {ok, normalize_cookie_value(Cookie)}; @@ -92,6 +99,7 @@ get_cookie(Base, Req, RawOpts) -> end. %% @doc Return the parsed and normalized cookies from a message. +-spec extract(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. extract(Msg, Req, Opts) -> {ok, MsgWithCookie} = from(Msg, Req, Opts), Cookies = hb_private:get(<<"cookie">>, MsgWithCookie, #{}, Opts), @@ -100,6 +108,7 @@ extract(Msg, Req, Opts) -> %% @doc Set the keys in the request message in the cookies of the caller. Removes %% a set of base keys from the request message before setting the remainder as %% cookies. +-spec store(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. store(Base, Req, RawOpts) -> Opts = opts(RawOpts), ?event({store, {base, Base}, {req, Req}}), @@ -108,7 +117,7 @@ store(Base, Req, RawOpts) -> {ok, ResetBase} = reset(Base, Opts), ?event({store, {reset_base, ResetBase}}), MsgToSet = - hb_maps:without( + maps:without( [ <<"path">>, <<"accept-bundle">>, @@ -117,11 +126,10 @@ store(Base, Req, RawOpts) -> <<"method">>, <<"body">> ], - hb_private:reset(Req), - Opts + hb_private:reset(Req) ), ?event({store, {msg_to_set, MsgToSet}}), - NewCookies = hb_maps:merge(ExistingCookies, MsgToSet, Opts), + NewCookies = maps:merge(ExistingCookies, MsgToSet), NewBase = hb_private:set(ResetBase, <<"cookie">>, NewCookies, Opts), {ok, NewBase}. @@ -129,12 +137,7 @@ store(Base, Req, RawOpts) -> %% `set-cookie' in the base, and `priv/cookie' in the request message). reset(Base, RawOpts) -> Opts = opts(RawOpts), - WithoutBaseCookieKeys = - hb_maps:without( - [<<"cookie">>, <<"set-cookie">>], - Base, - Opts - ), + WithoutBaseCookieKeys = maps:without([<<"cookie">>, <<"set-cookie">>], Base), WithoutPrivCookie = hb_private:set( WithoutBaseCookieKeys, @@ -158,12 +161,15 @@ reset(Base, _Req, Opts) -> %% %% Note that the `format: cookie' form is information lossy: All provided %% attributes and flags are discarded. +-spec to( + #{ cookie => binary() | [binary()], 'set-cookie' => binary() | [binary()], _ => _ }, + #{ format => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ cookie => binary(), 'set-cookie' => [binary()], _ => _ }}. to(Msg, Req, Opts) -> ?event({to, {msg, Msg}, {req, Req}}), CookieOpts = opts(Opts), - LoadedMsg = hb_cache:ensure_all_loaded(Msg, CookieOpts), - ?event({to, {loaded_msg, LoadedMsg}}), - do_to(LoadedMsg, Req, CookieOpts). + do_to(Msg, Req, CookieOpts). do_to(Msg, Req = #{ <<"format">> := <<"set-cookie">> }, Opts) when is_map(Msg) -> ?event({to_set_cookie, {msg, Msg}, {req, Req}}), {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts), @@ -184,15 +190,7 @@ do_to(Msg, Req = #{ <<"format">> := <<"cookie">> }, Opts) when is_map(Msg) -> ?event({to_cookie, {msg, Msg}, {req, Req}}), {ok, ExtractedParsedCookies} = extract(Msg, Req, Opts), {ok, ResetBase} = reset(Msg, Opts), - CookieLines = - hb_maps:values( - hb_maps:map( - fun to_cookie_line/2, - ExtractedParsedCookies, - Opts - ), - Opts - ), + CookieLines = maps:values(maps:map(fun to_cookie_line/2, ExtractedParsedCookies)), ?event({to_cookie, {cookie_lines, CookieLines}}), CookieLine = join(CookieLines, <<"; ">>), {ok, ResetBase#{ <<"cookie">> => CookieLine }}; @@ -258,19 +256,41 @@ to_cookie_line(Key, Cookie) -> %% @doc Normalize a message containing a `cookie', `set-cookie', and potentially %% a `priv/cookie' key into a message with only the `priv/cookie' key. +-spec from( + #{ cookie => binary() | [binary()], 'set-cookie' => binary() | [binary()], _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. from(Msg, Req, Opts) -> CookieOpts = opts(Opts), - LoadedMsg = hb_cache:ensure_all_loaded(Msg, Opts), - do_from(LoadedMsg, Req, CookieOpts). + do_from(load_cookie_fields(Msg, Opts), Req, CookieOpts). + +load_cookie_fields(Msg, Opts) when is_map(Msg) -> + lists:foldl( + fun(Key, Acc) -> + case maps:find(Key, Msg) of + {ok, Value} -> Acc#{ Key => hb_cache:ensure_all_loaded(Value, Opts) }; + error -> Acc + end + end, + Msg, + [<<"cookie">>, <<"set-cookie">>] + ); +load_cookie_fields(Msg, _Opts) -> + Msg. + do_from(Msg, Req, Opts) when is_map(Msg) -> {ok, ResetBase} = reset(Msg, Opts), % Get the cookies, parsed, from each available source. {ok, FromCookie} = from_cookie(Msg, Req, Opts), {ok, FromSetCookie} = from_set_cookie(Msg, Req, Opts), - FromPriv = hb_private:get(<<"cookie">>, Msg, #{}, Opts), + FromPriv = hb_cache:ensure_all_loaded( + hb_private:get(<<"cookie">>, Msg, #{}, Opts), + Opts + ), % Merge all found cookies into a single map. - MergedMsg = hb_maps:merge(FromCookie, FromSetCookie, Opts), - AllParsed = hb_maps:merge(MergedMsg, FromPriv, Opts), + MergedMsg = maps:merge(FromCookie, FromSetCookie), + AllParsed = maps:merge(MergedMsg, FromPriv), % Set the cookies in the private element of the message. {ok, hb_private:set(ResetBase, <<"cookie">>, AllParsed, Opts)}; do_from(CookiesMsg, _Req, _Opts) -> @@ -287,7 +307,7 @@ from_cookie(Cookies, Req, Opts) when is_list(Cookies) -> lists:foldl( fun(Cookie, Acc) -> {ok, Parsed} = from_cookie(Cookie, Req, Opts), - hb_maps:merge(Acc, Parsed, Opts) + maps:merge(Acc, Parsed) end, #{}, Cookies @@ -323,13 +343,13 @@ from_set_cookie(Lines, Req, Opts) when is_list(Lines) -> lists:foldl( fun(Line, Acc) -> {ok, Parsed} = from_set_cookie(Line, Req, Opts), - hb_maps:merge(Acc, Parsed) + maps:merge(Acc, Parsed) end, #{}, Lines ), {ok, MergedParsed}; -from_set_cookie(Line, _Req, Opts) when is_binary(Line) -> +from_set_cookie(Line, _Req, _Opts) when is_binary(Line) -> {[Key, Value], Rest} = split(pair, Line), ValueDecoded = hb_escape:decode(Value), % If there is no remaining binary after the pair, we have a simple key-value @@ -383,7 +403,7 @@ from_set_cookie(Line, _Req, Opts) when is_binary(Line) -> if length(UnquotedFlags) > 0 -> #{ <<"flags">> => UnquotedFlags }; true -> #{} end, - MaybeAllAttributes = hb_maps:merge(MaybeAttributes, MaybeFlags, Opts), + MaybeAllAttributes = maps:merge(MaybeAttributes, MaybeFlags), {ok, #{ Key => MaybeAllAttributes#{ <<"value">> => ValueDecoded }}} end. diff --git a/src/preloaded/auth/dev_cookie_auth.erl b/src/preloaded/auth/dev_cookie_auth.erl index 748d4e2be..7cd986d5e 100644 --- a/src/preloaded/auth/dev_cookie_auth.erl +++ b/src/preloaded/auth/dev_cookie_auth.erl @@ -11,6 +11,7 @@ %% key for the `httpsig@1.0' commitment. If a `committer' is given, we search %% for it in the cookie message instead of generating a new secret. See the %% module documentation of `dev_cookie' for more details on its scheme. +-spec generate(#{ _ => _ }, #{ committer => binary(), generator => _, _ => _ }, map()) -> term(). generate(Base, Request, Opts) -> {WithCookie, Secrets} = case find_secrets(Request, Opts) of @@ -33,11 +34,12 @@ generate(Base, Request, Opts) -> %% messages. The inbound request has the same structure as a normal request %% hook: The message sequence is the body of the request, and the request is %% the request message. +-spec finalize(#{ _ => _ }, #{ request := #{ _ => _ }, body := list(), _ => _ }, map()) -> term(). finalize(Base, Request, Opts) -> ?event(debug_auth, {finalize, {base, Base}, {request, Request}}), maybe - {ok, SignedMsg} ?= hb_maps:find(<<"request">>, Request, Opts), - {ok, MessageSequence} ?= hb_maps:find(<<"body">>, Request, Opts), + SignedMsg = maps:get(<<"request">>, Request), + MessageSequence = maps:get(<<"body">>, Request), % Cookie auth adds set-cookie to response {ok, #{ <<"set-cookie">> := SetCookie }} = dev_cookie:to( @@ -58,6 +60,7 @@ finalize(Base, Request, Opts) -> %% key for the `httpsig@1.0' commitment. If a `committer' is given, we search %% for it in the cookie message instead of generating a new secret. See the %% module documentation of `dev_cookie' for more details on its scheme. +-spec commit(#{ _ => _ }, #{ secret => binary(), committer => binary(), generator => _, _ => _ }, map()) -> term(). commit(Base, Request, RawOpts) when ?IS_LINK(Request) -> Opts = dev_cookie:opts(RawOpts), commit(Base, hb_cache:ensure_loaded(Request, Opts), Opts); @@ -111,6 +114,7 @@ store_secret(Secret, Msg, Opts) -> %% @doc Verify the HMAC commitment with the key being the secret from the %% request cookies. We find the appropriate cookie from the cookie message by %% the committer ID given in the request message. +-spec verify(#{ _ => _ }, #{ secret => binary(), committer => binary(), _ => _ }, map()) -> term(). verify(Base, ReqLink, RawOpts) when ?IS_LINK(ReqLink) -> Opts = dev_cookie:opts(RawOpts), verify(Base, hb_cache:ensure_loaded(ReqLink, Opts), Opts); @@ -150,7 +154,7 @@ verify(Base, Request, RawOpts) -> %% A `generator` may be either a path or full message. If no path is present in %% a generator message, the `generate` path is assumed. generate_secret(_Base, Request, Opts) -> - case hb_maps:get(<<"generator">>, Request, undefined, Opts) of + case maps:get(<<"generator">>, Request, undefined) of undefined -> % If no generator is specified, use the default generator. case hb_opts:get(cookie_default_generator, <<"random">>, Opts) of @@ -172,7 +176,7 @@ default_generator(_Opts) -> execute_generator(GeneratorPath, Opts) when is_binary(GeneratorPath) -> hb_ao:resolve(GeneratorPath, Opts); execute_generator(Generator, Opts) -> - Path = hb_maps:get(<<"path">>, Generator, <<"generate">>, Opts), + Path = maps:get(<<"path">>, Generator, <<"generate">>), hb_ao:resolve(Generator#{ <<"path">> => Path }, Opts). %% @doc Find all secrets in the cookie of a message. @@ -180,9 +184,9 @@ find_secrets(Request, Opts) -> maybe {ok, Cookie} ?= dev_cookie:extract(Request, #{}, Opts), [ - hb_maps:get(SecretRef, Cookie, secret_unavailable, Opts) + maps:get(SecretRef, Cookie, secret_unavailable) || - SecretRef = <<"secret-", _/binary>> <- hb_maps:keys(Cookie) + SecretRef = <<"secret-", _/binary>> <- maps:keys(Cookie) ] else error -> [] end. @@ -190,7 +194,7 @@ find_secrets(Request, Opts) -> %% @doc Find the secret key for the given committer, if it exists in the cookie. find_secret(Request, Opts) -> maybe - {ok, Committer} ?= hb_maps:find(<<"committer">>, Request, Opts), + {ok, Committer} ?= maps:find(<<"committer">>, Request), find_secret(Committer, Request, Opts) else error -> {error, no_secret} end. @@ -224,7 +228,7 @@ directly_invoke_commit_verify_test() -> CommittedMsg, #{} ), - VerifyReqWithoutComms = hb_maps:without([<<"commitments">>], VerifyReq, #{}), + VerifyReqWithoutComms = maps:remove(<<"commitments">>, VerifyReq), ?event({verify_req_without_comms, VerifyReqWithoutComms}), ?assert(hb_message:verify(CommittedMsg, VerifyReqWithoutComms, #{})), ok. diff --git a/src/preloaded/auth/dev_green_zone.erl b/src/preloaded/auth/dev_green_zone.erl index 9fa00cb03..0ff5bc09f 100644 --- a/src/preloaded/auth/dev_green_zone.erl +++ b/src/preloaded/auth/dev_green_zone.erl @@ -18,6 +18,9 @@ %% %% @param _ Ignored parameter %% @returns A map with the `exports' key containing a list of allowed functions +-spec info(#{ _ => _ }) -> #{ exports := [binary()], _ => _ }. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := #{ _ => _ }, _ => _ }}. info(_) -> #{ exports => @@ -92,7 +95,7 @@ info(_Base, _Req, _Opts) -> %% %% @param Opts A map of configuration options from which to derive defaults %% @returns A map of required configuration options for the green zone --spec default_zone_required_opts(Opts :: map()) -> map(). +-spec default_zone_required_opts(#{ _ => _ }) -> #{ _ => _ }. default_zone_required_opts(_Opts) -> #{ % <<"trusted-device-signers">> => @@ -114,7 +117,7 @@ default_zone_required_opts(_Opts) -> %% @param Config The configuration map to process %% @param Opts The options map to fetch replacement values from %% @returns A new map with <<"self">> values replaced --spec replace_self_values(Config :: map(), Opts :: map()) -> map(). +-spec replace_self_values(#{ _ => _ }, #{ _ => _ }) -> #{ _ => _ }. replace_self_values(Config, Opts) -> maps:map( fun(Key, Value) -> @@ -129,6 +132,7 @@ replace_self_values(Config, Opts) -> ). %% @doc Returns `true' if the request is signed by a trusted node. +-spec is_trusted(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()}. is_trusted(_M1, Req, Opts) -> Signers = hb_message:signers(Req, Opts), {ok, @@ -164,7 +168,7 @@ is_trusted(_M1, Req, Opts) -> %% @param Opts A map of configuration options %% @returns `{ok, Binary}' on success with confirmation message, or %% `{error, Binary}' on failure with error message. --spec init(M1 :: term(), M2 :: term(), Opts :: map()) -> {ok, binary()} | {error, binary()}. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()} | {error, binary()}. init(_M1, _M2, Opts) -> ?event(green_zone, {init, start}), case hb_opts:get(green_zone_initialized, false, Opts) of @@ -235,8 +239,7 @@ init(_M1, _M2, Opts) -> %% @param Opts A map of configuration options for join operations %% @returns `{ok, Map}' on success with join response details, or %% `{error, Binary}' on failure with error message. --spec join(M1 :: term(), M2 :: term(), Opts :: map()) -> - {ok, map()} | {error, binary()}. +-spec join(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. join(M1, M2, Opts) -> ?event(green_zone, {join, start}), PeerLocation = hb_opts:get(<<"green-zone-peer-location">>, undefined, Opts), @@ -268,8 +271,8 @@ join(M1, M2, Opts) -> %% @param Opts A map of configuration options %% @returns `{ok, Map}' containing the encrypted key and IV on success, or %% `{error, Binary}' if the node is not part of a green zone --spec key(M1 :: term(), M2 :: term(), Opts :: map()) -> - {ok, map()} | {error, binary()}. +-spec key(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), encrypted_key := binary(), iv := binary(), _ => _ }} | {error, binary()}. key(_M1, _M2, Opts) -> ?event(green_zone, {get_key, start}), % Retrieve the shared AES key and the node's wallet. @@ -330,8 +333,7 @@ key(_M1, _M2, Opts) -> %% @returns `{ok, Map}' on success with confirmation details, or %% `{error, Binary}' if the node is not part of a green zone or %% identity adoption fails. --spec become(M1 :: term(), M2 :: term(), Opts :: map()) -> - {ok, map()} | {error, binary()}. +-spec become(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, binary()}. become(_M1, _M2, Opts) -> ?event(green_zone, {become, start}), % 1. Retrieve the target node's address from the incoming message. @@ -435,9 +437,9 @@ finalize_become(KeyResp, NodeLocation, NodeID, GreenZoneAES, Opts) -> -spec join_peer( PeerLocation :: binary(), PeerID :: binary(), - M1 :: term(), - M2 :: term(), - Opts :: map()) -> {ok, map()} | {error, map() | binary()}. + _, + _, + _) -> {ok, _} | {error, _}. join_peer(PeerLocation, PeerID, _M1, _M2, InitOpts) -> % Check here if the node is already part of a green zone. GreenZoneAES = hb_opts:get(priv_green_zone_aes, undefined, InitOpts), @@ -521,11 +523,7 @@ join_peer(PeerLocation, PeerID, _M1, _M2, InitOpts) -> end; false -> ?event(green_zone, {join, already_joined}), - {error, <<"Node already part of green zone.">>}; - {error, Reason} -> - % Log the error and return the initial options. - ?event(green_zone, {join, error, Reason}), - {error, Reason} + {error, <<"Node already part of green zone.">>} end. %%%-------------------------------------------------------------------- @@ -548,8 +546,7 @@ join_peer(PeerLocation, PeerID, _M1, _M2, InitOpts) -> %% @param Opts A map of configuration options %% @returns `{ok, Map}' on success with encrypted AES key, or %% `{error, Binary}' on failure with error message --spec validate_join(M1 :: term(), Req :: map(), Opts :: map()) -> - {ok, map()} | {error, binary()}. +-spec validate_join(_, _, _) -> {ok, _} | {error, binary()}. validate_join(M1, Req, Opts) -> case validate_peer_opts(Req, Opts) of true -> do_nothing; @@ -620,7 +617,7 @@ validate_join(M1, Req, Opts) -> %% @param Req The request message containing the peer's configuration %% @param Opts A map of the local node's configuration options %% @returns true if the peer's configuration is valid, false otherwise --spec validate_peer_opts(Req :: map(), Opts :: map()) -> boolean(). +-spec validate_peer_opts(_, _) -> boolean(). validate_peer_opts(Req, Opts) -> ?event(green_zone, {validate_peer_opts, start, Req}), % Get the required config from the local node's configuration. @@ -669,10 +666,7 @@ validate_peer_opts(Req, Opts) -> %% @param RequesterPubKey The joining node's public key %% @param Opts A map of configuration options %% @returns ok --spec add_trusted_node( - NodeAddr :: binary(), - Report :: map(), - RequesterPubKey :: term(), Opts :: map()) -> ok. +-spec add_trusted_node(_, _, _, _) -> ok. add_trusted_node(NodeAddr, Report, RequesterPubKey, Opts) -> % Retrieve the current trusted nodes map. TrustedNodes = hb_opts:get(trusted_nodes, #{}, Opts), @@ -696,7 +690,7 @@ add_trusted_node(NodeAddr, Report, RequesterPubKey, Opts) -> %% @param AESKey The shared AES key (256-bit binary) %% @param RequesterPubKey The node's public RSA key %% @returns The encrypted AES key --spec encrypt_payload(AESKey :: binary(), RequesterPubKey :: term()) -> binary(). +-spec encrypt_payload(binary(), _) -> binary(). encrypt_payload(AESKey, RequesterPubKey) -> ?event(green_zone, {encrypt_payload, start}), %% Expect RequesterPubKey in the form: { {rsa, E}, Pub } @@ -720,8 +714,7 @@ encrypt_payload(AESKey, RequesterPubKey) -> %% @param EncZoneKey The encrypted zone AES key (Base64 encoded or binary) %% @param Opts A map of configuration options %% @returns {ok, DecryptedKey} on success with the decrypted AES key --spec decrypt_zone_key(EncZoneKey :: binary(), Opts :: map()) -> - {ok, binary()} | {error, binary()}. +-spec decrypt_zone_key(binary(), _) -> {ok, binary()} | {error, binary()}. decrypt_zone_key(EncZoneKey, Opts) -> % Decode if necessary RawEncKey = case is_binary(EncZoneKey) of diff --git a/src/preloaded/auth/dev_http_auth.erl b/src/preloaded/auth/dev_http_auth.erl index 091eca742..514923df2 100644 --- a/src/preloaded/auth/dev_http_auth.erl +++ b/src/preloaded/auth/dev_http_auth.erl @@ -46,6 +46,19 @@ %% @doc Generate or extract a new secret and commit to the message with the %% `~httpsig@1.0/commit?type=hmac-sha256&scheme=secret' commitment mechanism. +-spec commit(_, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. commit(Base, Req, Opts) -> case generate(Base, Req, Opts) of {ok, Key} -> @@ -68,6 +81,19 @@ commit(Base, Req, Opts) -> %% @doc Verify a given `Base' message with a derived `Key' using the %% `~httpsig@1.0' secret key HMAC commitment scheme. +-spec verify(_, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, boolean()}. verify(Base, RawReq, Opts) -> ?event({verify_invoked, {base, Base}, {req, RawReq}}), {ok, Key} = generate(Base, RawReq, Opts), @@ -88,18 +114,29 @@ verify(Base, RawReq, Opts) -> %% @doc Collect authentication information from the client. If the `raw' flag %% is set to `true', return the raw authentication information. Otherwise, %% derive a key from the authentication information and return it. -generate(_Msg, ReqLink, Opts) when ?IS_LINK(ReqLink) -> - generate(_Msg, hb_cache:ensure_loaded(ReqLink, Opts), Opts); +-spec generate(_, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, binary()} | {error, #{ status := integer(), _ => _ }}. generate(_Msg, #{ <<"secret">> := Secret }, _Opts) -> {ok, Secret}; -generate(_Msg, Req, Opts) -> - case hb_maps:get(<<"authorization">>, Req, undefined, Opts) of +generate(_Msg, Req, _Opts) -> + case maps:get(<<"authorization">>, Req, undefined) of <<"Basic ", Auth/binary>> -> Decoded = base64:decode(Auth), ?event(key_gen, {generated_key, {auth, Auth}, {decoded, Decoded}}), - case hb_maps:get(<<"raw">>, Req, false, Opts) of + case maps:get(<<"raw">>, Req, false) of true -> {ok, Decoded}; - false -> derive_key(Decoded, Req, Opts) + false -> derive_key(Decoded, Req, _Opts) end; undefined -> {error, @@ -121,17 +158,11 @@ generate(_Msg, Req, Opts) -> %% @doc Derive a key from the authentication information using the PBKDF2 %% algorithm and user specified parameters. -derive_key(Decoded, Req, Opts) -> - Alg = hb_util:atom(hb_maps:get(<<"alg">>, Req, <<"sha256">>, Opts)), - Salt = - hb_maps:get( - <<"salt">>, - Req, - hb_crypto:sha256(?DEFAULT_SALT), - Opts - ), - Iterations = hb_maps:get(<<"iterations">>, Req, 2 * 600_000, Opts), - KeyLength = hb_maps:get(<<"key-length">>, Req, 64, Opts), +derive_key(Decoded, Req, _Opts) -> + Alg = maps:get(<<"alg">>, Req, sha256), + Salt = maps:get(<<"salt">>, Req, hb_crypto:sha256(?DEFAULT_SALT)), + Iterations = maps:get(<<"iterations">>, Req, 2 * 600_000), + KeyLength = maps:get(<<"key-length">>, Req, 64), ?event(key_gen, {derive_key, {alg, Alg}, diff --git a/src/preloaded/auth/dev_secret.erl b/src/preloaded/auth/dev_secret.erl index 98c184ed0..ff01f1622 100644 --- a/src/preloaded/auth/dev_secret.erl +++ b/src/preloaded/auth/dev_secret.erl @@ -174,6 +174,8 @@ %% @doc Generate a new wallet for a user and register it on the node. If the %% `committer' field is provided, we first check whether there is a wallet %% already registered for it. If there is, we return the wallet details. +-spec generate(#{ _ => _ }, #{ committer => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, _}. generate(Base, Request, Opts) -> case request_to_wallets(Base, Request, Opts) of [] -> @@ -199,6 +201,8 @@ generate(Base, Request, Opts) -> %% @doc Import a wallet for hosting on the node. Expects the keys to be either %% provided as a list of keys, or a single key in the `key' field. If neither %% are provided, the keys are extracted from the cookie. +-spec import(#{ _ => _ }, #{ key => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. import(Base, Request, Opts) -> Wallets = case hb_maps:find(<<"key">>, Request, Opts) of @@ -388,10 +392,12 @@ persist_registered_wallet(WalletDetails, RespBase, Opts) -> end. %% @doc List all hosted wallets +-spec list(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]}. list(_Base, _Request, Opts) -> {ok, list_wallets(Opts)}. %% @doc Sign a message with a wallet. +-spec commit(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, binary()}. commit(Base, Request, Opts) -> ?event({commit_invoked, {base, Base}, {request, Request}}), case request_to_wallets(Base, Request, Opts) of @@ -577,6 +583,8 @@ commit_message(Message, #{ <<"wallet">> := Key }, Opts) -> %% @doc Export wallets from a request. The request should contain a source of %% wallets (cookies, keys, or wallet names), or a specific list/name of a %% wallet to authenticate and export. +-spec export(#{ _ => _ }, #{ keyids => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, binary()}. export(Base, Request, Opts) -> PrivOpts = priv_store_opts(Opts), ModReq = @@ -604,6 +612,8 @@ export(Base, Request, Opts) -> end. %% @doc Sync wallets from a remote node +-spec sync(#{ _ => _ }, #{ node => binary(), as => binary(), keyids => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, _}. sync(_Base, Request, Opts) -> case hb_ao:get(<<"node">>, Request, undefined, Opts) of undefined -> diff --git a/src/preloaded/auth/dev_snp.erl b/src/preloaded/auth/dev_snp.erl index 57d064059..2593574d2 100644 --- a/src/preloaded/auth/dev_snp.erl +++ b/src/preloaded/auth/dev_snp.erl @@ -56,8 +56,7 @@ %% @param NodeOpts A map of configuration options for verification %% @returns `{ok, Binary}' with "true" on successful verification, or %% `{error, Reason}' on failure with specific error details --spec verify(M1 :: term(), M2 :: term(), NodeOpts :: map()) -> - {ok, binary()} | {error, term()}. +-spec verify(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()} | {error, _}. verify(M1, M2, NodeOpts) -> ?event(snp_verify, verify_called), maybe @@ -123,8 +122,7 @@ verify(M1, M2, NodeOpts) -> %% @param Opts A map of configuration options for report generation %% @returns `{ok, Map}' on success with the complete report message, or %% `{error, Reason}' on failure with error details --spec generate(M1 :: term(), M2 :: term(), Opts :: map()) -> - {ok, map()} | {error, term()}. +-spec generate(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. generate(_M1, _M2, Opts) -> maybe LoadedOpts = hb_cache:ensure_all_loaded(Opts, Opts), @@ -201,8 +199,8 @@ generate(_M1, _M2, Opts) -> %% @param NodeOpts A map of configuration options %% @returns `{ok, {Msg, Address, NodeMsgID, ReportJSON, MsgWithJSONReport}}' %% on success with all extracted components, or `{error, Reason}' on failure --spec extract_and_normalize_message(M2 :: term(), NodeOpts :: map()) -> - {ok, {map(), binary(), binary(), binary(), map()}} | {error, term()}. +-spec extract_and_normalize_message(_, _) -> + {ok, {_, binary(), binary(), binary(), _}} | {error, _}. extract_and_normalize_message(M2, NodeOpts) -> maybe % Search for a `body' key in the message, and if found use it as the source @@ -269,8 +267,7 @@ extract_and_normalize_message(M2, NodeOpts) -> %% @param NodeOpts A map of configuration options %% @returns `{ok, NodeMsgID}' on success with the extracted ID, or %% `{error, missing_node_msg_id}' if no ID can be found --spec extract_node_message_id(Msg :: map(), NodeOpts :: map()) -> - {ok, binary()} | {error, missing_node_msg_id}. +-spec extract_node_message_id(_, _) -> {ok, binary()} | {error, missing_node_msg_id}. extract_node_message_id(Msg, NodeOpts) -> case {hb_ao:get(<<"node-message">>, Msg, NodeOpts#{ <<"hashpath">> => ignore }), hb_ao:get(<<"node-message-id">>, Msg, NodeOpts)} of @@ -293,8 +290,7 @@ extract_node_message_id(Msg, NodeOpts) -> %% @param Msg The normalized SNP message containing the nonce %% @param NodeOpts A map of configuration options %% @returns `{ok, true}' if the nonce matches, or `{error, nonce_mismatch}' on failure --spec verify_nonce(Address :: binary(), NodeMsgID :: binary(), - Msg :: map(), NodeOpts :: map()) -> {ok, true} | {error, nonce_mismatch}. +-spec verify_nonce(binary(), binary(), _, _) -> {ok, true} | {error, nonce_mismatch}. verify_nonce(Address, NodeMsgID, Msg, NodeOpts) -> Nonce = hb_util:decode(hb_ao:get(<<"nonce">>, Msg, NodeOpts)), ?event({snp_nonce, Nonce}), @@ -316,8 +312,7 @@ verify_nonce(Address, NodeMsgID, Msg, NodeOpts) -> %% @param NodeOpts A map of configuration options %% @returns `{ok, true}' if both signature and address are valid, or %% `{error, signature_or_address_invalid}' on failure --spec verify_signature_and_address(MsgWithJSONReport :: map(), - Address :: binary(), NodeOpts :: map()) -> +-spec verify_signature_and_address(_, binary(), _) -> {ok, true} | {error, signature_or_address_invalid}. verify_signature_and_address(MsgWithJSONReport, Address, NodeOpts) -> Signers = hb_message:signers(MsgWithJSONReport, NodeOpts), @@ -338,7 +333,7 @@ verify_signature_and_address(MsgWithJSONReport, Address, NodeOpts) -> %% %% @param Msg The normalized SNP message containing the policy %% @returns `{ok, true}' if debug is disabled, or `{error, debug_enabled}' if enabled --spec verify_debug_disabled(Msg :: map()) -> {ok, true} | {error, debug_enabled}. +-spec verify_debug_disabled(_) -> {ok, true} | {error, debug_enabled}. verify_debug_disabled(Msg) -> DebugDisabled = not is_debug(Msg), ?event({debug_disabled, DebugDisabled}), @@ -358,8 +353,7 @@ verify_debug_disabled(Msg) -> %% @param NodeOpts A map of configuration options including trusted software list %% @returns `{ok, true}' if the software is trusted, or `{error, untrusted_software}' %% on failure --spec verify_trusted_software(M1 :: term(), Msg :: map(), NodeOpts :: map()) -> - {ok, true} | {error, untrusted_software}. +-spec verify_trusted_software(_, _, _) -> {ok, true} | {error, untrusted_software}. verify_trusted_software(M1, Msg, NodeOpts) -> {ok, IsTrustedSoftware} = execute_is_trusted(M1, Msg, NodeOpts), ?event({trusted_software, IsTrustedSoftware}), @@ -380,8 +374,7 @@ verify_trusted_software(M1, Msg, NodeOpts) -> %% @param NodeOpts A map of configuration options %% @returns `{ok, true}' if the measurement is valid, or %% `{error, measurement_invalid}' on failure --spec verify_measurement(Msg :: map(), ReportJSON :: binary(), - NodeOpts :: map()) -> {ok, true} | {error, measurement_invalid}. +-spec verify_measurement(_, binary(), _) -> {ok, true} | {error, measurement_invalid}. verify_measurement(Msg, ReportJSON, NodeOpts) -> Args = extract_measurement_args(Msg, NodeOpts), ?event({args, { explicit, Args}}), @@ -410,7 +403,8 @@ verify_measurement(Msg, ReportJSON, NodeOpts) -> %% @param Msg The normalized SNP message containing local hashes %% @param NodeOpts A map of configuration options %% @returns A map of measurement arguments with atom keys --spec extract_measurement_args(Msg :: map(), NodeOpts :: map()) -> map(). +-spec extract_measurement_args(#{ 'local-hashes' => #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + #{ _ => _ }. extract_measurement_args(Msg, NodeOpts) -> LocalHashes = hb_cache:ensure_all_loaded( @@ -458,7 +452,7 @@ verify_report_integrity(ReportJSON) -> %% %% @param Report The SNP report containing the policy field %% @returns `true' if debug mode is enabled, `false' otherwise --spec is_debug(Report :: map()) -> boolean(). +-spec is_debug(_) -> boolean(). is_debug(Report) -> (hb_ao:get(<<"policy">>, Report, #{}) band (1 bsl ?DEBUG_FLAG_BIT)) =/= 0. @@ -481,8 +475,7 @@ is_debug(Report) -> %% @param Msg The SNP message containing local software hashes %% @param NodeOpts A map of configuration options including trusted software %% @returns `{ok, true}' if software is trusted, `{ok, false}' otherwise --spec execute_is_trusted(M1 :: term(), Msg :: map(), NodeOpts :: map()) -> - {ok, boolean()}. +-spec execute_is_trusted(_, _, _) -> {ok, boolean()}. execute_is_trusted(_M1, Msg, NodeOpts) -> FilteredLocalHashes = get_filtered_local_hashes(Msg, NodeOpts), TrustedSoftware = hb_opts:get(snp_trusted, [#{}], NodeOpts), @@ -504,7 +497,8 @@ execute_is_trusted(_M1, Msg, NodeOpts) -> %% @param Msg The SNP message containing local hashes %% @param NodeOpts A map of configuration options %% @returns A map of filtered local hashes with only enforced keys --spec get_filtered_local_hashes(Msg :: map(), NodeOpts :: map()) -> map(). +-spec get_filtered_local_hashes(#{ 'local-hashes' => #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + #{ _ => _ }. get_filtered_local_hashes(Msg, NodeOpts) -> LocalHashes = hb_ao:get(<<"local-hashes">>, Msg, NodeOpts), EnforcedKeys = get_enforced_keys(NodeOpts), @@ -523,7 +517,7 @@ get_filtered_local_hashes(Msg, NodeOpts) -> %% %% @param NodeOpts A map of configuration options %% @returns A list of binary keys that should be enforced --spec get_enforced_keys(NodeOpts :: map()) -> [binary()]. +-spec get_enforced_keys(_) -> [binary()]. get_enforced_keys(NodeOpts) -> lists:map( fun canonical_hash_key/1, @@ -554,7 +548,7 @@ canonical_hash_key(Key) when is_binary(Key) -> %% @param TrustedSoftware List of trusted software configurations or invalid input %% @param NodeOpts Configuration options for matching %% @returns `true' if hashes match a trusted configuration, `false' otherwise --spec is_software_trusted(map(), [] | [map()] | term(), map()) -> boolean(). +-spec is_software_trusted(_, _, _) -> boolean(). is_software_trusted(_FilteredLocalHashes, [], _NodeOpts) -> false; is_software_trusted(FilteredLocalHashes, TrustedSoftware, NodeOpts) diff --git a/src/preloaded/codec/dev_ans104.erl b/src/preloaded/codec/dev_ans104.erl index 2b4b5facc..eaeda9f06 100644 --- a/src/preloaded/codec/dev_ans104.erl +++ b/src/preloaded/codec/dev_ans104.erl @@ -13,12 +13,15 @@ content_type(_) -> {ok, <<"application/ans104">>}. %% @doc Serialize a message or TX to a binary. +-spec serialize(binary() | #tx{} | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()}. serialize(Msg, Req, Opts) when is_map(Msg) -> serialize(to(Msg, Req, Opts), Req, Opts); serialize(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, ar_bundles:serialize(TX)}. %% @doc Deserialize a binary ans104 message to a TABM. +-spec deserialize(binary() | #tx{} | #{ body := binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }}. deserialize(#{ <<"body">> := Binary }, Req, Opts) -> deserialize(Binary, Req, Opts); deserialize(Binary, Req, Opts) when is_binary(Binary) -> @@ -29,6 +32,7 @@ deserialize(TX, Req, Opts) when is_record(TX, tx) -> %% @doc Sign a message using the `priv-wallet' key in the options. Supports both %% the `hmac-sha256' and `rsa-pss-sha256' algorithms, offering unsigned and %% signed commitments. +-spec commit(#{ _ => _ }, #{ type := binary(), _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -58,7 +62,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> { ok, hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), + maps:remove(<<"commitments">>, Msg), <<"ans104@1.0">>, <<"structured@1.0">>, Opts @@ -77,6 +81,7 @@ sign_tx(TX, Wallet, Opts) -> {ok, SignedStructured}. %% @doc Verify an ANS-104 commitment. +-spec verify(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. verify(Msg, Req, Opts) -> ?event({verify, {base, Msg}, {req, Req}}), OnlyWithCommitment = @@ -94,6 +99,7 @@ verify(Msg, Req, Opts) -> {ok, Res}. %% @doc Convert a #tx record into a message map recursively. +-spec from(binary() | #tx{}, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; from(TX, Req, Opts) when is_record(TX, tx) -> case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of @@ -133,6 +139,8 @@ do_from(RawTX, Req, Opts) -> %% message's device in order to get the keys that we will be checkpointing. We %% do this recursively to handle nested messages. The base case is that we hit %% a binary, which we return as is. +-spec to(binary() | #tx{} | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #tx{}}. to(Binary, _Req, _Opts) when is_binary(Binary) -> % ar_bundles cannot serialize just a simple binary or get an ID for it, so % we turn it into a TX record with a special tag, tx_to_message will diff --git a/src/preloaded/codec/dev_flat.erl b/src/preloaded/codec/dev_flat.erl index ec0da1117..fc9f6cfba 100644 --- a/src/preloaded/codec/dev_flat.erl +++ b/src/preloaded/codec/dev_flat.erl @@ -9,6 +9,7 @@ -include("include/hb.hrl"). %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -19,6 +20,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -29,6 +31,7 @@ verify(Msg, Req, Opts) -> }. %% @doc Convert a flat map to a TABM. +-spec from(binary() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; from(Map, Req, Opts) when is_map(Map) -> {ok, @@ -59,6 +62,8 @@ from(Map, Req, Opts) when is_map(Map) -> }. %% @doc Convert a TABM to a flat map. +-spec to(binary() | [_] | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }}. to(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; to(List, Req, Opts) when is_list(List) -> to( diff --git a/src/preloaded/codec/dev_gzip.erl b/src/preloaded/codec/dev_gzip.erl index 298125dcf..46c980201 100644 --- a/src/preloaded/codec/dev_gzip.erl +++ b/src/preloaded/codec/dev_gzip.erl @@ -8,10 +8,12 @@ %% containting a gzip-encoded payload. Returns the rest of the base message %% unchanged, with the `content-encoding' key unset. %% +-spec unzip(#{ body => binary(), 'content-encoding' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body => binary(), _ => _ }}. unzip(Base, _Req, Opts) -> - case hb_maps:get(<<"content-encoding">>, Base, <<"gzip">>, Opts) of + case maps:get(<<"content-encoding">>, Base, <<"gzip">>) of <<"gzip">> -> - case hb_maps:find(<<"body">>, Base, Opts) of + case maps:find(<<"body">>, Base) of error -> ?event( debug_gzip, @@ -25,17 +27,11 @@ unzip(Base, _Req, Opts) -> {unzipping_body, {size, byte_size(Body)}}, Opts ), - { - ok, - hb_ao:set( - Base, - #{ - <<"body">> => zlib:gunzip(Body), - <<"content-encoding">> => unset - }, - Opts - ) - } + {ok, + maps:remove( + <<"content-encoding">>, + Base#{ <<"body">> => zlib:gunzip(Body) } + )} end; _ -> ?event( @@ -48,20 +44,16 @@ unzip(Base, _Req, Opts) -> %% @doc Take a base message with a `body' key and return it zipped, in-place. %% Add a `content-encoding' key with the value `gzip'. -zip(Base, _Req, Opts) -> - case hb_maps:find(<<"body">>, Base, Opts) of +-spec zip(#{ body => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), 'content-encoding' := binary(), _ => _ }} | {error, binary()}. +zip(Base, _Req, _Opts) -> + case maps:find(<<"body">>, Base) of {ok, Body} -> - { - ok, - hb_ao:set( - Base, - #{ - <<"body">> => zlib:gzip(Body), - <<"content-encoding">> => <<"gzip">> - }, - Opts - ) - }; + {ok, + Base#{ + <<"body">> => zlib:gzip(Body), + <<"content-encoding">> => <<"gzip">> + }}; error -> {error, <<"No `body' key to zip found in message.">>} end. @@ -79,4 +71,4 @@ unzip_encoded_response_test() -> <>, Opts ), - ?assertEqual(<<"Hello, world!">>, Unzipped). \ No newline at end of file + ?assertEqual(<<"Hello, world!">>, Unzipped). diff --git a/src/preloaded/codec/dev_httpsig.erl b/src/preloaded/codec/dev_httpsig.erl index f4ad1d8cc..bd08465e0 100644 --- a/src/preloaded/codec/dev_httpsig.erl +++ b/src/preloaded/codec/dev_httpsig.erl @@ -20,7 +20,9 @@ -include_lib("eunit/include/eunit.hrl"). %%% Routing functions for the `dev_httpsig_conv' module +-spec to(#{ _ => _ }, #{ _ => _ }, map()) -> term(). to(Msg, Req, Opts) -> dev_httpsig_conv:to(Msg, Req, Opts). +-spec from(#{ _ => _ }, #{ _ => _ }, map()) -> term(). from(Msg, Req, Opts) -> dev_httpsig_conv:from(Msg, Req, Opts). %% @doc Generate the `Opts' to use during AO-Core operations in the codec. @@ -61,6 +63,8 @@ proxy_verify(_Base, Req, Opts) -> %% %% Optionally, the `index` key can be set to override resolution of the default %% index page into HTTP responses that do not contain their own `body` field. +-spec serialize(#{ _ => _ }, #{ format => binary(), index => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ headers := #{ _ => _ }, body := _, _ => _ }}. serialize(Msg, Opts) -> serialize(Msg, #{}, Opts). serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) -> % Convert to HTTPSig via TABM through calling `hb_message:convert` rather @@ -69,8 +73,8 @@ serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) -> {ok, EncMsg} = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), {ok, #{ - <<"body">> => hb_maps:get(<<"body">>, EncMsg, <<>>), - <<"headers">> => hb_maps:without([<<"body">>], EncMsg) + <<"body">> => maps:get(<<"body">>, EncMsg, <<>>), + <<"headers">> => maps:remove(<<"body">>, EncMsg) } }; serialize(Msg, _Req, Opts) -> @@ -79,6 +83,11 @@ serialize(Msg, _Req, Opts) -> HTTPSig = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), {ok, dev_httpsig_conv:encode_http_msg(HTTPSig, Opts) }. +-spec verify( + #{ _ => _ }, + #{ signature := binary(), type := binary(), _ => _ }, + #{ _ => _ } +) -> {ok, boolean()} | {failure, _}. verify(Base, Req, RawOpts) -> % A rsa-pss-sha512 commitment is verified by regenerating the signature % base and validating against the signature. @@ -137,6 +146,11 @@ verify(Base, Req, RawOpts) -> %% parameter to determine the type of commitment to use. If the `type' parameter %% is `signed', we default to the rsa-pss-sha512 algorithm. If the `type' %% parameter is `unsigned', we default to the hmac-sha256 algorithm. +-spec commit( + #{ _ => _ }, + #{ type := binary(), bundle => boolean(), committed => [_], _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"hmac-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -336,6 +350,8 @@ add_content_digest(Msg, _Opts) -> %% @doc Given a base message and a commitment, derive the message and commitment %% normalized for encoding. +-spec normalize_for_encoding(#{ _ => _ }, #{ committed => [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }, #{ committed := [_], _ => _ }, [_]}. normalize_for_encoding(Msg, Commitment, Opts) -> % Extract the requested keys to include in the signature base. RawInputs = diff --git a/src/preloaded/codec/dev_httpsig_conv.erl b/src/preloaded/codec/dev_httpsig_conv.erl index 950b32f8a..25bbad433 100644 --- a/src/preloaded/codec/dev_httpsig_conv.erl +++ b/src/preloaded/codec/dev_httpsig_conv.erl @@ -376,7 +376,7 @@ to(TABM, Req, FormatOpts, Opts) when is_map(TABM) -> % Convert back to the fully loaded structured@1.0 message, then % convert to TABM with bundling enabled. Structured = hb_message:convert(TABM, <<"structured@1.0">>, Opts), - Loaded = hb_cache:ensure_all_loaded(Structured, Opts), + Loaded = load_bundle_message(Structured, Opts), encode_ids( hb_message:convert( Loaded, @@ -445,6 +445,64 @@ to(TABM, Req, FormatOpts, Opts) when is_map(TABM) -> ) }. +load_bundle_message(Structured, Opts) -> + PreserveKeys = [<<"commitments">>, <<"hashpath">>, <<"priv">>, <<"process">>], + Preserved = maps:filter(fun(Key, _Value) -> + lists:member(Key, PreserveKeys) + end, Structured), + Loaded = + hb_cache:ensure_all_loaded( + maps:filter(fun(Key, _Value) -> + not lists:member(Key, PreserveKeys) + end, Structured), + Opts + ), + link_bundle_provenance(maps:merge(Loaded, Preserved), Opts). + +link_bundle_provenance(Msg, Opts) when is_map(Msg) -> + maps:from_list(lists:flatmap( + fun({Key, Value}) -> + case lists:member(Key, [<<"commitments">>, <<"priv">>]) of + true -> + [{Key, Value}]; + false -> + case bundle_link_id(Key, Value, Opts) of + {ok, ID} -> + [{<<(hb_util:bin(Key))/binary, "+link">>, ID}]; + false -> + [{Key, link_bundle_provenance(Value, Opts)}] + end + end + end, + maps:to_list(Msg) + )); +link_bundle_provenance(List, Opts) when is_list(List) -> + lists:map(fun(Value) -> link_bundle_provenance(Value, Opts) end, List); +link_bundle_provenance(Value, _Opts) -> + Value. + +bundle_link_id(Key, Link, Opts) when ?IS_LINK(Link) -> + case Key == <<"process">> of + true -> {ok, link_id(Link, Opts)}; + false -> false + end; +bundle_link_id(Key, Value, Opts) when is_map(Value) -> + case {Key, maps:get(<<"commitments">>, Value, undefined)} of + {<<"process">>, Commitments} when is_map(Commitments), map_size(Commitments) > 0 -> + {ok, hb_message:id(Value, all, Opts)}; + _ -> + false + end; +bundle_link_id(_Key, _Value, _Opts) -> + false. + +link_id({link, ID, #{ <<"type">> := <<"link">>, <<"lazy">> := false }}, _Opts) -> + ID; +link_id({link, ID, #{ <<"type">> := <<"link">>, <<"lazy">> := true }}, Opts) -> + hb_util:ok(hb_cache:read(ID, Opts)); +link_id({link, ID, _LinkOpts}, _Opts) -> + ID. + do_to(Binary, _FormatOpts, _Opts) when is_binary(Binary) -> Binary; do_to(TABM, FormatOpts, Opts) when is_map(TABM) -> InlineKey = diff --git a/src/preloaded/codec/dev_json.erl b/src/preloaded/codec/dev_json.erl index dffff0820..b1bc7f720 100644 --- a/src/preloaded/codec/dev_json.erl +++ b/src/preloaded/codec/dev_json.erl @@ -11,6 +11,7 @@ content_type(_) -> {ok, <<"application/json">>}. %% @doc Encode a message to a JSON string, using JSON-native typing. +-spec to(binary() | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> {ok, binary()}. to(Msg, _Req, _Opts) when is_binary(Msg) -> {ok, hb_util:bin(json:encode(Msg))}; to(Msg, Req, Opts) -> @@ -27,9 +28,10 @@ to(Msg, Req, Opts) -> tabm, ConvOpts ), + Bundle = hb_maps:get(<<"bundle">>, Req, false, Opts), Loaded = - case hb_maps:get(<<"bundle">>, Req, false, Opts) of - true -> hb_cache:ensure_all_loaded(Restructured, Opts); + case Bundle of + true -> load_available_links(hb_link:decode_all_links(Restructured), Opts); false -> Restructured end, JSONStructured = @@ -38,13 +40,40 @@ to(Msg, Req, Opts) -> tabm, #{ <<"device">> => <<"structured@1.0">>, + <<"bundle">> => Bundle, <<"encode-types">> => [<<"atom">>] }, ConvOpts ), {ok, hb_json:encode(JSONStructured)}. +%% @doc Eager-load resolvable links for bundled JSON responses, while leaving +%% missing lazy links in place so optional response fields do not fail encoding. +load_available_links(Msg, Opts) -> + load_available_links([], Msg, Opts). + +load_available_links(_Ref, Link, Opts) when ?IS_LINK(Link) -> + try hb_cache:ensure_loaded(Link, Opts) of + Loaded -> load_available_links([], Loaded, Opts) + catch + throw:{necessary_message_not_found, _, _} -> Link + end; +load_available_links(Ref, Msg, Opts) when is_map(Msg) -> + maps:map( + fun(K, V) -> load_available_links([K|Ref], V, Opts) end, + Msg + ); +load_available_links(Ref, Msg, Opts) when is_list(Msg) -> + lists:map( + fun({N, V}) -> load_available_links([N|Ref], V, Opts) end, + hb_util:number(Msg) + ); +load_available_links(_Ref, Msg, _Opts) -> + Msg. + %% @doc Decode a JSON string to a message. +-spec from(binary() | #{ _ => _ }, #{ 'accept-codec' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. from(Map, _Req, _Opts) when is_map(Map) -> {ok, Map}; from(JSON, Req, Opts) -> ConvOpts = Opts#{ <<"hashpath">> => ignore }, @@ -61,7 +90,7 @@ from(JSON, Req, Opts) -> ConvOpts ), ?event(debug_json, {structured, Structured}, Opts), - case hb_maps:get(<<"accept-codec">>, Req, undefined, Opts) of + case maps:get(<<"accept-codec">>, Req, undefined) of <<"structured@1.0">> -> {ok, Structured}; _ -> % Re-encode the structured message back to TABM for the caller. @@ -77,6 +106,7 @@ from(JSON, Req, Opts) -> end. %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -87,6 +117,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -96,25 +127,18 @@ verify(Msg, Req, Opts) -> ) }. +-spec committed(binary() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> [binary()]. committed(Msg, Req, Opts) when is_binary(Msg) -> committed(hb_util:ok(from(Msg, Req, Opts)), Req, Opts); committed(Msg, _Req, Opts) -> hb_message:committed(Msg, all, Opts). %% @doc Deserialize the JSON string found at the given path. +-spec deserialize(#{ _ => _ }, #{ target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. deserialize(Base, Req, Opts) -> - Payload = - hb_ao:get( - Target = - hb_ao:get( - <<"target">>, - Req, - <<"body">>, - Opts - ), - Base, - Opts - ), + Target = maps:get(<<"target">>, Req, <<"body">>), + Payload = hb_ao:get(Target, Base, Opts), case Payload of not_found -> {error, #{ <<"status">> => 404, @@ -129,6 +153,8 @@ deserialize(Base, Req, Opts) -> end. %% @doc Serialize a message to a JSON string. +-spec serialize(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary(), _ => _ }}. serialize(Base, Msg, Opts) -> {ok, #{ diff --git a/src/preloaded/codec/dev_json_iface.erl b/src/preloaded/codec/dev_json_iface.erl index b37621d6c..9b39d93fe 100644 --- a/src/preloaded/codec/dev_json_iface.erl +++ b/src/preloaded/codec/dev_json_iface.erl @@ -42,10 +42,17 @@ -include("include/hb.hrl"). %% @doc Initialize the device. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ function := binary(), _ => _ }}. init(M1, _M2, Opts) -> {ok, hb_ao:set(M1, #{<<"function">> => <<"handle">>}, Opts)}. %% @doc On first pass prepare the call, on second pass get the results. +-spec compute( + #{ pass => integer(), process => #{ _ => _ }, _ => _ }, + #{ body => #{ _ => _ }, 'block-height' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. compute(M1, M2, Opts) -> case hb_ao:get(<<"pass">>, M1, Opts) of 1 -> prep_call(M1, M2, Opts); @@ -208,7 +215,10 @@ prepare_tags(Msg, Opts) -> {ok, OriginalTags} -> Res = hb_util:message_to_ordered_list(OriginalTags), ?event({using_original_tags, Res}), - Res; + case complete_tags(Res) of + true -> Res; + false -> prepare_header_case_tags(Msg, Opts) + end; error -> prepare_header_case_tags(Msg, Opts) end; @@ -216,6 +226,14 @@ prepare_tags(Msg, Opts) -> prepare_header_case_tags(Msg, Opts) end. +complete_tags(Tags) -> + lists:all( + fun(#{ <<"name">> := _, <<"value">> := _ }) -> true; + (_) -> false + end, + Tags + ). + %% @doc Convert a message without an `original-tags' field into a list of %% key-value pairs, with the keys in HTTP header-case. prepare_header_case_tags(TABM, Opts) -> @@ -508,6 +526,8 @@ normalize_test_opts(Opts) -> test_init() -> application:ensure_all_started(hb). +-spec generate_stack(binary() | list(), binary(), #{ _ => _ }) -> + #{ _ => _ }. generate_stack(File) -> generate_stack(File, <<"WASM">>). generate_stack(File, Mode) -> diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index f93339031..c5acfd62e 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -26,6 +26,7 @@ -define(SUPPORTED_TYPES, [<<"integer">>, <<"float">>, <<"atom">>, <<"list">>]). %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -36,6 +37,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -46,6 +48,10 @@ verify(Msg, Req, Opts) -> }. %% @doc Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). +-spec from(_, + #{ 'encode-types' => [binary()], bundle => boolean(), _ => _ }, + #{ _ => _ } +) -> {ok, binary() | [_] | #{ _ => _ }}. from(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; from(List, Req, Opts) when is_list(List) -> % Encode the list as a map, then -- if our request indicates that we are @@ -60,7 +66,7 @@ from(List, Req, Opts) when is_list(List) -> Opts ), EncodingLists = lists:member(<<"list">>, find_encode_types(Req, Opts)), - EncodingHasAOTypes = hb_maps:is_key(<<"ao-types">>, DecodedAsMap, Opts), + EncodingHasAOTypes = maps:is_key(<<"ao-types">>, DecodedAsMap), case EncodingLists orelse EncodingHasAOTypes of true -> AOTypes = decode_ao_types(DecodedAsMap, Opts), @@ -164,8 +170,8 @@ from(Msg, Req, Opts) when is_map(Msg) -> from(Other, _Req, _Opts) -> {ok, hb_path:to_binary(Other)}. %% @doc Find the types that should be encoded from the request and options. -find_encode_types(Req, Opts) -> - hb_maps:get(<<"encode-types">>, Req, ?SUPPORTED_TYPES, Opts). +find_encode_types(Req, _Opts) -> + maps:get(<<"encode-types">>, Req, ?SUPPORTED_TYPES). %% @doc Determine the type for a value. type(Int) when is_integer(Int) -> <<"integer">>; @@ -176,7 +182,7 @@ type(Other) -> Other. %% @doc Discern the linkify mode from the request and the options. linkify_mode(Req, Opts) -> - case hb_maps:get(<<"bundle">>, Req, not_found, Opts) of + case maps:get(<<"bundle">>, Req, not_found) of not_found -> hb_opts:get(linkify_mode, offload, Opts); true -> % The request is asking for a bundle, so we should _not_ linkify. @@ -187,6 +193,8 @@ linkify_mode(Req, Opts) -> end. %% @doc Convert a TABM into a native HyperBEAM message. +-spec to(binary() | [_] | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | [_] | #{ _ => _ }}. to(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; to(TABM0, Req, Opts) when is_list(TABM0) -> % If we receive a list, we convert it to a message and run `to/3' on it. diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index b11a0c9a3..67efe2b2c 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -13,6 +13,7 @@ %% @doc Sign a message using the `priv-wallet' key in the options. Supports both %% the `hmac-sha256' and `rsa-pss-sha256' algorithms, offering unsigned and %% signed commitments. +-spec commit(#{ _ => _ }, #{ type := binary(), _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -39,7 +40,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> { ok, hb_message:convert( - hb_maps:without([<<"commitments">>], Msg, Opts), + maps:remove(<<"commitments">>, Msg), <<"tx@1.0">>, <<"structured@1.0">>, Opts @@ -47,6 +48,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> }. %% @doc Verify an L1 TX commitment. +-spec verify(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. verify(Msg, Req, Opts) -> ?event({verify, {base, Msg}, {req, Req}}), OnlyWithCommitment = @@ -64,6 +66,7 @@ verify(Msg, Req, Opts) -> {ok, Res}. %% @doc Convert a #tx record into a message map recursively. +-spec from(binary() | #tx{}, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; from(TX, Req, Opts) when is_record(TX, tx) -> case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of @@ -107,6 +110,8 @@ do_from(RawTX, Req, Opts) -> %% message's device in order to get the keys that we will be checkpointing. We %% do this recursively to handle nested messages. The base case is that we hit %% a binary, which we return as is. +-spec to(binary() | #tx{} | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #tx{}}. to(Binary, _Req, _Opts) when is_binary(Binary) -> % ar_tx cannot serialize just a simple binary or get an ID for it, so % we turn it into a TX record with a special tag, tx_to_message will diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index aa0951aea..8a4fd8a90 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -535,7 +535,9 @@ original_tags_to_tags(TagMap) -> ?event({ordered_tagmap, {explicit, OrderedList}, {input, {explicit, TagMap}}}), lists:map( fun(#{ <<"name">> := Key, <<"value">> := Value }) -> - {Key, Value} + {Key, Value}; + (#{ <<"name">> := Key }) -> + {Key, <<>>} end, OrderedList ). diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 837e974a9..02971111d 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -47,6 +47,8 @@ info() -> %% was a device name. %% 3. Execute the `default_index_path` (base: `index') upon the message, %% giving the rest of the request unchanged. +-spec index(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. index(Msg, Req, Opts) -> case hb_opts:get(default_index, not_found, Opts) of not_found -> @@ -82,6 +84,14 @@ index(Msg, Req, Opts) -> %% Note: This function _does not_ use AO-Core's `get/3' function, as it %% would require significant computation. We may want to change this %% if/when non-map message structures are created. +-spec id(binary() | [#{ _ => _ }] | #{ commitments => #{ _ => _ }, _ => _ }, + #{ committers => _, + 'commitment-ids' => _, + 'id-device' => binary(), + _ => _ + }, + #{ _ => _ } +) -> {ok, binary()}. id(Base) -> id(Base, #{}). id(Base, Req) -> id(Base, Req, #{}). id(Base, _, NodeOpts) when is_binary(Base) -> @@ -204,6 +214,8 @@ id_device(_, _) -> {ok, ?DEFAULT_ID_DEVICE}. %% @doc Return the committers of a message that are present in the given request. +-spec committers(#{ commitments => #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [_]}. committers(Base) -> committers(Base, #{}). committers(Base, Req) -> committers(Base, Req, #{}). committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> @@ -228,6 +240,10 @@ committers(_, _, _) -> %% @doc Commit to a message, using the `commitment-device' key to specify the %% device that should be used to commit to the message. If the key is not set, %% the default device (`httpsig@1.0') is used. +-spec commit(#{ _ => _ }, + #{ 'commitment-device' => binary(), type => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ commitments := #{ _ => _ }, _ => _ }}. commit(Self, Req, Opts) -> {ok, Base} = hb_message:find_target(Self, Req, Opts), AttDev = @@ -266,6 +282,10 @@ commit(Self, Req, Opts) -> %% `committers' key in the request can be used to specify that only the %% commitments from specific committers should be verified. Similarly, specific %% commitments can be specified using the `commitments' key. +-spec verify(#{ _ => _ }, + #{ committers => _, 'commitment-ids' => _, commitments => _, _ => _ }, + #{ _ => _ } +) -> {ok, boolean()}. verify(Self, Req, Opts) -> % Get the target message of the verification request. {ok, RawBase} = hb_message:find_target(Self, Req, Opts), @@ -337,6 +357,10 @@ verify_commitment(Base, Commitment, Opts) -> hb_ao:raw(AttDev, <<"verify">>, Base, Commitment, Opts). %% @doc Return the list of committed keys from a message. +-spec committed(#{ _ => _ }, + #{ raw => boolean(), committers => _, 'commitment-ids' => _, _ => _ }, + #{ _ => _ } +) -> {ok, [binary()]}. committed(Self, Req, Opts) -> % Get the target message of the verification request and ensure its % commitments are loaded. @@ -563,6 +587,8 @@ commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) -> %% @doc Deep merge keys in a message. Takes a map of key-value pairs and sets %% them in the message, overwriting any existing values. +-spec set(#{ _ => _ }, #{ 'set-mode' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. set(Base, NewValuesMsg, Opts) -> OriginalPriv = hb_private:from_message(Base), % Filter keys that are in the default device (this one). @@ -751,6 +777,8 @@ do_deep_merge(BaseValues, NewValues, Opts) -> %% transmit the present key that is being executed. Subsequently, to call `path' %% we would need to set `path' to `set', removing the ability to specify its %% new value. +-spec set_path(#{ path => _, _ => _ }, #{ value => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. set_path(Base, #{ <<"value">> := Value }, Opts) -> set_path(Base, Value, Opts); set_path(Base, Value, Opts) when not is_map(Value) -> @@ -777,6 +805,8 @@ set_path(Base, Value, Opts) when not is_map(Value) -> end. %% @doc Remove a key or keys from a message. +-spec remove(#{ _ => _ }, #{ item => _, items => [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. remove(Base, #{ <<"item">> := Key }, Opts) -> remove(Base, #{ <<"items">> => [Key] }, Opts); remove(Base, #{ <<"items">> := Keys }, Opts) -> diff --git a/src/preloaded/message/dev_trie.erl b/src/preloaded/message/dev_trie.erl index ff953c408..fd9685dca 100644 --- a/src/preloaded/message/dev_trie.erl +++ b/src/preloaded/message/dev_trie.erl @@ -65,8 +65,12 @@ collect_keys(TrieNode, Prefix, Opts, Acc) -> %% @doc Get the value associated with a key from a trie represented in a base %% message. +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, binary()}. get(Key, Trie, Req, Opts) -> get(Trie, Req#{<<"key">> => Key}, Opts). +-spec get(#{ _ => _ }, #{ key := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, binary()}. get(TrieNode, Req, Opts) -> case hb_maps:find(<<"key">>, Req, Opts) of error -> {error, <<"'key' parameter is required for trie lookup.">>}; @@ -74,6 +78,8 @@ get(TrieNode, Req, Opts) -> end. %% @doc Set keys and their values in the trie. +-spec set(#{ _ => _ }, #{ path => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. set(Trie, Req, Opts) -> Insertable = hb_maps:without([<<"path">>], Req, Opts), KeyVals = hb_maps:to_list(Insertable, Opts), diff --git a/src/preloaded/name/dev_b32_name.erl b/src/preloaded/name/dev_b32_name.erl index e86eada51..2c4210268 100644 --- a/src/preloaded/name/dev_b32_name.erl +++ b/src/preloaded/name/dev_b32_name.erl @@ -12,6 +12,8 @@ info(_Opts) -> }. %% @doc Try to resolve 52char subdomain back to its original TX ID +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, not_found}. get(Key, _, _HookMsg, _Opts) -> ?event({resolve_52char, {key, Key}}), case decode(Key) of diff --git a/src/preloaded/name/dev_local_name.erl b/src/preloaded/name/dev_local_name.erl index efcf15246..5f923b4ee 100644 --- a/src/preloaded/name/dev_local_name.erl +++ b/src/preloaded/name/dev_local_name.erl @@ -17,8 +17,9 @@ info(_Opts) -> }. %% @doc Takes a `key' argument and returns the value of the name, if it exists. -lookup(_, Req, Opts) -> - Key = hb_ao:get(<<"key">>, Req, no_key_specified, Opts), +-spec lookup(#{ _ => _ }, #{ key := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. +lookup(_, #{ <<"key">> := Key }, Opts) -> ?event(local_name, {lookup, Key}), hb_ao:resolve( find_names(Opts), @@ -27,11 +28,15 @@ lookup(_, Req, Opts) -> ). %% @doc Handle all other requests by delegating to the lookup function. +-spec default_lookup(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. default_lookup(Key, _, Req, Opts) -> lookup(Key, Req#{ <<"key">> => Key }, Opts). %% @doc Takes a `key' and `value' argument and registers the name. The caller %% must be the node operator in order to register a name. +-spec register(#{ _ => _ }, #{ key := binary(), value := _, _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, #{ status := integer(), message := binary() }} | not_found. register(_, Req, Opts) -> case hb_ao:resolve( #{ <<"device">> => <<"meta@1.0">> }, @@ -52,9 +57,9 @@ register(_, Req, Opts) -> %% @doc Register a name without checking if the caller is an operator. Exported %% for use by other devices, but not publicly available. direct_register(Req, Opts) -> - case hb_cache:write(hb_ao:get(<<"value">>, Req, Opts), Opts) of + case hb_cache:write(maps:get(<<"value">>, Req), Opts) of {ok, MsgPath} -> - NormKey = hb_ao:normalize_key(hb_ao:get(<<"key">>, Req, Opts)), + NormKey = hb_ao:normalize_key(maps:get(<<"key">>, Req)), hb_cache:link( MsgPath, LinkPath = << ?DEV_CACHE/binary, "/", NormKey/binary >>, diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 04aa4689c..c63e81360 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -25,6 +25,8 @@ info(_) -> %% pointer and its contents is loaded from the cache. For example, %% `GET /~name@1.0/reference' yields the message at the path specified by the %% `reference' key. +-spec resolve(binary(), #{ _ => _ }, #{ load => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, _} | not_found. resolve(Key, _, Req, Opts) -> Resolvers = hb_opts:get(name_resolvers, [], Opts), ?event({resolvers, Resolvers}), @@ -78,11 +80,16 @@ execute_resolver(Key, Resolver, Opts) when is_map(Resolver) -> %% @doc Implements an `on/request' compatible hook that resolves names given in %% the `host` key to their corresponding ID and prepends it to the execution path. +-spec request( + #{ _ => _ }, + #{ request := #{ host := binary(), _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. request(HookMsg, HookReq, Opts) -> ?event({request_hook, {hook_msg, HookMsg}, {hook_req, HookReq}}), maybe - {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), - {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), + {ok, Req} ?= maps:find(<<"request">>, HookReq), + {ok, Host} ?= maps:find(<<"host">>, Req), {ok, Name} ?= name_from_host( Host, @@ -92,7 +99,7 @@ request(HookMsg, HookReq, Opts) -> ModReq = maybe_append_named_message( ResolvedMsg, - hb_util:ok(hb_maps:find(<<"body">>, HookReq, Opts)), + maps:get(<<"body">>, HookReq), Opts ), ?event( @@ -136,7 +143,7 @@ maybe_append_named_message(ResolvedMsg, OldReq = [OldBase|ReqMsgsRest], Opts) -> true when is_map(OldBase) or is_list(OldBase) -> OldReq; true -> [ResolvedMsg|ReqMsgsRest]; false -> - case is_map(OldBase) andalso hb_maps:get(<<"path">>, OldBase, not_found, Opts) of + case is_map(OldBase) andalso maps:get(<<"path">>, OldBase, not_found) of not_found -> ?event( {skipping_old_base, diff --git a/src/preloaded/node/dev_blacklist.erl b/src/preloaded/node/dev_blacklist.erl index 726cb7583..3685a98b3 100644 --- a/src/preloaded/node/dev_blacklist.erl +++ b/src/preloaded/node/dev_blacklist.erl @@ -44,6 +44,8 @@ <<"/~hyperbuddy@1.0/bundle.js">>]). %% @doc Hook handler: block requests that involve blacklisted IDs. +-spec request(#{ _ => _ }, #{ request := #{ path => binary(), _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ _ => _ }} | {blocked_txid, binary()}. request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), case hb_opts:get(blacklist_providers, false, Opts) of @@ -78,7 +80,7 @@ request(_Base, HookReq, Opts) -> %% @doc Check if the message contains any blacklisted IDs. is_match(Msg, Opts) -> WhitelistRoutes = hb_opts:get(blacklist_whitelist, ?DEFAULT_WHITELIST, Opts), - Path = hb_maps:get(<<"path">>, maps:get(<<"request">>, Msg, #{}), no_path), + Path = maps:get(<<"path">>, maps:get(<<"request">>, Msg, #{}), no_path), case lists:member(Path, WhitelistRoutes) of false -> ?event({path_do_not_match_whitelist, {path, Path}}), diff --git a/src/preloaded/node/dev_cache.erl b/src/preloaded/node/dev_cache.erl index 2d90dac56..69d005a2f 100644 --- a/src/preloaded/node/dev_cache.erl +++ b/src/preloaded/node/dev_cache.erl @@ -21,14 +21,15 @@ %% @returns {ok, Data} on success, %% {error, not_found} if the key does not exist, %% {error, Reason} or {failure, Reason} on failure. -read(_M1, M2, Opts) -> - Location = hb_ao:get(<<"read">>, M2, Opts), +-spec read(#{ _ => _ }, #{ read := binary(), accept => binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _} | {failure, _}. +read(_M1, M2 = #{ <<"read">> := Location }, Opts) -> ?event({read, {key_extracted, Location}}), ?event(debug_gateway, cache_read), case hb_cache:read(Location, Opts) of {ok, Res} -> ?event({read, {cache_result, ok, Res}}), - case hb_ao:get(<<"accept">>, M2, Opts) of + case maps:get(<<"accept">>, M2, not_found) of <<"application/aos-2">> -> ?event(dev_cache, {read, @@ -80,13 +81,15 @@ read(_M1, M2, Opts) -> %% @param Opts A map of configuration options. %% @returns {ok, Path} on success, where Path indicates where the data was %% stored, {error, Reason} or {failure, Reason} on failure. +-spec write(#{ _ => _ }, #{ body => binary() | #{ _ => _ }, type => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _} | {failure, _} | #{ _ => _ }. write(_M1, M2, Opts) -> case is_trusted_writer(M2, Opts) of true -> ?event(dev_cache, {write, {trusted_writer, true}}), - Body = hb_ao:get(<<"body">>, M2, not_found, Opts), + Body = maps:get(<<"body">>, M2, not_found), Type = - case hb_maps:get(<<"type">>, M2, <<"single">>, Opts) of + case maps:get(<<"type">>, M2, <<"single">>) of <<"batch">> -> <<"batch">>; _ -> <<"single">> end, @@ -99,13 +102,12 @@ write(_M1, M2, Opts) -> ?event(dev_cache, {write, {write_batch_called}}), case Body of Batch when is_map(Batch) -> - hb_maps:map( + maps:map( fun(_, Value) -> ?event(dev_cache, {write, {batch_item, Value}}), write_single(Value, Opts) end, - Batch, - Opts + Batch ); _ -> {error, @@ -135,22 +137,24 @@ write(_M1, M2, Opts) -> end. %% @doc Link a source to a destination in the cache. -link(_Base, Req, Opts) -> +-spec link(#{ _ => _ }, #{ destination := binary(), source := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. +link(_Base, Req = #{ <<"destination">> := Destination, <<"source">> := Source }, Opts) -> case is_trusted_writer(Req, Opts) of true -> - Destination = hb_ao:get(<<"destination">>, Req, Opts), - Source = hb_ao:get(<<"source">>, Req, Opts), wrap_store_result(hb_store:link(#{ Destination => Source }, Opts)); false -> {error, not_authorized} end. -group(_Base, Req, Opts) -> +-spec group(#{ _ => _ }, #{ group := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. +group(_Base, Req = #{ <<"group">> := Group }, Opts) -> case is_trusted_writer(Req, Opts) of true -> wrap_store_result( hb_store:group( - #{ <<"group">> => hb_ao:get(<<"group">>, Req, Opts) }, + #{ <<"group">> => Group }, Opts ) ); diff --git a/src/preloaded/node/dev_cacheviz.erl b/src/preloaded/node/dev_cacheviz.erl index 64a79e658..fac027de4 100644 --- a/src/preloaded/node/dev_cacheviz.erl +++ b/src/preloaded/node/dev_cacheviz.erl @@ -6,16 +6,15 @@ %% @doc Output the dot representation of the cache, or a specific path within %% the cache set by the `target' key in the request. +-spec dot(#{ _ => _ }, #{ target => binary(), 'render-data' => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary() }}. dot(_, Req, Opts) -> - Target = hb_ao:get(<<"target">>, Req, all, Opts), + Target = maps:get(<<"target">>, Req, all), Dot = hb_cache_render:cache_path_to_dot( Target, #{ - render_data => - hb_util:atom( - hb_ao:get(<<"render-data">>, Req, false, Opts) - ) + render_data => maps:get(<<"render-data">>, Req, false) }, Opts ), @@ -23,6 +22,8 @@ dot(_, Req, Opts) -> %% @doc Output the SVG representation of the cache, or a specific path within %% the cache set by the `target' key in the request. +-spec svg(#{ _ => _ }, #{ target => binary(), 'render-data' => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary() }}. svg(Base, Req, Opts) -> {ok, #{ <<"body">> := Dot }} = dot(Base, Req, Opts), ?event(cacheviz, {dot, Dot}), @@ -33,10 +34,12 @@ svg(Base, Req, Opts) -> %% the `graph.js' library. If the request specifies a `target' key, we use that %% target. Otherwise, we generate a new target by writing the message to the %% cache and using the ID of the written message. +-spec json(#{ _ => _ }, #{ target => binary(), 'max-size' => integer(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. json(Base, Req, Opts) -> ?event({json, {base, Base}, {req, Req}}), Target = - case hb_ao:get(<<"target">>, Req, Opts) of + case maps:get(<<"target">>, Req, not_found) of not_found -> case map_size(maps:without([<<"device">>], hb_private:reset(Base))) of 0 -> @@ -52,7 +55,7 @@ json(Base, Req, Opts) -> <<".">> -> all; ReqTarget -> ReqTarget end, - MaxSize = hb_util:int(hb_ao:get(<<"max-size">>, Req, 250, Opts)), + MaxSize = maps:get(<<"max-size">>, Req, 250), ?event({max_size, MaxSize}), ?event({generating_json_for, {target, Target}}), Res = hb_cache_render:get_graph_data(Target, MaxSize, Opts), @@ -60,10 +63,12 @@ json(Base, Req, Opts) -> Res. %% @doc Return a renderer in HTML form for the JSON format. +-spec index(#{ _ => _ }, #{ _ => _ }, map()) -> term(). index(Base, _, Opts) -> ?event({cacheviz_index, {base, Base}}), hb_http_server:static(<<"cacheviz@1.0">>, <<"graph.html">>, Opts). %% @doc Return a JS library that can be used to render the JSON format. +-spec js(#{ _ => _ }, #{ _ => _ }, map()) -> term(). js(_, _, Opts) -> hb_http_server:static(<<"cacheviz@1.0">>, <<"graph.js">>, Opts). diff --git a/src/preloaded/node/dev_cron.erl b/src/preloaded/node/dev_cron.erl index 929d86ba4..9cf2a3360 100644 --- a/src/preloaded/node/dev_cron.erl +++ b/src/preloaded/node/dev_cron.erl @@ -6,6 +6,8 @@ -include_lib("eunit/include/eunit.hrl"). %% @doc Exported function for getting device info. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := #{ _ => _ }, _ => _ }}. info(_) -> #{ default => fun handler/4 }. @@ -23,6 +25,7 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Default handler: Assume that the key is an interval descriptor. +-spec handler(term(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). handler(<<"set">>, Base, Req, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"set">>, Base, Req, Opts); handler(<<"keys">>, Base, _Req, _Opts) -> @@ -31,6 +34,8 @@ handler(Interval, Base, Req, Opts) -> every(Base, Req#{ <<"interval">> => Interval }, Opts). %% @doc Exported function for scheduling a one-time message. +-spec once(#{ _ => _ }, #{ 'cron-path' => binary(), once => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := binary(), _ => _ }} | {error, _}. once(_Base, Req, Opts) -> case extract_path(<<"once">>, Req, Opts) of not_found -> @@ -76,10 +81,12 @@ once_worker(Path, Req, Opts) -> %% @doc Exported function for scheduling a recurring message. +-spec every(#{ _ => _ }, #{ interval := binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := binary(), _ => _ }} | {error, _}. every(_Base, Req, Opts) -> case { extract_path(Req, Opts), - hb_ao:get(<<"interval">>, Req, Opts) + maps:get(<<"interval">>, Req, not_found) } of {not_found, _} -> {error, <<"No cron path found in message.">>}; @@ -95,14 +102,13 @@ every(_Base, Req, Opts) -> end, ReqMsgID = hb_message:id(Req, all, Opts), ModifiedReq = - hb_maps:without( + maps:without( [ <<"interval">>, <<"cron-path">>, - hb_maps:get(<<"every">>, Req, <<"every">>, Opts) + maps:get(<<"every">>, Req, <<"every">>) ], - Req, - Opts + Req ), Pid = spawn( @@ -136,8 +142,10 @@ every(_Base, Req, Opts) -> end. %% @doc Exported function for stopping a scheduled task. -stop(_Base, Req, Opts) -> - case hb_ao:get(<<"task">>, Req, Opts) of +-spec stop(#{ _ => _ }, #{ task := binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := _, _ => _ }} | {error, _}. +stop(_Base, Req, _Opts) -> + case maps:get(<<"task">>, Req, not_found) of not_found -> {error, <<"No task ID found in message.">>}; TaskID -> @@ -204,7 +212,7 @@ parse_time(BinString) -> %% @doc Extract the path from the request message, given the name of the key %% that was invoked. extract_path(Req, Opts) -> - extract_path(hb_maps:get(<<"path">>, Req, Opts), Req, Opts). + extract_path(maps:get(<<"path">>, Req), Req, Opts). extract_path(Key, Req, Opts) -> hb_ao:get_first([{Req, Key}, {Req, <<"cron-path">>}], Opts). diff --git a/src/preloaded/node/dev_hyperbuddy.erl b/src/preloaded/node/dev_hyperbuddy.erl index f162f52a5..2e9e1c5c1 100644 --- a/src/preloaded/node/dev_hyperbuddy.erl +++ b/src/preloaded/node/dev_hyperbuddy.erl @@ -34,6 +34,7 @@ info(Opts) -> }. %% @doc The main HTML page for the REPL device. +-spec metrics(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ body := binary(), _ => _ }}. metrics(_, Req, Opts) -> case hb_opts:get(prometheus, not hb_features:test(), Opts) of true -> @@ -63,6 +64,7 @@ metrics(_, Req, Opts) -> end. %% @doc Return the current event counters as a message. +-spec events(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. events(_, _Req, _Opts) -> {ok, hb_event:counters()}. @@ -86,6 +88,11 @@ events(_, _Req, _Opts) -> %% ``` %% GET /.../~hyperbuddy@1.0/format=request?truncate-keys=20 %% ``` +-spec format( + #{ _ => _ }, + #{ format => binary() | [binary()], 'truncate-keys' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := binary(), _ => _ }}. format(Base, Req, Opts) -> % Find the scope of the environment that should be printed. Scope = @@ -140,6 +147,7 @@ format(Base, Req, Opts) -> }. %% @doc Test key for validating the behavior of the `500` HTTP response. +-spec throw(#{ _ => _ }, #{ _ => _ }, #{ mode => atom(), _ => _ }) -> {error, binary()}. throw(_Msg, _Req, Opts) -> case hb_opts:get(mode, prod, Opts) of prod -> {error, <<"Forced-throw unavailable in `prod` mode.">>}; @@ -148,6 +156,7 @@ throw(_Msg, _Req, Opts) -> %% @doc Serve a file from the priv directory. Only serves files that are explicitly %% listed in the `routes' field of the `info/1' return value. +-spec serve(term(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). serve(<<"keys">>, M1, _M2, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, Opts); serve(<<"set">>, M1, M2, Opts) -> diff --git a/src/preloaded/node/dev_location.erl b/src/preloaded/node/dev_location.erl index dfbc0a2f8..719072dda 100644 --- a/src/preloaded/node/dev_location.erl +++ b/src/preloaded/node/dev_location.erl @@ -33,13 +33,16 @@ info() -> %% @doc Route either `POST' or `GET' requests to the correct handler for known %% location records. +-spec known(#{ _ => _ }, #{ method => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. known(Base, Req, Opts) -> - case hb_ao:get(<<"method">>, Req, <<"GET">>, Opts) of + case maps:get(<<"method">>, Req, <<"GET">>) of <<"POST">> -> write_foreign(Base, Req, Opts); <<"GET">> -> all(Base, Req, Opts) end. %% @doc List all known location records. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]} | {error, _}. all(_Base, _Req, Opts) -> dev_location_cache:list(Opts). @@ -47,6 +50,8 @@ all(_Base, _Req, Opts) -> %% cache. If an address is provided, we search for the location of that %% specific scheduler. Otherwise, we return the location record for the current %% node's scheduler, if it has been established. +-spec read(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. read(Address, _Base, _Req, Opts) -> read(Address, Opts). read(Address, Opts) -> @@ -100,6 +105,7 @@ find_target(Base, RawReq, Opts) -> %% @doc Generate a new scheduler location record and register it. We both send %% the new scheduler-location to the given registry, and return it to the caller. +-spec node(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. node(Base, RawReq, RawOpts) -> Opts = case hb_ao:resolve( diff --git a/src/preloaded/node/dev_meta.erl b/src/preloaded/node/dev_meta.erl index 5b319586b..8c40b1422 100644 --- a/src/preloaded/node/dev_meta.erl +++ b/src/preloaded/node/dev_meta.erl @@ -58,6 +58,7 @@ is_operator(_Base, Req, NodeMsg) -> %% Subsequently, rather than embedding the `git-short-hash-length', for the %% avoidance of doubt, we include the short hash separately, as well as its long %% hash. +-spec build(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. build(_, _, _NodeMsg) -> {ok, #{ @@ -114,6 +115,7 @@ handle_initialize([], _NodeMsg) -> %% @doc Get/set the node message. If the request is a `POST', we check that the %% request is signed by the owner of the node. If not, we return the node message %% as-is, aside all keys that are private (according to `hb_private'). +-spec info(#{ _ => _ }, #{ _ => _ }, map()) -> term(). info(_, Request, NodeMsg) -> case hb_ao:get(<<"method">>, Request, NodeMsg) of <<"POST">> -> @@ -409,6 +411,8 @@ maybe_sign(Res, NodeMsg) -> %% @doc Check if the request in question is signed by a given `role' on the node. %% The `role' can be one of `operator' or `initiator'. +-spec is(atom(), #{ _ => _ }, #{ _ => _ }) -> + boolean() | {ok, boolean()} | {error, #{ status := integer(), _ => _ }}. is(Request, NodeMsg) -> is(operator, Request, NodeMsg). is(admin, Request, NodeMsg) -> diff --git a/src/preloaded/node/dev_node_process.erl b/src/preloaded/node/dev_node_process.erl index 58a4b7934..4ea0f2a09 100644 --- a/src/preloaded/node/dev_node_process.erl +++ b/src/preloaded/node/dev_node_process.erl @@ -18,6 +18,8 @@ info(_Opts) -> }. %% @doc Lookup a process by name. +-spec lookup(binary(), #{ _ => _ }, #{ spawn => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. lookup(Name, _Base, Req, Opts) -> ?event(node_process, {lookup, {name, Name}}), LookupRes = @@ -158,7 +160,8 @@ generate_test_opts() -> generate_test_opts(Name, Def) -> #{ <<"node-processes">> => #{ Name => Def }, - <<"priv-wallet">> => ar_wallet:new() + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => hb_test_utils:test_store() }. lookup_no_spawn_test() -> diff --git a/src/preloaded/node/dev_profile.erl b/src/preloaded/node/dev_profile.erl index 2353b5781..b1331a761 100644 --- a/src/preloaded/node/dev_profile.erl +++ b/src/preloaded/node/dev_profile.erl @@ -34,6 +34,7 @@ info(_) -> %% is the result of the function or resolution. In `return-mode: message' mode, %% the return format will be `{ok, EngineMessage}' where `EngineMessage' is the %% output from the engine formatted as an AO-Core message. +-spec eval(function() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _} | {_, _}. eval(Fun) -> eval(Fun, #{}). eval(Fun, Opts) -> eval(Fun, #{}, Opts). eval(Fun, Req, Opts) when is_function(Fun) -> @@ -47,6 +48,8 @@ eval(Fun, Req, Opts) when is_function(Fun) -> ); eval(Base, Request, Opts) -> eval(<<"eval">>, Base, Request, Opts). + +-spec eval(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _} | {_, _}. eval(PathKey, Base, Req, Opts) when not is_function(Base) -> case hb_ao:get(PathKey, Req, undefined, Opts) of undefined -> diff --git a/src/preloaded/node/dev_rate_limit.erl b/src/preloaded/node/dev_rate_limit.erl index f30c05198..20df96cd2 100644 --- a/src/preloaded/node/dev_rate_limit.erl +++ b/src/preloaded/node/dev_rate_limit.erl @@ -40,9 +40,11 @@ %% 429 status code and response if the limit is exceeded. The response includes %% a `retry-after' header that indicates the number of seconds the client should %% wait before making the next request. +-spec request(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), reason := binary(), body := binary(), _ => _ }}. request(_, Msg, Opts) -> ?event(rate_limit, {request, {msg, Msg}}), - Reference = request_reference(hb_maps:get(<<"request">>, Msg, #{}, Opts), Opts), + Reference = request_reference(maps:get(<<"request">>, Msg), Opts), case is_limited(Reference, Opts) of {true, Balance} -> ?event( diff --git a/src/preloaded/node/dev_router.erl b/src/preloaded/node/dev_router.erl index 26fcae0d2..05d4fba63 100644 --- a/src/preloaded/node/dev_router.erl +++ b/src/preloaded/node/dev_router.erl @@ -32,6 +32,7 @@ %% @doc Exported function for getting device info, controls which functions are %% exposed via the device API. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. info(_) -> #{ exports => @@ -91,6 +92,7 @@ info(_Base, _Req, _Opts) -> %% @doc Register function that allows telling the current node to register %% a new route with a remote router node. This function should also be idempotent. %% so that it can be called only once. +-spec register(#{ _ => _ }, #{ as => binary(), _ => _ }, #{ _ => _ }) -> {ok, binary()}. register(_M1, M2, Opts) -> %% Extract all required parameters from options %% These values will be used to construct the registration message @@ -139,11 +141,13 @@ register(_M1, M2, Opts) -> {ok, <<"Routes registered.">>}. %% @doc Device function that returns all known routes. +-spec routes(#{ _ => _ }, #{ method => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | [_] | #{ _ => _ }} | {error, _}. routes(M1, M2, Opts) -> ?event({routes_msg, M1, M2}), Routes = load_routes(Opts), ?event({routes, Routes}), - case hb_ao:get(<<"method">>, M2, Opts) of + case maps:get(<<"method">>, M2, <<"GET">>) of <<"POST">> -> RouterOpts = hb_opts:get(router_opts, #{}, Opts), ?event(debug_route_reg, {router_opts, RouterOpts}), @@ -236,6 +240,8 @@ routes(M1, M2, Opts) -> %% Can operate as a `~router@1.0' device, which will ignore the base message, %% routing based on the Opts and request message provided, or as a standalone %% function, taking only the request message and the `Opts' map. +-spec route(#{ _ => _ }, #{ path => binary(), 'route-path' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, no_matches}. route(Msg, Opts) -> route(undefined, Msg, Opts). route(_, Msg, Opts) -> Routes = load_routes(Opts), @@ -406,6 +412,11 @@ do_apply_route( %% @doc Find the first matching template in a list of known routes. Allows the %% path to be specified by either the explicit `path' (for internal use by this %% module), or `route-path' for use by external devices and users. +-spec match(#{ routes => [_] | #{ _ => _ }, _ => _ }, + #{ path => binary(), 'route-path' => binary(), _ => _ }, + #{ _ => _ } +) -> + {ok, #{ _ => _ }} | {error, no_matching_route}. match(Base, Req, Opts) -> ?event(debug_preprocess, {matching_routes, @@ -725,8 +736,13 @@ binary_to_bignum(Bin) when ?IS_ID(Bin) -> Num. %% @doc Preprocess a request to check if it should be relayed to a different node. +-spec preprocess( + #{ 'commit-request' => boolean(), _ => _ }, + #{ request := #{ path := binary(), _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := [_], _ => _ }}. preprocess(Base, RawReq, Opts) -> - Req = hb_ao:get(<<"request">>, RawReq, Opts#{ <<"hashpath">> => ignore }), + Req = maps:get(<<"request">>, RawReq), ?event(debug_preprocess, {called_preprocess,Req}), TemplateRoutes = load_routes(Opts), ?event(debug_preprocess, {template_routes, TemplateRoutes}), @@ -740,11 +756,7 @@ preprocess(Base, RawReq, Opts) -> ?event(debug_preprocess, executing_locally), {ok, #{ <<"body">> => - hb_ao:get( - <<"body">>, - RawReq, - Opts#{ <<"hashpath">> => ignore } - ) + maps:get(<<"body">>, RawReq) }}; <<"error">> -> ?event(debug_preprocess, preprocessor_returning_error), @@ -760,15 +772,7 @@ preprocess(Base, RawReq, Opts) -> {ok, _Method, Node, _Path, _MsgWithoutMeta, _ReqOpts} -> ?event(debug_preprocess, {matched_route, {explicit, Res}}), CommitRequest = - hb_util:atom( - hb_ao:get_first( - [ - {Base, <<"commit-request">>} - ], - false, - Opts - ) - ), + maps:get(<<"commit-request">>, Base, false), MaybeCommit = case CommitRequest of true -> #{ <<"commit-request">> => true }; @@ -798,11 +802,9 @@ preprocess(Base, RawReq, Opts) -> Req end, UserPath = - case hb_maps:get(<<"path">>, Req, not_found, Opts) of + case maps:get(<<"path">>, Req) of P when is_binary(P), byte_size(P) > 0 -> P; - not_found -> - throw({error, missing_user_path}); _ -> throw({error, invalid_user_path}) end, diff --git a/src/preloaded/node/dev_whois.erl b/src/preloaded/node/dev_whois.erl index 19a174d22..ab4cf4082 100644 --- a/src/preloaded/node/dev_whois.erl +++ b/src/preloaded/node/dev_whois.erl @@ -9,11 +9,15 @@ -include_lib("eunit/include/eunit.hrl"). %% @doc Return the calculated host information for the requester. -echo(_, Req, Opts) -> - {ok, hb_maps:get(<<"ao-peer">>, Req, <<"unknown">>, Opts)}. +-spec echo(#{ _ => _ }, #{ 'ao-peer' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary()}. +echo(_, Req, _Opts) -> + {ok, maps:get(<<"ao-peer">>, Req, <<"unknown">>)}. %% @doc Return the host information for the node. Sets the `host' key in the %% node message if it is not already set. +-spec node(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, _}. node(_, _, Opts) -> case ensure_host(Opts) of {ok, NewOpts} -> diff --git a/src/preloaded/payment/dev_faff.erl b/src/preloaded/payment/dev_faff.erl index e047262e4..237bb03b1 100644 --- a/src/preloaded/payment/dev_faff.erl +++ b/src/preloaded/payment/dev_faff.erl @@ -21,6 +21,8 @@ -include("include/hb.hrl"). %% @doc Decide whether or not to service a request from a given address. +-spec estimate(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, integer() | binary()}. estimate(_, Msg, NodeMsg) -> ?event(payment, {estimate, {msg, Msg}}), % Check if the address is in the allow-list. @@ -32,7 +34,7 @@ estimate(_, Msg, NodeMsg) -> %% @doc Check whether all of the signers of the request are in the allow-list. is_admissible(Msg, NodeMsg) -> AllowList = hb_opts:get(faff_allow_list, [], NodeMsg), - Req = hb_ao:get(<<"request">>, Msg, NodeMsg), + Req = maps:get(<<"request">>, Msg), Signers = hb_message:signers(Req, NodeMsg), ?event(payment, {is_admissible, {signers, Signers}, {allow_list, AllowList}}), lists:all( @@ -41,6 +43,7 @@ is_admissible(Msg, NodeMsg) -> ). %% @doc Charge the user's account if the request is allowed. +-spec charge(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. charge(_, Req, _NodeMsg) -> ?event(payment, {charge, Req}), {ok, true}. diff --git a/src/preloaded/payment/dev_p4.erl b/src/preloaded/payment/dev_p4.erl index 926e1bafd..96d204645 100644 --- a/src/preloaded/payment/dev_p4.erl +++ b/src/preloaded/payment/dev_p4.erl @@ -47,6 +47,11 @@ %% @doc Estimate the cost of a transaction and decide whether to proceed with %% a request. The default behavior if `pricing-device' or `p4_balances' are %% not set is to proceed, so it is important that a user initialize them. +-spec request( + #{ 'pricing-device' => binary(), 'ledger-device' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := [_], _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := [_], _ => _ }} | {error, _}. request(State, Raw, NodeMsg) -> PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), @@ -169,6 +174,11 @@ request(State, Raw, NodeMsg) -> end. %% @doc Postprocess the request after it has been fulfilled. +-spec response( + #{ 'pricing-device' => binary(), 'ledger-device' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := _, _ => _ }} | {error, _}. response(State, RawResponse, NodeMsg) -> PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), @@ -265,6 +275,7 @@ response(State, RawResponse, NodeMsg) -> end. %% @doc Get the balance of a user in the ledger. +-spec balance(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. balance(_, Req, NodeMsg) -> case hb_hook:find(<<"request">>, NodeMsg) of [] -> diff --git a/src/preloaded/payment/dev_simple_pay.erl b/src/preloaded/payment/dev_simple_pay.erl index d42cd6f14..44bd14f71 100644 --- a/src/preloaded/payment/dev_simple_pay.erl +++ b/src/preloaded/payment/dev_simple_pay.erl @@ -26,6 +26,8 @@ %% @doc Estimate the cost of the request, using the rules outlined in the %% moduledoc. +-spec estimate(#{ _ => _ }, #{ request => #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, non_neg_integer()}. estimate(_Base, EstimateReq, NodeMsg) -> Req = hb_ao:get( @@ -136,6 +138,8 @@ price_from_count(Messages, NodeMsg) -> %% @doc Preprocess a request by checking the ledger and charging the user. We %% can charge the user at this stage because we know statically what the price %% will be +-spec charge(#{ _ => _ }, #{ request => #{ _ => _ }, quantity => integer(), _ => _ }, #{ _ => _ }) -> + {ok, boolean()} | {error, #{ status := integer(), body := binary(), _ => _ }}. charge(_, RawReq, NodeMsg) -> ?event(payment, {charge, RawReq}), Req = @@ -195,6 +199,8 @@ charge(_, RawReq, NodeMsg) -> end. %% @doc Get the balance of a user in the ledger. +-spec balance(#{ _ => _ }, #{ request => #{ _ => _ }, target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, integer()}. balance(_, RawReq, NodeMsg) -> Target = case @@ -260,6 +266,8 @@ get_balance(Signer, NodeMsg) -> hb_ao:get(NormSigner, Ledger, 0, NodeMsg). %% @doc Top up the user's balance in the ledger. +-spec topup(#{ _ => _ }, #{ amount => integer(), recipient => binary(), _ => _ }, #{ _ => _ }) -> + {ok, integer()} | {error, binary()}. topup(_, Req, NodeMsg) -> ?event({topup, {req, Req}, {node_msg, NodeMsg}}), case is_operator(Req, NodeMsg) of diff --git a/src/preloaded/process/dev_process.erl b/src/preloaded/process/dev_process.erl index 55eea8feb..ba9104a01 100644 --- a/src/preloaded/process/dev_process.erl +++ b/src/preloaded/process/dev_process.erl @@ -8,10 +8,9 @@ %%% device is (by default) a single device. %%% %%% This allows the devices to share state as needed. Additionally, after each -%%% computation step the device caches the result at a path relative to the -%%% process definition itself, such that the process message's ID can act as an -%%% immutable reference to the process's growing list of interactions. See -%%% `dev_process_cache' for details. +%%% computation step the device caches the result as AO-Core result edges, such +%%% that the process message's ID can act as an immutable reference to the +%%% process's growing list of interactions. %%% %%% The external API of the device is as follows: %%%
@@ -35,14 +34,14 @@
 %%% 
%%% %%% Runtime options: -%%% Cache-Frequency: The number of assignments that will be computed -%%% before the full (restorable) state should be cached. -%%% Cache-Keys: A list of the keys that should be cached for all -%%% assignments, in addition to `/Results'. +%%% Process-Snapshot-Slots: The number of slots between full restorable +%%% state snapshots. +%%% Process-Snapshot-Time: The number of seconds between full restorable +%%% state snapshots. -module(dev_process). -device_libraries([lib_process]). %%% Public API --export([info/1, as/3, compute/3, schedule/3, slot/3, now/3, push/3, snapshot/3]). +-export([info/1, as/3, compute/3, schedule/3, slot/3, latest/3, now/3, push/3, snapshot/3]). -export([target_slot/2]). -export([default_device/3]). -include_lib("eunit/include/eunit.hrl"). @@ -69,6 +68,7 @@ info(_Base) -> <<"info">>, <<"as">>, <<"compute">>, + <<"latest">>, <<"now">>, <<"schedule">>, <<"slot">>, @@ -79,17 +79,12 @@ info(_Base) -> %% @doc Return the process state with the device swapped out for the device %% of the given key. +-spec as(#{ 'input-prefix' => binary(), _ => _ }, + #{ as => binary(), 'as-device' => binary(), _ => _ }, + #{ _ => _ }) -> {ok, #{ device := binary(), _ => _ }}. as(RawBase, Req, Opts) -> {ok, Base} = ensure_loaded(RawBase, Req, Opts), - Key = - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, Req}, <<"as">>}, - {{as, <<"message@1.0">>, Req}, <<"as-device">>} - ], - <<"execution">>, - Opts - ), + Key = maps:get(<<"as">>, Req, maps:get(<<"as-device">>, Req, <<"execution">>)), {ok, hb_util:deep_merge( lib_process:ensure_process_key(Base, Opts), @@ -104,7 +99,7 @@ as(RawBase, Req, Opts) -> % Configure input prefix for proper message routing within the % device <<"input-prefix">> => - case hb_maps:get(<<"input-prefix">>, Base, not_found, Opts) of + case maps:get(<<"input-prefix">>, Base, not_found) of not_found -> <<"process">>; Prefix -> Prefix end, @@ -126,13 +121,19 @@ as(RawBase, Req, Opts) -> %% _must_ be set in all processes aside those marked with `ao.TN.1' variant. %% This is in order to ensure that post-mainnet processes do not default to %% using infrastructure that should not be present on nodes in the future. +-spec default_device(#{ 'process/variant' => binary(), _ => _ }, binary(), #{ _ => _ }) -> + binary(). default_device(Base, Key, Opts) -> lib_process:default_device(Base, Key, Opts). %% @doc Wraps functions in the Scheduler device. +-spec schedule(#{ scheduler => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. schedule(Base, Req, Opts) -> lib_process:run_as(<<"scheduler">>, Base, Req, Opts). +-spec slot(#{ scheduler => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. slot(Base, Req, Opts) -> ?event({slot_called, {base, Base}, {req, Req}}), lib_process:run_as(<<"scheduler">>, Base, Req, Opts). @@ -140,6 +141,8 @@ slot(Base, Req, Opts) -> next(Base, _Req, Opts) -> lib_process:run_as(<<"scheduler">>, Base, next, Opts). +-spec snapshot(#{ execution => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. snapshot(RawBase, _Req, Opts) -> Base = lib_process:ensure_process_key(RawBase, Opts), {ok, SnapshotMsg} = @@ -190,6 +193,19 @@ init(Base, Req, Opts) -> %% handlers and previewing results. The POST method is the key entry point %% for the dryrun functionality that allows external clients to test %% message processing without side effects. +-spec compute( + #{ initialized => binary(), 'at-slot' => integer(), _ => _ }, + #{ + compute => integer(), + slot => integer(), + init => binary(), + push => _, + 'result-depth' => _, + async => _, + 'max-depth' => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | {failure, _}. compute(Base, Req, Opts) -> ProcBase = lib_process:ensure_process_key(Base, Opts), ProcID = lib_process:process_id(ProcBase, #{}, Opts), @@ -198,16 +214,15 @@ compute(Base, Req, Opts) -> not_found -> % The slot is not set, so we need to serve the latest known state % unless the `init' key is set to a value aside from `now'. - % We do this by setting the `process-now-from-cache' option to `true'. - case hb_maps:get(<<"init">>, Req, <<"now">>, Opts) of + % We do this by setting the `process_now_from_cache' option to `true'. + case maps:get(<<"init">>, Req, <<"now">>) of <<"now">> -> - now(Base, Req, Opts#{ <<"process-now-from-cache">> => true }); + now(Base, Req, Opts#{ process_now_from_cache => true }); _ -> {error, not_found} end; - RawSlot -> - Slot = hb_util:int(RawSlot), - case dev_process_cache:read(ProcID, Slot, Opts) of + Slot -> + case read_process_slot(ProcID, Slot, Opts) of {ok, Result} -> % The result is already cached, so we can return it. ?event( @@ -217,7 +232,7 @@ compute(Base, Req, Opts) -> {result, Result} } ), - {ok, without_snapshot(Result, Opts)}; + {ok, compute_response(Result, Req, Opts)}; {error, not_found} -> {ok, Loaded} = ensure_loaded(ProcBase, Req, Opts), ?event(compute, @@ -236,14 +251,11 @@ compute(Base, Req, Opts) -> end. %% @doc Return the slot requested by a `compute' request, or `not_found'. -target_slot(Req, Opts) -> - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, Req}, <<"compute">>}, - {{as, <<"message@1.0">>, Req}, <<"slot">>} - ], - Opts - ). +target_slot(Req, _Opts) -> + case maps:get(<<"compute">>, Req, not_found) of + not_found -> maps:get(<<"slot">>, Req, not_found); + ComputeSlot -> ComputeSlot + end. %% @doc Continually get and apply the next assignment from the scheduler until %% we reach the target slot that the user has requested. @@ -259,7 +271,7 @@ compute_to_slot(ProcID, Base, Req, TargetSlot, Opts) -> Opts ), store_result(true, ProcID, TargetSlot, Base, Req, Opts), - {ok, without_snapshot(lib_process:as_process(Base, Opts), Opts)}; + {ok, compute_response(lib_process:as_process(Base, Opts), Req, Opts)}; CurrentSlot when CurrentSlot < TargetSlot -> % Compute the next state transition. NextSlot = CurrentSlot + 1, @@ -387,11 +399,10 @@ compute_slot(ProcID, State, RawInputMsg, InitReq, TargetSlot, Opts) -> {store_ms, StoreTimeMicroSecs div 1000}, {computed_slot_size, erlang:external_size(NewProcStateMsgWithSlot)}, {action, - hb_ao:get( - <<"body/action">>, - Req, - no_action_set, - Opts#{ <<"hashpath">> => ignore } + maps:get( + <<"action">>, + maps:get(<<"body">>, Req, #{}), + no_action_set ) } } @@ -438,7 +449,7 @@ compute_slot(ProcID, State, RawInputMsg, InitReq, TargetSlot, Opts) -> %% @doc Prepare the process state message for computing the next slot. prepare_next_slot(ProcID, State, RawReq, Opts) -> - Slot = hb_util:int(hb_ao:get(<<"slot">>, RawReq, Opts)), + Slot = hb_util:int(maps:get(<<"slot">>, RawReq)), ?event(compute, {next_slot, Slot}), % If the input message does not have a path, set it to `compute'. Req = @@ -566,11 +577,73 @@ store_result(ForceSnapshot, ProcID, Slot, Res, Req, Opts) -> } ), WithLastSnapshot - end, + end, + PublicResult = without_snapshot(ResMaybeWithSnapshot, Opts), ?event(compute, {caching_result, {proc_id, ProcID}, {slot, Slot}}, Opts), - dev_process_cache:write(ProcID, Slot, ResMaybeWithSnapshot, Opts), + CacheEdges = [{ProcID, slot_req(Slot)}, {ProcID, latest_req()}], + PublicCacheResult = hb_private:reset(PublicResult), + {ok, _} = hb_cache:write_result(CacheEdges, PublicCacheResult, Opts), + {ok, _} = + write_restore_edges( + ProcID, + Slot, + ResMaybeWithSnapshot, + maps:is_key(<<"snapshot">>, ResMaybeWithSnapshot), + Opts + ), ?event(compute, {caching_completed, {proc_id, ProcID}, {slot, Slot}}, Opts), - hb_maps:without([<<"snapshot">>], ResMaybeWithSnapshot, Opts). + PublicResult. + +read_process_slot(ProcID, Slot, Opts) -> + read_process_edge(ProcID, slot_req(Slot), Opts). + +read_process_edge(ProcID, Req, Opts) -> + case hb_cache:read_resolved(ProcID, Req, Opts) of + {hit, {ok, Msg}} -> {ok, Msg}; + {hit, Other} -> Other; + miss -> {error, not_found} + end. + +slot_req(Slot) -> + #{ <<"path">> => <<"compute">>, <<"slot">> => hb_util:int(Slot) }. + +latest_req() -> + #{ <<"path">> => <<"latest">> }. + +restore_req() -> + #{ <<"path">> => <<"restore">> }. + +restore_req(Slot) -> + #{ <<"path">> => <<"restore">>, <<"slot">> => hb_util:int(Slot) }. + +write_restore_edges(ProcID, Slot, Checkpoint, true, Opts) -> + hb_cache:write_result( + [{ProcID, restore_req(Slot)}, {ProcID, restore_req()}], + hb_private:reset(Checkpoint), + Opts + ); +write_restore_edges(_ProcID, _Slot, _Res, false, _Opts) -> + {ok, skipped}. + +read_restore_checkpoint(ProcID, undefined, Opts) -> + read_restore_checkpoint(ProcID, restore_req(), Opts); +read_restore_checkpoint(ProcID, Req, Opts) when is_map(Req) -> + case read_process_edge(ProcID, Req, hb_store:scope(Opts, local)) of + {ok, Msg = #{ <<"at-slot">> := Slot }} -> {ok, hb_util:int(Slot), Msg}; + {ok, _} -> {error, not_found}; + Other -> Other + end; +read_restore_checkpoint(ProcID, TargetSlot, Opts) -> + TargetSlotInt = hb_util:int(TargetSlot), + case read_restore_checkpoint(ProcID, restore_req(TargetSlotInt), Opts) of + {error, not_found} -> + case read_restore_checkpoint(ProcID, restore_req(), Opts) of + {ok, Slot, Msg} when Slot =< TargetSlotInt -> {ok, Slot, Msg}; + _ -> {error, not_found} + end; + Other -> + Other + end. %% @doc Should we snapshot a new full state result? First, we check if the %% `process_snapshot_time' option is set. If it is, we check if the elapsed time @@ -622,7 +695,14 @@ should_snapshot_time(Res, Opts) -> end. %% @doc Returns the known state of the process at either the current slot, or -%% the latest slot in the cache depending on the `process-now-from-cache' option. +%% the latest slot in the cache depending on the `process_now_from_cache' option. +-spec latest(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {failure, _} | {error, _}. +latest(Base, Req, Opts) -> + now(Base, Req, Opts#{ process_now_from_cache => always }). + +-spec now(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {failure, _} | {error, _}. now(RawBase, Req, Opts) -> Base = lib_process:ensure_process_key(RawBase, Opts), ProcessID = lib_process:process_id(Base, #{}, Opts), @@ -643,10 +723,10 @@ now(RawBase, Req, Opts) -> CacheParam -> % We are serving the latest known state from the cache, rather % than computing it. - LatestKnown = dev_process_cache:latest(ProcessID, [], Opts), + LatestKnown = read_process_edge(ProcessID, latest_req(), Opts), case LatestKnown of - {ok, LatestSlot, RawLatestMsg} -> - LatestMsg = without_snapshot(RawLatestMsg, Opts), + {ok, RawLatestMsg = #{ <<"at-slot">> := LatestSlot }} -> + LatestMsg = compute_response(RawLatestMsg, Req, Opts), ?event(compute_cache, {serving_latest_cached_state, {proc_id, ProcessID}, @@ -666,7 +746,7 @@ now(RawBase, Req, Opts) -> % The node is configured to use the cache if possible, % but forcing computation is also admissible. Subsequently, % as no other option is available, we compute the state. - now(Base, Req, Opts#{ <<"process-now-from-cache">> => false }); + now(Base, Req, Opts#{ process_now_from_cache => false }); true -> % The node is configured to only serve the latest known % state from the cache, so we return the latest slot. @@ -677,6 +757,8 @@ now(RawBase, Req, Opts) -> %% @doc Recursively push messages to the scheduler until we find a message %% that does not lead to any further messages being scheduled. +-spec push(#{ push => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. push(Base, Req, Opts) -> lib_process:run_as( <<"push">>, @@ -692,20 +774,14 @@ ensure_loaded(Base, Req, Opts) -> TargetSlot = hb_ao:get(<<"slot">>, Req, undefined, Opts), ProcID = lib_process:process_id(Base, #{}, Opts), ?event({ensure_loaded, {base, Base}, {req, Req}}), - case hb_ao:get(<<"initialized">>, Base, Opts) of + case maps:get(<<"initialized">>, Base, undefined) of <<"true">> -> ?event(already_initialized), {ok, Base}; _ -> ?event(not_initialized), % Try to load the latest complete state from disk. - LoadRes = - dev_process_cache:latest( - ProcID, - [<<"snapshot+link">>], - TargetSlot, - Opts - ), + LoadRes = read_restore_checkpoint(ProcID, TargetSlot, Opts), ?event(compute, {snapshot_load_res, {proc_id, ProcID}, @@ -738,23 +814,19 @@ ensure_loaded(Base, Req, Opts) -> #{ <<"commitments">> := SignCommits } = hb_message:with_commitments(ProcID, Process, Opts), UpdateProcess = - hb_maps:put( - <<"commitments">>, - hb_maps:merge(HmacCommits, SignCommits), - Process, - Opts - ), + Process#{ + <<"commitments">> => + maps:merge(HmacCommits, SignCommits) + }, SnapshotReq = SnapshotMsg#{ <<"process">> => UpdateProcess, <<"initialized">> => <<"true">> }, - LoadedSlot = - hb_cache:ensure_all_loaded(MaybeLoadedSlot, Opts), ?event(compute, {found_state_checkpoint, {proc_id, ProcID}, - {slot, LoadedSlot} + {slot, MaybeLoadedSlot} }, Opts ), @@ -770,7 +842,7 @@ ensure_loaded(Base, Req, Opts) -> ?event(snapshot, {loaded_state_checkpoint_result, {proc_id, ProcID}, - {slot, LoadedSlot}, + {slot, MaybeLoadedSlot}, {after_normalization, NormalizedWithoutSnapshot} } ), @@ -791,3 +863,17 @@ ensure_loaded(Base, Req, Opts) -> %% @doc Remove the `snapshot' key from a message and return it. without_snapshot(Msg, Opts) -> hb_ao:set(Msg, <<"snapshot">>, unset, Opts). + +%% @doc Format a compute response for the caller with its result payload inline. +compute_response(Msg, _Req, Opts) -> + with_loaded_results(without_snapshot(Msg, Opts), Opts). + +with_loaded_results(Msg, Opts) -> + Decoded = hb_link:decode_all_links(Msg), + case hb_maps:get(<<"results">>, Decoded, not_found, Opts) of + not_found -> Msg; + Results -> + (maps:remove(<<"results+link">>, Msg))#{ + <<"results">> => hb_cache:ensure_all_loaded(Results, Opts) + } + end. diff --git a/src/preloaded/process/dev_process_cache.erl b/src/preloaded/process/dev_process_cache.erl deleted file mode 100644 index b2617ee6e..000000000 --- a/src/preloaded/process/dev_process_cache.erl +++ /dev/null @@ -1,232 +0,0 @@ - -%%% @doc A wrapper around the hb_cache module that provides a more -%%% convenient interface for reading the result of a process at a given slot or -%%% message ID. --module(dev_process_cache). --export([latest/2, latest/3, latest/4, read/2, read/3, write/4]). --include_lib("eunit/include/eunit.hrl"). --include("include/hb.hrl"). - -%% @doc Read the result of a process at a given slot. -read(ProcID, Opts) -> - hb_util:ok(latest(ProcID, Opts)). -read(ProcID, SlotRef, Opts) -> - ?event({reading_computed_result, ProcID, SlotRef}), - Path = path(ProcID, SlotRef, Opts), - hb_cache:read(Path, Opts). - -%% @doc Write a process computation result to the cache. -write(ProcID, Slot, Msg, Opts) -> - % Write the item to the cache in the root of the store. - {ok, Root} = hb_cache:write(hb_private:reset(Msg), Opts), - % Link the item to the path in the store by slot number. - SlotNumPath = path(ProcID, Slot, Opts), - hb_cache:link(Root, SlotNumPath, Opts), - % Link the item to the message ID path in the store. - MsgIDPath = - path( - ProcID, - ID = hb_message:id(Msg, uncommitted, Opts), - Opts - ), - ?event( - {linking_id, - {proc_id, ProcID}, - {slot, Slot}, - {id, ID}, - {path, MsgIDPath} - } - ), - hb_cache:link(Root, MsgIDPath, Opts), - % Return the slot number path. - {ok, SlotNumPath}. - -%% @doc Calculate the path of a result, given a process ID and a slot. -path(ProcID, Ref, Opts) -> - path(ProcID, Ref, [], Opts). -path(ProcID, Ref, PathSuffix, _Opts) -> - hb_path:to_binary( - [ - <<"computed">>, - hb_util:human_id(ProcID) - ] ++ - case Ref of - Int when is_integer(Int) -> ["slot", integer_to_binary(Int)]; - root -> []; - slot_root -> ["slot"]; - _ -> [Ref] - end ++ PathSuffix - ). - -%% @doc Retrieve the latest slot for a given process. Optionally state a limit -%% on the slot number to search for, as well as a required path that the slot -%% must have. -latest(ProcID, Opts) -> latest(ProcID, [], Opts). -latest(ProcID, RequiredPath, Opts) -> - latest(ProcID, RequiredPath, undefined, Opts). -latest(ProcID, RawRequiredPath, Limit, RawOpts) -> - Scope = hb_opts:get(process_cache_scope, local, RawOpts), - % Normalize the store descriptor to a list of stores. - UnscopedStore = - case hb_opts:get(store, no_viable_store, RawOpts) of - StoreMsg when is_map(StoreMsg) -> [StoreMsg]; - Other -> Other - end, - % Apply the scope to the store and update the options message. - ScopedStore = hb_store:scope(UnscopedStore, Scope), - Opts = RawOpts#{ <<"store">> => ScopedStore }, - % Convert the required path to a list of _binary_ keys. - RequiredPath = - case RawRequiredPath of - undefined -> []; - [] -> []; - _ -> - hb_path:term_to_path_parts( - RawRequiredPath, - Opts - ) - end, - ?event({required_path_converted, {proc_id, ProcID}, {required_path, RequiredPath}}), - Path = path(ProcID, slot_root, Opts), - AllSlots = hb_cache:list_numbered(Path, Opts), - ?event({all_slots, {proc_id, ProcID}, {slots, AllSlots}}), - CappedSlots = - case Limit of - undefined -> AllSlots; - _ -> lists:filter(fun(Slot) -> Slot =< Limit end, AllSlots) - end, - ?event( - {finding_latest_slot, - {proc_id, hb_util:human_id(ProcID)}, - {limit, Limit}, - {path, Path}, - {slots_in_range, CappedSlots} - } - ), - % Find the highest slot that has the necessary path. - BestSlot = - first_with_path( - ProcID, - RequiredPath, - lists:reverse(lists:sort(CappedSlots)), - Opts - ), - case BestSlot of - {failure, _} = Failure -> - Failure; - {error, _} = Error -> - Error; - not_found -> - % No slot found with the necessary path was found. - {error, not_found}; - SlotNum -> - % Found. Return the slot number and the message at that slot. - {ok, Msg} = hb_cache:read(path(ProcID, SlotNum, Opts), Opts), - {ok, SlotNum, Msg} - end. - -%% @doc Find the latest assignment with the requested path suffix. -first_with_path(ProcID, RequiredPath, Slots, Opts) -> - first_with_path( - ProcID, - RequiredPath, - Slots, - Opts, - hb_opts:get(store, no_viable_store, Opts) - ). -first_with_path(_ProcID, _Required, [], _Opts, _Store) -> - not_found; -first_with_path(ProcID, RequiredPath, [Slot | Rest], Opts, Store) -> - RawPath = path(ProcID, Slot, RequiredPath, Opts), - ?event({trying_slot, {slot, Slot}, {path, RawPath}}), - case hb_store:read(Store, RawPath, Opts) of - {error, not_found} -> - first_with_path(ProcID, RequiredPath, Rest, Opts, Store); - {failure, _} = Failure -> - Failure; - {error, _} = Error -> - Error; - _ -> - Slot - end. - -%%% Tests - -process_cache_suite_test_() -> - hb_store:generate_test_suite( - [ - {"write and read process outputs", fun test_write_and_read_output/1}, - {"find latest output (with path)", fun find_latest_outputs/1} - ], - [ - {Name, Opts} - || - {Name, Opts} <- hb_store:test_stores() - ] - ). - -%% @doc Test for writing multiple computed outputs, then getting them by -%% their slot number and by their signed and unsigned IDs. -test_write_and_read_output(Opts) -> - Proc = hb_cache:test_signed( - #{ <<"test-item">> => hb_cache:test_unsigned(<<"test-body-data">>) }), - ProcID = hb_util:human_id(hb_ao:get(id, Proc)), - Item1 = hb_cache:test_signed(<<"Simple signed output #1">>), - Item2 = hb_cache:test_unsigned(<<"Simple unsigned output #2">>), - {ok, Path0} = write(ProcID, 0, Item1, Opts), - {ok, Path1} = write(ProcID, 1, Item2, Opts), - {ok, DirectReadItem1} = hb_cache:read(Path0, Opts), - ?assert(hb_message:match(Item1, DirectReadItem1)), - {ok, DirectReadItem2} = hb_cache:read(Path1, Opts), - ?assert(hb_message:match(Item2, DirectReadItem2)), - {ok, ReadItem1BySlotNum} = read(ProcID, 0, Opts), - ?assert(hb_message:match(Item1, ReadItem1BySlotNum)), - {ok, ReadItem2BySlotNum} = read(ProcID, 1, Opts), - ?assert(hb_message:match(Item2, ReadItem2BySlotNum)), - {ok, ReadItem1ByID} = - read(ProcID, hb_util:human_id(hb_ao:get(id, Item1)), Opts), - ?assert(hb_message:match(Item1, ReadItem1ByID)), - {ok, ReadItem2ByID} = - read(ProcID, hb_util:human_id(hb_message:id(Item2, all)), Opts), - ?assert(hb_message:match(Item2, ReadItem2ByID)). - -%% @doc Test for retrieving the latest computed output for a process. -find_latest_outputs(Opts) -> - % Create test environment. - Store = hb_opts:get(store, no_viable_store, Opts), - ResetRes = hb_store:reset(Store), - ?event({reset_store, {result, ResetRes}, {store, Store}}), - Proc1 = hb_process_test_vectors:aos_process(), - ProcID = hb_util:human_id(hb_ao:get(id, Proc1, Opts)), - % Create messages for the slots, with only the middle slot having a - % `/Process' field, while the top slot has a `/Deep/Process' field. - Msg0 = #{ <<"Results">> => #{ <<"Result-Number">> => 0 } }, - Base = - #{ - <<"Results">> => #{ <<"Result-Number">> => 1 }, - <<"Process">> => Proc1 - }, - Req = - #{ - <<"Results">> => #{ <<"Result-Number">> => 2 }, - <<"Deep">> => #{ <<"Process">> => Proc1 } - }, - % Write the messages to the cache. - {ok, _} = write(ProcID, 0, Msg0, Opts), - {ok, _} = write(ProcID, 1, Base, Opts), - {ok, _} = write(ProcID, 2, Req, Opts), - ?event(wrote_items), - % Read the messages with various qualifiers. - {ok, 2, ReadReq} = latest(ProcID, Opts), - ?event({read_latest, ReadReq}), - ?assert(hb_message:match(Req, ReadReq)), - ?event(read_latest_slot_without_qualifiers), - {ok, 1, ReadBaseRequired} = latest(ProcID, <<"Process">>, Opts), - ?event({read_latest_with_process, ReadBaseRequired}), - ?assert(hb_message:match(Base, ReadBaseRequired)), - ?event(read_latest_slot_with_shallow_key), - {ok, 2, ReadReqRequired} = latest(ProcID, <<"Deep/Process">>, Opts), - ?assert(hb_message:match(Req, ReadReqRequired)), - ?event(read_latest_slot_with_deep_key), - {ok, 1, ReadBase} = latest(ProcID, [], 1, Opts), - ?assert(hb_message:match(Base, ReadBase)). diff --git a/src/preloaded/process/dev_process_worker.erl b/src/preloaded/process/dev_process_worker.erl index 28bbcfe05..a65872156 100644 --- a/src/preloaded/process/dev_process_worker.erl +++ b/src/preloaded/process/dev_process_worker.erl @@ -43,16 +43,38 @@ compute_group(Base, Req, Opts) -> %% @doc Return `true' if the requested compute result is already cached. compute_cached(ProcID, not_found, Opts) -> - case dev_process_cache:latest(ProcID, Opts) of - {ok, _Slot, _Msg} -> true; + case read_process_edge(ProcID, latest_req(), Opts) of + {ok, _Msg} -> true; _ -> false end; compute_cached(ProcID, RawSlot, Opts) -> - case dev_process_cache:read(ProcID, hb_util:int(RawSlot), Opts) of + case read_process_edge(ProcID, slot_req(hb_util:int(RawSlot)), Opts) of {ok, _Msg} -> true; _ -> false end. +read_process_edge(ProcID, Req, Opts) -> + case hb_cache:read_resolved(ProcID, Req, process_cache_opts(Opts)) of + {hit, {ok, Msg}} -> {ok, Msg}; + {hit, Other} -> Other; + miss -> {error, not_found} + end. + +slot_req(Slot) -> + #{ <<"path">> => <<"compute">>, <<"slot">> => hb_util:int(Slot) }. + +latest_req() -> + #{ <<"path">> => <<"latest">> }. + +process_cache_opts(RawOpts) -> + Scope = hb_opts:get(process_cache_scope, local, RawOpts), + UnscopedStore = + case hb_opts:get(store, no_viable_store, RawOpts) of + StoreMsg when is_map(StoreMsg) -> [StoreMsg]; + Other -> Other + end, + RawOpts#{ store => hb_store:scope(UnscopedStore, Scope) }. + process_to_group_name(Base, Opts) -> Initialized = lib_process:ensure_process_key(Base, Opts), ProcMsg = @@ -236,10 +258,13 @@ grouper_skips_when_slot_cached_test() -> ?assertNotEqual(ungrouped_exec, ProcessGroup), % Write slot 5 into the cache. The same request now has a result % available and the grouper should step out of the queue. + {ok, _} = hb_cache:write(M1, Opts), {ok, _} = - dev_process_cache:write( - ProcessGroup, - 5, + hb_cache:write_result( + [ + {ProcessGroup, #{ <<"path">> => <<"compute">>, <<"slot">> => 5 }}, + {ProcessGroup, #{ <<"path">> => <<"latest">> }} + ], #{ <<"hello">> => <<"cached">> }, Opts ), diff --git a/src/preloaded/process/dev_push.erl b/src/preloaded/process/dev_push.erl index 8a549e16d..fd82cf9bb 100644 --- a/src/preloaded/process/dev_push.erl +++ b/src/preloaded/process/dev_push.erl @@ -37,6 +37,11 @@ %% `N > 0' - recurse, with the inner `/push' %% inheriting `max-depth = N - 1'. %% Unwinds at most `N' levels deep. +-spec push( + #{ _ => _ }, + #{ slot => integer(), body => #{ _ => _ }, async => boolean(), 'max-depth' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | pid(). push(Base, Req, Opts) -> Process = lib_process:as_process(Base, Opts), ?event(push, {push_base, {base, Process}, {req, Req}}, Opts), @@ -44,7 +49,7 @@ push(Base, Req, Opts) -> no_slot -> case schedule_initial_message(Process, Req, Opts) of {ok, Assignment} -> - case find_type(hb_ao:get(<<"body">>, Assignment, Opts), Opts) of + case find_type(Req, Opts) of <<"Process">> -> ?event(push, {initializing_process, @@ -459,12 +464,15 @@ push_downstream_local(TargetID, NextSlotOnProc, Origin, Opts) -> {origin, Origin} } ), + ResultDepth = + decrement_result_depth( + hb_maps:get(<<"result-depth">>, Origin, 1, Opts) + ), BaseReq = #{ <<"path">> => <<"push">>, <<"slot">> => NextSlotOnProc, - <<"result-depth">> => - hb_maps:get(<<"result-depth">>, Origin, 1, Opts) - 1 + <<"result-depth">> => ResultDepth }, Req = case parse_max_depth(hb_maps:get(<<"max-depth">>, Origin, undefined, Opts)) of @@ -492,6 +500,15 @@ parse_max_depth(Bin) when is_binary(Bin) -> end; parse_max_depth(_) -> undefined. +decrement_result_depth(Depth) when is_integer(Depth), Depth > 0 -> Depth - 1; +decrement_result_depth(Depth) when is_binary(Depth) -> + try hb_util:int(Depth) of + N -> decrement_result_depth(N) + catch + _:_ -> 0 + end; +decrement_result_depth(_) -> 0. + %% @doc Augment the message with from-* keys, if it doesn't already have them. normalize_message(MsgToPush, Opts) -> hb_ao:set( @@ -990,7 +1007,7 @@ test_push_as_identity() -> test_multi_process_push() -> {Sender, _Receiver, MsgSlot, Opts} = setup_two_process_message(), - %% Install a catch-all `Pong' handler on the Sender so the Receiver's + %% Install a `Pong' handler on the Sender so the Receiver's %% reply (the helper's `reply_script' fires on `Action = "Ping"' and %% sends back `Action = "Reply"') is observable as `GOT PONG' in the %% Sender's `now/results/data'. @@ -999,7 +1016,9 @@ test_multi_process_push() -> Sender, << "Handlers.add(\"Pong\",\n" - " function (test) return true end,\n" + " function (test)\n" + " return (test.Action or test.action) == \"Reply\"\n" + " end,\n" " function(m)\n" " print(\"GOT PONG\")\n" " end\n" @@ -1585,9 +1604,11 @@ test_nested_push_prompts_encoding_change() -> ping_pong_script(Limit) -> << "Handlers.add(\"Ping\",\n" - " function (test) return true end,\n" + " function (test)\n" + " return (test.Action or test.action) == \"Ping\"\n" + " end,\n" " function(m)\n" - " C = tonumber(m.Count)\n" + " C = tonumber(m.Count or m.count)\n" " if C <= ", (integer_to_binary(Limit))/binary, " then\n" " Send({ Target = ao.id, Action = \"Ping\", Count = C + 1 })\n" " print(\"Ping\", C + 1)\n" @@ -1603,11 +1624,14 @@ reply_script() -> << """ Handlers.add("Reply", - { Action = "Ping" }, function(m) + return (m.Action or m.action) == "Ping" + end, + function(m) + local from = m.From or m.from print("Replying to...") - print(m.From) - Send({ Target = m.From, Action = "Reply", Message = "Pong!" }) + print(from) + Send({ Target = from, Action = "Reply", Message = "Pong!" }) print("Done.") end ) diff --git a/src/preloaded/process/dev_scheduler.erl b/src/preloaded/process/dev_scheduler.erl index c06cfee69..f1ebbdc52 100644 --- a/src/preloaded/process/dev_scheduler.erl +++ b/src/preloaded/process/dev_scheduler.erl @@ -15,7 +15,7 @@ %%% -module(dev_scheduler). --device_libraries([lib_process]). +-device_libraries([lib_process, lib_scheduler_formats]). %%% AO-Core API functions: -export([info/0]). %%% Local scheduling functions: @@ -71,6 +71,8 @@ parse_schedulers(SchedLoc) when is_binary(SchedLoc) -> ). %% @doc The default handler for the scheduler device. +-spec router(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. router(_, Base, Req, Opts) -> ?event({scheduler_router_called, {req, Req}, {opts, Opts}}), schedule(Base, Req, Opts). @@ -79,19 +81,14 @@ router(_, Base, Req, Opts) -> %% assignment. Assumes that Base is a `dev_process' or similar message, having %% a `Current-Slot' key. It stores a local cache of the schedule in the %% `priv/To-Process' key. +-spec next(#{ 'at-slot' := integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := #{ _ => _ }, state := #{ _ => _ }, _ => _ }} | {error, _}. next(Base, Req, Opts) -> ?event(debug_next, {scheduler_next_called, {base, Base}, {req, Req}}), ?event(next, started_next), ?event(next_profiling, started_next), Schedule = message_cached_assignments(Base, Opts), - LastProcessed = - hb_util:int( - hb_ao:get( - <<"at-slot">>, - Base, - Opts#{ <<"hashpath">> => ignore } - ) - ), + LastProcessed = maps:get(<<"at-slot">>, Base), ?event(next_profiling, got_last_processed), ?event(debug_next, {in_message_cache, {schedule, Schedule}}), ?event(next, {last_processed, LastProcessed, {message_cache, length(Schedule)}}), @@ -346,6 +343,8 @@ check_lookahead_and_local_cache(undefined, ProcID, TargetSlot, Opts) -> end. %% @doc Returns information about the entire scheduler. +-spec status(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ address := binary(), processes := [binary()], 'cache-control' := binary(), _ => _ }}. status(_M1, _M2, _Opts) -> ?event(getting_scheduler_status), Wallet = dev_scheduler_registry:get_wallet(), @@ -363,11 +362,16 @@ status(_M1, _M2, _Opts) -> %% @doc A router for choosing between getting the existing schedule, or %% scheduling a new message. +-spec schedule( + #{ _ => _ }, + #{ method => binary(), from => integer(), to => integer(), accept => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ } | binary()} | {error, _}. schedule(Base, Req, Opts) -> ?event({resolving_schedule_request, {req, Req}, {state_msg, Base}}), - case hb_util:key_to_atom(hb_ao:get(<<"method">>, Req, <<"GET">>, Opts)) of - post -> post_schedule(Base, Req, Opts); - get -> get_schedule(Base, Req, Opts) + case hb_util:to_lower(maps:get(<<"method">>, Req, <<"GET">>)) of + <<"post">> -> post_schedule(Base, Req, Opts); + <<"get">> -> get_schedule(Base, Req, Opts) end. %% @doc Schedules a new message on the SU. Searches Base for the appropriate ID, @@ -377,17 +381,29 @@ post_schedule(Base, Req, Opts) -> ?event(scheduling_message), % Find the target message to schedule: RawToSched = find_message_to_schedule(Base, Req, Opts), - % If the message can not be properly loaded, this will throw an error - % before scheduling the message. - try hb_cache:ensure_all_loaded(RawToSched, Opts) of - ToSched -> - do_post_schedule(Base, Req, ToSched, Opts) - catch - error:{necessary_message_not_found, _, _} -> + % Filter before loading so uncommitted HTTP wrapper links do not block a + % valid signed message from being scheduled. + case hb_message:with_only_committed(RawToSched, Opts) of + {ok, OnlyCommitted} -> + try hb_cache:ensure_all_loaded(OnlyCommitted, Opts) of + ToSched -> + do_post_schedule(Base, Req, ToSched, Opts) + catch + _: {necessary_message_not_found, _, _} -> + {error, + #{ + <<"status">> => 404, + <<"body">> => <<"Cannot fully load message to schedule.">> + } + } + end; + {error, Err} -> {error, #{ - <<"status">> => 404, - <<"body">> => <<"Cannot fully load message to schedule.">> + <<"status">> => 400, + <<"body">> => <<"Message invalid: ", + "Committed components cannot be validated.">>, + <<"reason">> => Err } } end. @@ -482,14 +498,6 @@ post_local_schedule(ProcID, PID, Req, Opts) -> }; {true, <<"Process">>} -> {ok, _} = hb_cache:write(Req, Opts), - spawn( - fun() -> - {ok, Results} = hb_client_remote:upload(Req, Opts), - ?event( - {uploaded_process, {proc_id, ProcID}, {results, Results}} - ) - end - ), ?event( {registering_new_process, {proc_id, ProcID}, @@ -715,6 +723,7 @@ find_remote_scheduler(ProcID, Scheduler, Opts) -> end. %% @doc Returns information about the current slot for a process. +-spec slot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. slot(M1, M2, Opts) -> ?event({getting_current_slot, {msg, M1}}), ProcID = find_target_id(M1, M2, Opts), @@ -776,7 +785,7 @@ remote_slot(<<"ao.TN.1">>, ProcID, Node, Opts) -> % Convert the JSON object for the latest assignment into the % standardized `~scheduler@1.0' format. A = - dev_scheduler_formats:aos2_to_assignment( + lib_scheduler_formats:aos2_to_assignment( JSON, Opts ), @@ -815,17 +824,17 @@ remote_slot(<<"ao.TN.1">>, ProcID, Node, Opts) -> get_schedule(Base, Req, Opts) -> ProcID = hb_util:human_id(find_target_id(Base, Req, Opts)), From = - case hb_ao:get(<<"from">>, Req, not_found, Opts) of + case maps:get(<<"from">>, Req, not_found) of not_found -> 0; X when X < 0 -> 0; - FromRes -> hb_util:int(FromRes) + FromRes -> FromRes end, To = - case hb_ao:get(<<"to">>, Req, not_found, Opts) of + case maps:get(<<"to">>, Req, not_found) of not_found -> undefined; - ToRes -> hb_util:int(ToRes) + ToRes -> ToRes end, - Format = hb_ao:get(<<"accept">>, Req, <<"application/http">>, Opts), + Format = maps:get(<<"accept">>, Req, <<"application/http">>), ?event( {parsed_get_schedule, {process, ProcID}, @@ -845,7 +854,7 @@ get_schedule(Base, Req, Opts) -> {ok, Res} -> case uri_string:percent_decode(Format) of <<"application/aos-2">> -> - dev_scheduler_formats:assignments_to_aos2( + lib_scheduler_formats:assignments_to_aos2( ProcID, hb_ao:get( <<"assignments">>, Res, [], Opts), @@ -896,7 +905,7 @@ do_get_remote_schedule(ProcID, LocalAssignments, From, To, _, Opts) % as a bundle. We set the 'more' to `undefined' to indicate that there may % be more assignments to fetch, but we don't know for sure. Res = - dev_scheduler_formats:assignments_to_bundle( + lib_scheduler_formats:assignments_to_bundle( ProcID, LocalAssignments, undefined, @@ -995,7 +1004,7 @@ do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> cache_remote_schedule(Variant, ProcID, JSONRes, Opts), ?event(debug_aos2, {json_res, {json, JSONRes}}), Filtered = filter_json_assignments(JSONRes, To, From, Opts), - dev_scheduler_formats:aos2_to_assignments( + lib_scheduler_formats:aos2_to_assignments( ProcID, Filtered, Opts @@ -1018,7 +1027,7 @@ do_get_remote_schedule(ProcID, LocalAssignments, From, To, Redirect, Opts) -> % Merge the local assignments with the remote assignments, % and normalize the keys. Merged = - dev_scheduler_formats:assignments_to_bundle( + lib_scheduler_formats:assignments_to_bundle( ProcID, MergedAssignments = LocalAssignments ++ RemoteAssignments, hb_ao:get(<<"continues">>, NormSched, false, Opts), @@ -1272,7 +1281,7 @@ post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> ), ?event({assignment_json, AssignmentJSON}), Assignment = - dev_scheduler_formats:aos2_to_assignment( + lib_scheduler_formats:aos2_to_assignment( AssignmentJSON, Opts ), @@ -1386,9 +1395,9 @@ generate_local_schedule(Format, ProcID, From, To, Opts) -> FormatterFun = case uri_string:percent_decode(Format) of <<"application/aos-2">> -> - fun dev_scheduler_formats:assignments_to_aos2/4; + fun lib_scheduler_formats:assignments_to_aos2/4; _ -> - fun dev_scheduler_formats:assignments_to_bundle/4 + fun lib_scheduler_formats:assignments_to_bundle/4 end, Res = FormatterFun(ProcID, Assignments, More, Opts), ?event({assignments_bundle_outbound, {format, Format}, {res, Res}}), diff --git a/src/preloaded/process/dev_scheduler_cache.erl b/src/preloaded/process/dev_scheduler_cache.erl index 66f8acb7f..7c2ce6b86 100644 --- a/src/preloaded/process/dev_scheduler_cache.erl +++ b/src/preloaded/process/dev_scheduler_cache.erl @@ -91,7 +91,7 @@ read(ProcID, Slot, RawOpts) -> case hb_ao:get(<<"variant">>, Assignment, Opts) of <<"ao.TN.1">> -> Loaded = hb_cache:ensure_all_loaded(Assignment, Opts), - Norm = dev_scheduler_formats:aos2_to_assignment(Loaded, Opts), + Norm = lib_scheduler_formats:aos2_to_assignment(Loaded, Opts), ?event({normalized_aos2_assignment, Norm}), {ok, Norm}; <<"ao.N.1">> -> diff --git a/src/preloaded/process/dev_scheduler_server.erl b/src/preloaded/process/dev_scheduler_server.erl index bba82eba7..95a9bca99 100644 --- a/src/preloaded/process/dev_scheduler_server.erl +++ b/src/preloaded/process/dev_scheduler_server.erl @@ -145,7 +145,9 @@ schedule(ErlangProcID, Message) -> ErlangProcID ! {schedule, Message, self(), AbortTime}, receive {scheduled, Message, Assignment} -> - Assignment + Assignment; + {schedule_failed, Message, Reason} -> + throw({scheduler_error, {proc_id, ErlangProcID}, {reason, Reason}}) after ?DEFAULT_TIMEOUT -> throw({scheduler_timeout, {proc_id, ErlangProcID}, {message, Message}}) end. @@ -194,8 +196,9 @@ assign(State, Message, ReplyPID) -> try do_assign(State, Message, ReplyPID) catch - _Class:Reason:Stack -> + Class:Reason:Stack -> ?event({error_scheduling, {reason, Reason}, {trace, Stack}}), + ReplyPID ! {schedule_failed, Message, {Class, Reason, Stack}}, State end. @@ -253,6 +256,18 @@ do_assign(State, Message, ReplyPID) -> Assignment, State ), + CommitmentSpec = maps:get(committment_spec, State), + CommitmentDevice = commitment_device(CommitmentSpec), + UploadOpts = upload_opts(State), + ?event( + {uploading_message, + {commitment_spec, CommitmentSpec}, + {commitment_device, CommitmentDevice} + } + ), + ok = upload_with_commitment(Message, UploadOpts, CommitmentSpec), + ok = upload_assignment(Assignment, State, UploadOpts, CommitmentSpec), + ?event(uploads_complete), ?event(starting_message_write), ok = dev_scheduler_cache:write(Assignment, Opts), maybe_inform_recipient( @@ -263,10 +278,6 @@ do_assign(State, Message, ReplyPID) -> State ), ?event(writes_complete), - ?event(uploading_message), - hb_client_remote:upload(Message, Opts), - hb_client_remote:upload(Assignment, Opts), - ?event(uploads_complete), maybe_inform_recipient( remote_confirmation, ReplyPID, @@ -293,18 +304,115 @@ commit_assignment(BaseAssignment, State) -> Wallets = maps:get(wallets, State), Opts = maps:get(opts, State), CommittmentSpec = maps:get(committment_spec, State), - lists:foldr( - fun(Wallet, Assignment) -> - hb_message:commit( - Assignment, - Opts#{ <<"priv-wallet">> => Wallet }, - CommittmentSpec - ) + lists:foldl( + fun(Wallet, Acc) -> + Signed = + hb_message:commit( + BaseAssignment, + Opts#{ <<"priv-wallet">> => Wallet }, + CommittmentSpec + ), + merge_commitments(Acc, Signed) end, BaseAssignment, Wallets ). +merge_commitments(Base, Signed) -> + Signed#{ + <<"commitments">> => + maps:merge( + maps:get(<<"commitments">>, Base, #{}), + maps:get(<<"commitments">>, Signed, #{}) + ) + }. + +%% @doc Ensure an upload target is committed with the scheduler's commitment +%% spec before asking the remote uploader to publish it using that spec. +ensure_committed(Msg, Opts, CommitmentSpec) -> + Device = commitment_device(CommitmentSpec), + case lists:member(Device, hb_message:commitment_devices(Msg, Opts)) of + true -> Msg; + false -> hb_message:commit(Msg, Opts, CommitmentSpec) + end. + +commitment_device(CommitmentSpec) when is_binary(CommitmentSpec) -> + CommitmentSpec; +commitment_device(CommitmentSpec) -> + maps:get(<<"commitment-device">>, CommitmentSpec). + +upload_opts(#{ opts := Opts, wallets := [Wallet | _] }) -> + case maps:is_key(<<"priv-wallet">>, Opts) of + true -> Opts; + false -> Opts#{ <<"priv-wallet">> => Wallet } + end; +upload_opts(#{ opts := Opts }) -> + Opts. + +upload_with_commitment(Msg, Opts, CommitmentSpec) -> + Device = commitment_device(CommitmentSpec), + Committed = ensure_committed(Msg, Opts, CommitmentSpec), + Results = + lists:map( + fun(UploadMsg) -> + hb_client_remote:upload(UploadMsg, Opts, Device) + end, + upload_variants(Committed, Device, Opts) + ), + case lists:filter(fun upload_failed/1, Results) of + [] -> ok; + Errors -> {error, Errors} + end. + +upload_variants(Msg = #{ <<"commitments">> := Commitments }, Device, Opts) -> + DeviceCommitments = + maps:filter( + fun(_ID, Commitment) -> + hb_ao:get(<<"commitment-device">>, Commitment, Opts) =:= Device + end, + Commitments + ), + case maps:to_list(DeviceCommitments) of + [] -> [Msg]; + [_] -> [Msg#{ <<"commitments">> => DeviceCommitments }]; + DeviceCommitmentsList -> + [ + Msg#{ <<"commitments">> => #{ ID => Commitment } } + || + {ID, Commitment} <- DeviceCommitmentsList + ] + end; +upload_variants(Msg, _Device, _Opts) -> + [Msg]. + +upload_failed({ok, _}) -> false; +upload_failed(_) -> true. + +upload_assignment(Assignment, #{ wallets := [] }, Opts, CommitmentSpec) -> + upload_with_commitment(Assignment, Opts, CommitmentSpec); +upload_assignment(Assignment, #{ wallets := Wallets }, Opts, CommitmentSpec) -> + Device = commitment_device(CommitmentSpec), + BaseAssignment = hb_message:uncommitted(Assignment, Opts), + Results = + lists:map( + fun(Wallet) -> + hb_client_remote:upload( + hb_message:commit( + BaseAssignment, + Opts#{ <<"priv-wallet">> => Wallet }, + CommitmentSpec + ), + Opts, + Device + ) + end, + Wallets + ), + case lists:filter(fun upload_failed/1, Results) of + [] -> ok; + Errors -> {error, Errors} + end. + %% @doc Potentially inform the caller that the assignment has been scheduled. %% The main assignment loop calls this function repeatedly at different stages %% of the assignment process. The scheduling mode determines which stages diff --git a/src/preloaded/process/dev_scheduler_formats.erl b/src/preloaded/process/lib_scheduler_formats.erl similarity index 97% rename from src/preloaded/process/dev_scheduler_formats.erl rename to src/preloaded/process/lib_scheduler_formats.erl index 8d178f209..ad176b68d 100644 --- a/src/preloaded/process/dev_scheduler_formats.erl +++ b/src/preloaded/process/lib_scheduler_formats.erl @@ -1,12 +1,12 @@ -%%% @doc This module is used by dev_scheduler in order to produce outputs that -%%% are compatible with various forms of AO clients. It features two main formats: +%%% @doc Shared scheduler response format helpers for devices that need outputs +%%% compatible with various forms of AO clients. It features two main formats: %%% %%% - `application/json' %%% - `application/http' %%% %%% The `application/json' format is a legacy format that is not recommended for %%% new integrations of the AO protocol. --module(dev_scheduler_formats). +-module(lib_scheduler_formats). -export([assignments_to_bundle/4, assignments_to_aos2/4]). -export([aos2_to_assignments/3, aos2_to_assignment/2]). -export([aos2_normalize_types/1]). diff --git a/src/preloaded/query/dev_copycat.erl b/src/preloaded/query/dev_copycat.erl index 61126a93d..1277831ae 100644 --- a/src/preloaded/query/dev_copycat.erl +++ b/src/preloaded/query/dev_copycat.erl @@ -11,10 +11,12 @@ %% @doc Fetch data from a GraphQL endpoint for replication. See %% `dev_copycat_graphql' for implementation details. +-spec graphql(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. graphql(Base, Request, Opts) -> dev_copycat_graphql:graphql(Base, Request, Opts). %% @doc Fetch data from an Arweave node for replication. See `dev_copycat_arweave' %% for implementation details. +-spec arweave(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. arweave(Base, Request, Opts) -> - dev_copycat_arweave:arweave(Base, Request, Opts). \ No newline at end of file + dev_copycat_arweave:arweave(Base, Request, Opts). diff --git a/src/preloaded/query/dev_match.erl b/src/preloaded/query/dev_match.erl index 2095aa98f..d3770ecaa 100644 --- a/src/preloaded/query/dev_match.erl +++ b/src/preloaded/query/dev_match.erl @@ -1,6 +1,6 @@ %%% @doc A reverse index for finding all message IDs with a given key-value pair. -module(dev_match). --export([info/0, all/3]). +-export([info/0, all/3, write/3]). -include("include/hb.hrl"). -define(CACHE_PREFIX, <<"~match@1.0">>). @@ -9,7 +9,7 @@ %% index. info() -> #{ - excludes => [<<"set">>, <<"remove">>, <<"id">>, <<"verify">>], + excludes => [<<"set">>, <<"remove">>, <<"id">>, <<"verify">>, <<"write">>], default => fun match/4 }. @@ -45,6 +45,9 @@ address(Key, Value) -> KeyBin = to_match_bin(Key), ValueBin = to_match_bin(Value), iolist_to_binary([?CACHE_PREFIX, "&", KeyBin, "=", ValueBin]). +address(Key, Value, ID) -> + IDBin = to_match_bin(ID), + <<(address(Key, Value))/binary, "/", IDBin/binary>>. to_match_bin(Bin) when is_binary(Bin) -> Bin; to_match_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom); @@ -78,8 +81,41 @@ value_path(List, Opts) when is_list(List) -> value_path(Other, Opts) -> value_path(hb_path:to_binary(Other), Opts). +%% @doc Write all keys in the base message to the match index. Expects the `Base' +%% message to already be converted to a TABM. +-spec write([binary()], #{ _ => _ }, #{ _ => _ }) -> + ok | {skip, binary()}. +write(IDs, Base, Opts) -> + case store(Opts) of + [] -> {skip, <<"No store configured for match index.">>}; + Store -> + IndexBase = hb_message:uncommitted(hb_private:reset(Base)), + maps:foreach( + fun(RawKey, Value) -> + Key = hb_ao:normalize_key(RawKey), + ValuePath = value_path(Value, Opts), + ok = hb_store:group(Store, address(Key, ValuePath), Opts), + lists:foreach( + fun(ID) -> + Address = address(Key, ValuePath, ID), + ?event( + debug_match, + {writing_reverse_index, {address, Address}, + Opts + }), + hb_store:write(Store, #{ Address => <<"">> }, Opts) + end, + IDs + ) + end, + IndexBase + ) + end. + %% @doc Match a single key-value pair in the index, returning all message IDs that %% contain the key-value pair. +-spec match(binary() | atom(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [binary()]} | {error, not_found}. match(Key, Base, _Req, Opts) -> match(Key, Base, Opts). match(Key, Base, Opts) -> Store = store(Opts), @@ -98,12 +134,11 @@ match(Key, Base, Opts) -> %% @doc Match the full base message against the index, returning the intersection %% of all matches for each key. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [binary()]} | {error, not_found}. all(Base, _Req, Opts) -> IndexBase = hb_message:uncommitted(hb_private:reset(Base)), - Keys = - hb_maps:keys( - IndexBase - ), + Keys = maps:keys(IndexBase), case Keys of [] -> {ok, []}; [FirstKey | Rest] -> diff --git a/src/preloaded/query/dev_query.erl b/src/preloaded/query/dev_query.erl index 9e6b676bb..2e4d24112 100644 --- a/src/preloaded/query/dev_query.erl +++ b/src/preloaded/query/dev_query.erl @@ -44,12 +44,15 @@ info(_Opts) -> }. %% @doc Execute the query via GraphQL. +-spec graphql(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. graphql(Req, Base, Opts) -> dev_query_graphql:handle(Req, Base, Opts). %% @doc Return whether a GraphQL esponse in a message has transaction results. %% This key is used in HB's gateway client multirequest configuration to %% determine if the response from the node should be considered admissible. +-spec has_results(#{ body => binary(), _ => _ }, #{ body => binary(), _ => _ }, #{ _ => _ }) -> + {ok, boolean()}. has_results(Base, Req, Opts) -> JSON = hb_ao:get_first( @@ -70,24 +73,32 @@ has_results(Base, Req, Opts) -> end. %% @doc Search for the keys specified in the request message. +-spec default(_, #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. default(_, Base, Req, Opts) -> all(Base, Req, Opts). %% @doc Search the node's store for all of the keys and values in the request, %% aside from the `commitments' and `path' keys. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. all(Base, Req, Opts) -> match(Req, Base, Req, Opts). %% @doc Search the node's store for all of the keys and values in the base %% message, aside from the `commitments' and `path' keys. +-spec base(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. base(Base, Req, Opts) -> match(Base, Base, Req, Opts). %% @doc Search only for the (list of) key(s) specified in `only' in the request. %% The `only' key can be a binary, a map, or a list of keys. See the moduledoc %% for semantics. +-spec only( + #{ _ => _ }, + #{ only => binary() | [binary()] | #{ _ => _ }, exclude => [binary()], return => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, _} | {error, _}. only(Base, Req, Opts) -> - case hb_maps:get(<<"only">>, Req, not_found, Opts) of + case maps:get(<<"only">>, Req, not_found) of KeyBin when is_binary(KeyBin) -> % The descriptor is a binary, so we split it on commas to get a % list of keys to search for. If there is only one key, we @@ -134,11 +145,11 @@ match(Keys, Base, Req, Opts) when is_list(Keys) -> match(UserSpec, _Base, Req, Opts) -> ?event({matching, {spec, UserSpec}}), FilteredSpec = - hb_maps:without( - hb_maps:get(<<"exclude">>, Req, ?DEFAULT_EXCLUDES, Opts), + maps:without( + maps:get(<<"exclude">>, Req, ?DEFAULT_EXCLUDES), UserSpec ), - ReturnType = hb_maps:get(<<"return">>, Req, <<"paths">>, Opts), + ReturnType = maps:get(<<"return">>, Req, <<"paths">>), ?event({matching, {spec, FilteredSpec}, {return, ReturnType}}), case hb_cache:match(FilteredSpec, Opts) of {ok, RawMatches} -> @@ -365,4 +376,4 @@ http_test() -> Opts ), ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), - ok. \ No newline at end of file + ok. diff --git a/src/preloaded/test/hb_process_test_vectors.erl b/src/preloaded/test/hb_process_test_vectors.erl index 9fb6b9285..9661f5102 100644 --- a/src/preloaded/test/hb_process_test_vectors.erl +++ b/src/preloaded/test/hb_process_test_vectors.erl @@ -296,6 +296,86 @@ wasm_compute_from_id_test_parallel() -> ?event(process_compute, {computed_message, {res, Res}}), ?assertEqual([120.0], hb_ao:get(<<"results/output">>, Res, Opts)). +compute_native_cache_ignores_request_noise_test_parallel() -> + Opts = test_opts(#{ cache_control => <<"always">>, process_async_cache => false }), + Base = + hb_message:commit( + hb_message:uncommitted(test_process(Opts), Opts), + Opts + ), + schedule_test_message(Base, <<"TEST TEXT">>, Opts), + {ok, Process} = hb_message:with_only_committed(Base, Opts), + ProcID = hb_message:id(Process, signed, Opts), + Req1 = #{ + <<"path">> => <<"compute">>, + <<"slot">> => <<"0">>, + <<"accept">> => <<"text/html">> + }, + Req2 = Req1#{ <<"accept">> := <<"application/json">> }, + {ok, Res1} = hb_ao:resolve(ProcID, Req1, Opts), + {ok, Res2} = + hb_ao:resolve( + ProcID, + Req2, + Opts#{ cache_control => <<"only-if-cached">> } + ), + ?assertEqual(0, hb_ao:get(<<"results/assignment-slot">>, Res1, Opts)), + ?assertEqual(0, hb_ao:get(<<"results/assignment-slot">>, Res2, Opts)). + +compute_native_http_hook_cache_ignores_request_noise_test_parallel_() -> + {timeout, 30, fun() -> + rand:seed(default), + Wallet = ar_wallet:new(), + Opts = test_opts(#{ + port => 10000 + rand:uniform(10000), + priv_wallet => Wallet, + cache_control => <<"always">>, + process_async_cache => false, + on => + #{ + <<"request">> => + #{ + <<"device">> => <<"rate-limit@1.0">> + } + }, + rate_limit_requests => 100, + rate_limit_period => 1_000_000, + rate_limit_max => 100 + }), + Node = hb_http_server:start_node(Opts), + Base = + hb_message:commit( + hb_message:uncommitted(test_process(Opts), Opts), + Opts + ), + ok = hb_cache:write(Base, Opts), + schedule_test_message(Base, <<"TEST TEXT">>, Opts), + ProcID = hb_util:human_id(hb_message:id(Base, all, Opts)), + Req1 = #{ + <<"path">> => << ProcID/binary, "/compute">>, + <<"slot">> => <<"0">>, + <<"accept">> => <<"text/html">>, + <<"x-real-ip">> => <<"1.2.3.4">> + }, + Req2 = Req1#{ <<"accept">> := <<"application/json">> }, + {ok, Res1} = hb_http:get(Node, Req1, Opts), + ServerID = hb_util:human_id(ar_wallet:to_address(Wallet)), + NodeOpts = hb_http_server:get_opts(#{ http_server => ServerID }), + ok = hb_http_server:set_opts(NodeOpts#{ cache_control => <<"only-if-cached">> }), + {ok, Res2} = hb_http:get(Node, Req2, Opts), + RateLimitPID = hb_name:lookup({dev_rate_limit, ServerID}), + RateLimitPID ! {balance, self(), <<"1.2.3.4">>}, + Balance = + receive + {balance, CurrentBalance} -> CurrentBalance + after 1000 -> + timeout + end, + ?assertEqual(0, hb_ao:get(<<"results/assignment-slot">>, Res1, Opts)), + ?assertEqual(0, hb_ao:get(<<"results/assignment-slot">>, Res2, Opts)), + ?assert(Balance < 99) + end}. + http_wasm_process_by_id_test_parallel() -> rand:seed(default), SchedWallet = ar_wallet:new(), @@ -495,7 +575,8 @@ do_test_restore() -> % 2. Return the variable. % Execute the first computation, then the second as a disconnected process. Opts = test_opts(#{ - <<"process-cache-frequency">> => 1 + process_snapshot_slots => 1, + process_async_cache => false }), Base = aos_process(Opts), schedule_aos_call(Base, <<"X = 42">>, Opts), diff --git a/src/preloaded/util/dev_apply.erl b/src/preloaded/util/dev_apply.erl index c1d7e813e..b1e327f78 100644 --- a/src/preloaded/util/dev_apply.erl +++ b/src/preloaded/util/dev_apply.erl @@ -29,10 +29,12 @@ info(_) -> %% @doc The default handler. If the `base' and `request' keys are present in %% the given request, then the `pair' function is called. Otherwise, the `eval' %% key is used to resolve the request. +-spec default(binary(), #{ _ => _ }, #{ base => _, request => _, source => _, _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. default(Key, Base, Request, Opts) -> ?event(debug_apply, {req, {key, Key}, {base, Base}, {request, Request}}), - FoundBase = hb_maps:get(<<"base">>, Request, not_found, Opts), - FoundRequest = hb_maps:get(<<"request">>, Request, not_found, Opts), + FoundBase = maps:get(<<"base">>, Request, not_found), + FoundRequest = maps:get(<<"request">>, Request, not_found), case {FoundBase, FoundRequest} of {B, R} when B =/= not_found andalso R =/= not_found -> pair(Key, Base, Request, Opts); @@ -55,7 +57,7 @@ eval(Base, Request, Opts) -> % If the base is not found, we return the base for this % request, minus the device (which will, inherently, be % `apply@1.0' and cause recursion). - {ok, hb_maps:without([<<"device">>], Base, Opts)} + {ok, maps:remove(<<"device">>, Base)} end, ?event({eval, {apply_base, ApplyBase}}), case find_path(<<"apply-path">>, Base, Request, Opts) of @@ -85,6 +87,8 @@ eval(Base, Request, Opts) -> end. %% @doc Apply the message found at `request' to the message found at `base'. +-spec pair(#{ _ => _ }, #{ base => _, request => _, _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. pair(Base, Request, Opts) -> pair(<<"undefined">>, Base, Request, Opts). pair(PathToSet, Base, Request, Opts) -> diff --git a/src/preloaded/util/dev_dedup.erl b/src/preloaded/util/dev_dedup.erl index eb9459cc1..62ac82071 100644 --- a/src/preloaded/util/dev_dedup.erl +++ b/src/preloaded/util/dev_dedup.erl @@ -31,6 +31,8 @@ info(_M1) -> %% @doc Forward the keys and `set' functions to the message device, handle all %% others with deduplication. This allows the device to be used in any context %% where a key is called. If the `dedup-key +-spec handle(binary(), #{ 'dedup-subject' => binary(), pass => integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {skip, #{ _ => _ }}. handle(<<"keys">>, M1, _M2, _Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, #{}); handle(<<"set">>, M1, M2, Opts) -> @@ -145,8 +147,8 @@ dedup_test() -> <<"device-stack">> => #{ <<"1">> => <<"dedup@1.0">>, - <<"2">> => append_device(<<"+D2">>), - <<"3">> => append_device(<<"+D3">>) + <<"2">> => generate_append_device(<<"+D2">>), + <<"3">> => generate_append_device(<<"+D3">>) }, <<"result">> => <<"INIT">> }, @@ -177,8 +179,8 @@ dedup_with_multipass_test() -> <<"device-stack">> => #{ <<"1">> => <<"dedup@1.0">>, - <<"2">> => append_device(<<"+D2">>), - <<"3">> => append_device(<<"+D3">>), + <<"2">> => generate_append_device(<<"+D2">>), + <<"3">> => generate_append_device(<<"+D3">>), <<"4">> => <<"multipass@1.0">> }, <<"result">> => <<"INIT">>, @@ -196,13 +198,14 @@ dedup_with_multipass_test() -> Msg5 ). -%% @doc Generate a test device that appends to a `result' key. -append_device(Separator) -> +generate_append_device(Separator) -> #{ append => fun(M1 = #{ <<"pass">> := 3 }, _) -> + % Stop after 3 passes. {ok, M1}; (M1 = #{ <<"result">> := Existing }, #{ <<"bin">> := New }) -> + ?event({appending, {existing, Existing}, {new, New}}), {ok, M1#{ <<"result">> => << Existing/binary, Separator/binary, New/binary>> }} diff --git a/src/preloaded/util/dev_multipass.erl b/src/preloaded/util/dev_multipass.erl index 49695d9ef..fcc3e235e 100644 --- a/src/preloaded/util/dev_multipass.erl +++ b/src/preloaded/util/dev_multipass.erl @@ -13,6 +13,8 @@ info(_M1) -> %% @doc Forward the keys function to the message device, handle all others %% with deduplication. We only act on the first pass. +-spec handle(binary(), #{ passes => integer(), pass => integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {pass, #{ _ => _ }}. handle(<<"keys">>, M1, _M2, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, Opts); handle(<<"set">>, M1, M2, Opts) -> diff --git a/src/preloaded/util/dev_patch.erl b/src/preloaded/util/dev_patch.erl index 4e288ab9e..8c46446ca 100644 --- a/src/preloaded/util/dev_patch.erl +++ b/src/preloaded/util/dev_patch.erl @@ -28,20 +28,28 @@ -include_lib("include/hb.hrl"). %% @doc Necessary hooks for compliance with the `execution-device' standard. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Base, _Req, _Opts) -> {ok, Base}. +-spec normalize(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. normalize(Base, _Req, _Opts) -> {ok, Base}. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. snapshot(Base, _Req, _Opts) -> {ok, Base}. +-spec compute(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. compute(Base, Req, Opts) -> patches(Base, Req, Opts). %% @doc Get the value found at the `patch-from' key of the message, or the %% `from' key if the former is not present. Remove it from the message and set %% the new source to the value found. +-spec all(#{ _ => _ }, #{ from => binary(), to => binary(), 'patch-from' => binary(), 'patch-to' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. all(Base, Req, Opts) -> move(all, Base, Req, Opts). %% @doc Find relevant `PATCH' messages in the given source key of the execution %% and request messages, and apply them to the given destination key of the %% request. +-spec patches(#{ _ => _ }, #{ from => binary(), to => binary(), 'patch-from' => binary(), 'patch-to' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. patches(Base, Req, Opts) -> move(patches, Base, Req, Opts). @@ -107,18 +115,17 @@ move(Mode, Base, Req, Opts) -> patches -> maps:fold( fun(Key, Msg, {PatchAcc, NewSourceAcc}) -> - Method = hb_ao:get(<<"method">>, Msg, Opts) + Method = maps:get(<<"method">>, Msg, undefined) == <<"PATCH">>, - Device = hb_ao:get(<<"device">>, Msg, Opts) + Device = maps:get(<<"device">>, Msg, undefined) == <<"patch@1.0">>, if Method orelse Device -> { PatchAcc#{ Key => - hb_maps:without( + maps:without( [<<"commitments">>, <<"Tags">>], - Msg, - Opts + Msg ) }, NewSourceAcc @@ -405,4 +412,4 @@ custom_set_patch_test() -> ?event(debug_test, {resolved_state, State3}), ?assertEqual(<<"1">>, hb_ao:get(<<"balances/", ID1/binary>>, State3, #{})), ?assertEqual(<<"50">>, hb_ao:get(<<"balances/A">>, State3, #{})), - ?assertEqual(<<"500">>, hb_ao:get(<<"balances/", ID2/binary>>, State3, #{})). \ No newline at end of file + ?assertEqual(<<"500">>, hb_ao:get(<<"balances/", ID2/binary>>, State3, #{})). diff --git a/src/preloaded/util/dev_relay.erl b/src/preloaded/util/dev_relay.erl index 924635e1f..639e321c8 100644 --- a/src/preloaded/util/dev_relay.erl +++ b/src/preloaded/util/dev_relay.erl @@ -29,6 +29,11 @@ %% - `method': The method to use for the request. Defaults to the original method. %% - `commit-request': Whether the request should be committed before dispatching. %% Defaults to `false'. +-spec call( + #{ _ => _ }, + #{ target => binary(), 'relay-path' => binary(), method => binary(), peer => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. call(M1, RawM2, Opts) -> ?event({relay_call, {m1, M1}, {raw_m2, RawM2}}), {ok, BaseTarget} = hb_message:find_target(M1, RawM2, Opts), @@ -94,15 +99,10 @@ call(M1, RawM2, Opts) -> }, TargetMod3 = case RelayDevice of - not_found -> hb_maps:without([<<"device">>], TargetMod2); + not_found -> maps:remove(<<"device">>, TargetMod2); _ -> TargetMod2#{<<"device">> => RelayDevice} end, - TargetMod4 = - hb_maps:without( - [<<"commitments">>], - TargetMod3, - Opts - ), + TargetMod4 = maps:remove(<<"commitments">>, TargetMod3), Commit = hb_ao:get_first( [ @@ -157,19 +157,22 @@ call(M1, RawM2, Opts) -> end, case Res of {ok, R} -> - {ok, hb_maps:without([<<"set-cookie">>], R)}; + {ok, maps:remove(<<"set-cookie">>, R)}; Err -> Err end. %% @doc Execute a request in the same way as `call/3', but asynchronously. Always %% returns `<<"OK">>'. +-spec cast(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()}. cast(M1, M2, Opts) -> spawn(fun() -> call(M1, M2, Opts) end), {ok, <<"OK">>}. %% @doc Preprocess a request to check if it should be relayed to a different node. -request(_Base, Req, Opts) -> +-spec request(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ body := [#{ _ => _ }], _ => _ }}. +request(_Base, Req, _Opts) -> {ok, #{ <<"body">> => @@ -178,8 +181,7 @@ request(_Base, Req, Opts) -> #{ <<"path">> => <<"call">>, <<"target">> => <<"body">>, - <<"body">> => - hb_ao:get(<<"request">>, Req, Opts#{ <<"hashpath">> => ignore }) + <<"body">> => maps:get(<<"request">>, Req) } ] } diff --git a/src/preloaded/util/dev_stack.erl b/src/preloaded/util/dev_stack.erl index a818c82f4..a92553eb5 100644 --- a/src/preloaded/util/dev_stack.erl +++ b/src/preloaded/util/dev_stack.erl @@ -117,20 +117,28 @@ info(Msg, Opts) -> ). %% @doc Return the default prefix for the stack. +-spec prefix(#{ 'output-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). prefix(Base, _Req, Opts) -> hb_ao:get(<<"output-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc Return the input prefix for the stack. +-spec input_prefix(#{ 'input-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). input_prefix(Base, _Req, Opts) -> hb_ao:get(<<"input-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc Return the output prefix for the stack. +-spec output_prefix(#{ 'output-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). output_prefix(Base, _Req, Opts) -> hb_ao:get(<<"output-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc The device stack key router. Sends the request to `resolve_stack', %% except for `set/2' which is handled by the default implementation in %% `dev_message'. +-spec router(binary(), _, _, #{ _ => _ }) -> + {ok, _} | {error, _}. router(<<"keys">>, Base, Request, Opts) -> ?event({keys_called, {base, Base}, {req, Request}}), hb_ao:raw(<<"message@1.0">>, <<"keys">>, Base, #{}, Opts); diff --git a/src/preloaded/util/dev_test.erl b/src/preloaded/util/dev_test.erl index 8851ecd88..aa672e1bc 100644 --- a/src/preloaded/util/dev_test.erl +++ b/src/preloaded/util/dev_test.erl @@ -4,6 +4,7 @@ -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). -export([mangle/3, update_state/3, increment_counter/3, delay/3, append/3]). -export([index/3, postprocess/3, load/3]). +-export([varied/3, varied_request/3, compute_nested/3, compute_all/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). @@ -17,6 +18,8 @@ %% @doc Exports a default_handler function that can be used to test the %% handler resolution mechanism. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := #{ _ => _ } }}. info(_) -> #{ <<"default">> => <<"message@1.0">>, @@ -49,6 +52,8 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Example index handler. +-spec index(#{ name => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), 'content-type' := binary(), _ => _ }}. index(Msg, _Req, Opts) -> Name = hb_ao:get(<<"name">>, Msg, <<"turtles">>, Opts), {ok, @@ -59,18 +64,30 @@ index(Msg, _Req, Opts) -> }. %% @doc Return a message with the device set to this module. +-spec load(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ device := binary(), _ => _ }}. load(Base, _, _Opts) -> {ok, Base#{ <<"device">> => <<"test-device@1.0">> }}. test_func(_) -> {ok, <<"GOOD FUNCTION">>}. +-spec varied(#{ x := integer() }, #{}, _) -> {ok, #{ x := integer(), _ => base }}. +varied(#{ <<"x">> := X }, _Req, _Opts) -> + {ok, #{ <<"x">> => X + 1 }}. + +-spec varied_request(#{}, #{ x := integer() }, _) -> {ok, #{ y := integer(), _ => request }}. +varied_request(_Base, #{ <<"x">> := X }, _Opts) -> + {ok, #{ <<"y">> => X + 1 }}. + %% @doc Example implementation of a `compute' handler. Makes a running list of %% the slots that have been computed in the state message and places the new %% slot number in the results key. +-spec compute(#{ 'already-seen' => [integer()], _ => _ }, #{ slot := integer() }, #{ _ => _ }) -> + {ok, #{ 'already-seen' := [integer()], results := #{ 'assignment-slot' := integer() }, _ => _ }}. compute(Base, Req, Opts) -> - AssignmentSlot = hb_ao:get(<<"slot">>, Req, Opts), - Seen = hb_ao:get(<<"already-seen">>, Base, Opts), + AssignmentSlot = maps:get(<<"slot">>, Req), + Seen = maps:get(<<"already-seen">>, Base, []), ?event({compute_called, {base, Base}, {req, Req}, {opts, Opts}}), {ok, hb_ao:set( @@ -85,13 +102,51 @@ compute(Base, Req, Opts) -> ) }. +-spec compute_nested( + #{ 'already-seen' => [integer()], _ => _ }, + #{ outer := #{ slot := integer() } }, + #{ _ => _ } +) -> {ok, #{ 'already-seen' := [integer()], results := #{ 'assignment-slot' := integer() }, _ => _ }}. +compute_nested(Base, Req, Opts) -> + AssignmentSlot = maps:get(<<"slot">>, maps:get(<<"outer">>, Req)), + Seen = maps:get(<<"already-seen">>, Base, []), + ?event({compute_called, {base, Base}, {req, Req}, {opts, Opts}}), + {ok, + hb_ao:set( + Base, + #{ + <<"random-key">> => <<"random-value">>, + <<"results">> => + #{ <<"assignment-slot">> => AssignmentSlot }, + <<"already-seen">> => [AssignmentSlot | Seen] + }, + Opts + ) + }. + +-spec compute_all(#{ a => integer(), _ => _ }, #{ slot := integer(), _ => _ }, #{ _ => _ }) -> + {ok, #{ all := binary(), _ => _ }}. +compute_all(Base, Req, Opts) -> + {ok, Base#{ <<"all">> => <<"done">> }}. + +-spec compute_all_nested( + #{ nested := #{ a := integer() }, _ => _ }, + #{ slot := integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ nested := #{ all := binary() }, _ => _ }}. +compute_all_nested(Base, Req, Opts) -> + {ok, Base#{ <<"nested">> => #{ <<"all">> => <<"done">> } }}. %% @doc Example `init/3' handler. Sets the `Already-Seen' key to an empty list. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ 'already-seen' := list(), _ => _ }}. init(Msg, _Req, Opts) -> ?event({init_called_on_dev_test, Msg}), {ok, hb_ao:set(Msg, #{ <<"already-seen">> => [] }, Opts)}. %% @doc Example `restore/3' handler. Sets the hidden key `Test/Started' to the %% value of `Current-Slot' and checks whether the `Already-Seen' key is valid. +-spec restore(#{ 'already-seen' => list(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, binary()}. restore(Msg, _Req, Opts) -> ?event({restore_called_on_dev_test, Msg}), case hb_ao:get(<<"already-seen">>, Msg, Opts) of @@ -119,6 +174,7 @@ mul(Base, Req) -> {ok, #{ <<"state">> => State, <<"results">> => [Arg1 * Arg2] }}. %% @doc Do nothing when asked to snapshot. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{}}. snapshot(Base, Req, _Opts) -> ?event({snapshot_called, {base, Base}, {req, Req}}), {ok, #{}}. @@ -133,12 +189,16 @@ append(Base, Req, Opts) -> {ok, Base#{ <<"result">> => <> }}. %% @doc Set the `postprocessor-called' key to true in the HTTP server. +-spec postprocess(#{ _ => _ }, #{ body := _, _ => _ }, #{ _ => _ }) -> + {ok, _}. postprocess(_Msg, #{ <<"body">> := Msgs }, Opts) -> ?event({postprocess_called, Opts}), hb_http_server:set_opts(Opts#{ <<"postprocessor-called">> => true }), {ok, Msgs}. %% @doc Find a test worker's PID and send it an update message. +-spec update_state(#{ _ => _ }, #{ 'test-id' => _, _ => _ }, #{ _ => _ }) -> + {ok, ok} | {error, binary()}. update_state(_Msg, Req, _Opts) -> case hb_ao:get(<<"test-id">>, Req) of not_found -> @@ -155,6 +215,8 @@ update_state(_Msg, Req, _Opts) -> end. %% @doc Find a test worker's PID and send it an increment message. +-spec increment_counter(#{ _ => _ }, #{ 'test-id' => _, _ => _ }, #{ _ => _ }) -> + {ok, ok} | {error, binary()}. increment_counter(_Base, Req, _Opts) -> case hb_ao:get(<<"test-id">>, Req) of not_found -> @@ -174,6 +236,8 @@ increment_counter(_Base, Req, _Opts) -> %% @doc Does nothing, just sleeps `Req/duration or 750' ms and returns the %% appropriate form in order to be used as a hook. +-spec delay(#{ _ => _ }, #{ duration => integer(), result => _, body => _, _ => _ }, #{ _ => _ }) -> + {ok, _}. delay(Base, Req, Opts) -> Duration = hb_ao:get_first( @@ -203,6 +267,8 @@ delay(Base, Req, Opts) -> %% %% Caution: This function is not safe to use in production, as it may cause %% state inconsistencies. +-spec mangle(#{ commitments => #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, binary()}. mangle(Base, _Req, Opts) -> case hb_opts:get(mode, prod, Opts) of prod -> {error, <<"`mangle' unavailable in `prod` mode.">>}; @@ -260,6 +326,56 @@ compute_test() -> ?assertEqual(2, hb_ao:get(<<"results/assignment-slot">>, Msg5, #{})), ?assertEqual([2, 1], hb_ao:get(<<"already-seen">>, Msg5, #{})). +varied_overlay_cache_test() -> + Store = hb_test_utils:test_store(), + Opts = + #{ + store => [Store], + priv_wallet => hb:wallet(), + cache_control => [<<"always">>] + }, + Req = #{ <<"path">> => <<"varied">> }, + Base1 = + #{ + <<"device">> => <<"test-device@1.0">>, + <<"x">> => <<"1">>, + <<"keep">> => <<"first">> + }, + {ok, Res1} = hb_ao:resolve(Base1, Req, Opts), + ?assertEqual(2, maps:get(<<"x">>, Res1)), + ?assertEqual(<<"first">>, maps:get(<<"keep">>, Res1)), + Base2 = Base1#{ <<"keep">> => <<"second">> }, + {ok, Res2} = + hb_ao:resolve( + Base2, + Req, + Opts#{ cache_control => [<<"only-if-cached">>] } + ), + ?assertEqual(2, maps:get(<<"x">>, Res2)), + ?assertEqual(<<"second">>, maps:get(<<"keep">>, Res2)). + +varied_request_overlay_hashpath_test() -> + Opts = + #{ + store => [hb_test_utils:test_store()], + priv_wallet => hb:wallet() + }, + Base = #{ <<"device">> => <<"test-device@1.0">> }, + Req = + #{ + <<"path">> => <<"varied-request">>, + <<"x">> => <<"1">>, + <<"keep">> => <<"request">> + }, + {ok, Res} = hb_ao:resolve(Base, Req, Opts), + ?assertEqual(2, maps:get(<<"y">>, Res)), + ?assertEqual(<<"request">>, maps:get(<<"keep">>, Res)), + VariedBase = hb_message:normalize_commitments(Base, Opts, fast), + ?assertEqual( + hb_path:hashpath(VariedBase, Req, Opts), + hb_path:hashpath(Res, Opts) + ). + restore_test() -> Base = #{ <<"device">> => <<"test-device@1.0">>, <<"already-seen">> => [1] }, {ok, Res} = hb_ao:resolve(Base, <<"restore">>, #{}), diff --git a/src/preloaded/vm/dev_delegated_compute.erl b/src/preloaded/vm/dev_delegated_compute.erl index 64daea028..ec90d1e7d 100644 --- a/src/preloaded/vm/dev_delegated_compute.erl +++ b/src/preloaded/vm/dev_delegated_compute.erl @@ -3,25 +3,28 @@ %%% bring trusted results into the local node, or as the `Execution-Device' of %%% an AO process. -module(dev_delegated_compute). --device_libraries([lib_process]). +-device_libraries([lib_process, lib_scheduler_formats]). -export([init/3, compute/3, normalize/3, snapshot/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Initialize or normalize the compute-lite device. For now, we don't %% need to do anything special here. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Base, _Req, _Opts) -> {ok, Base}. %% @doc We assume that the compute engine stores its own internal state, %% with snapshots triggered only when HyperBEAM requests them. Subsequently, %% to load a snapshot, we just need to return the original message. +-spec normalize(#{ snapshot => #{ type => binary(), data => _, _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. normalize(Base, _Req, Opts) -> - case hb_maps:find(<<"snapshot">>, Base, Opts) of + case maps:find(<<"snapshot">>, Base) of error -> {ok, Base}; {ok, Snapshot} -> - Unset = hb_ao:set(Base, #{ <<"snapshot">> => unset }, Opts), - case hb_maps:get(<<"type">>, Snapshot, Opts) == <<"Checkpoint">> of + Unset = maps:remove(<<"snapshot">>, Base), + case maps:get(<<"type">>, Snapshot, undefined) == <<"Checkpoint">> of false -> Unset; true -> load_state(Snapshot, Opts), @@ -32,8 +35,8 @@ normalize(Base, _Req, Opts) -> %% @doc Attempt to load a snapshot into the delegated compute server. load_state(Snapshot, Opts) -> ?event(debug_load_snapshot, {loading_snapshot, {snapshot, Snapshot}}), - Body = hb_maps:get(<<"data">>, Snapshot, Opts), - Headers = hb_maps:without([<<"data">>], Snapshot, Opts), + Body = maps:get(<<"data">>, Snapshot), + Headers = maps:remove(<<"data">>, Snapshot), Res = do_relay( <<"POST">>, <<"/state">>, @@ -50,6 +53,11 @@ load_state(Snapshot, Opts) -> %% @doc Call the delegated server to compute the result. The endpoint is %% `POST /compute' and the body is the JSON-encoded message that we want to %% evaluate. +-spec compute( + #{ _ => _ }, + #{ type => binary(), slot => integer(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. compute(Base, Req, Opts) -> OutputPrefix = hb_ao:get( @@ -63,14 +71,14 @@ compute(Base, Req, Opts) -> ProcessID = get_process_id(Base, Req, Opts), % If request is an assignment, we will compute the result % Otherwise, it is a dryrun - Type = hb_ao:get(<<"type">>, Req, not_found, Opts), + Type = maps:get(<<"type">>, Req, not_found), ?event({doing_delegated_compute, {req, Req}, {type, Type}}), % Execute the compute via external CU {Slot, Res} = case Type of <<"Assignment">> -> { - hb_ao:get(<<"slot">>, Req, Opts), + maps:get(<<"slot">>, Req), do_compute(ProcessID, Req, Opts) }; _ -> @@ -81,9 +89,9 @@ compute(Base, Req, Opts) -> %% @doc Execute computation on a remote machine via relay and the JSON-Iface. do_compute(ProcID, Req, Opts) -> ?event({do_compute_msg, {req, Req}}), - Slot = hb_ao:get(<<"slot">>, Req, Opts), + Slot = maps:get(<<"slot">>, Req), {ok, AOS2 = #{ <<"body">> := Body }} = - dev_scheduler_formats:assignments_to_aos2( + lib_scheduler_formats:assignments_to_aos2( ProcID, #{ Slot => Req @@ -137,12 +145,7 @@ do_dryrun(ProcID, Req, Opts) -> do_relay(Method, Path, Body, Headers, Opts) -> ContentType = - hb_maps:get( - <<"content-type">>, - Headers, - <<"application/json">>, - Opts - ), + maps:get(<<"content-type">>, Headers, <<"application/json">>), hb_ao:resolve( #{ <<"device">> => <<"relay@1.0">>, @@ -151,6 +154,7 @@ do_relay(Method, Path, Body, Headers, Opts) -> Headers#{ <<"path">> => <<"call">>, <<"target">> => <<"payload">>, + <<"peer">> => genesis_wasm_peer(Opts), <<"payload">> => Headers#{ <<"path">> => Path, @@ -169,7 +173,7 @@ extract_json_res(Response, Opts) -> JSONRes = hb_ao:get(<<"body">>, Res, Opts), ?event({ delegated_compute_res_metadata, - {req, hb_maps:without([<<"body">>], Res, Opts)} + {req, maps:remove(<<"body">>, Res)} }), {ok, JSONRes}; {Err, Error} when Err == error; Err == failure -> @@ -179,7 +183,7 @@ extract_json_res(Response, Opts) -> get_process_id(Base, Req, Opts) -> RawProcessID = lib_process:process_id(Base, #{}, Opts), case RawProcessID of - not_found -> hb_ao:get(<<"process-id">>, Req, Opts); + not_found -> maps:get(<<"process-id">>, Req); ProcID -> ProcID end. @@ -222,6 +226,7 @@ handle_relay_response(Base, Req, Opts, Response, OutputPrefix, ProcessID, Slot) %% @doc Generate a snapshot of a running computation by calling the %% `GET /snapshot' endpoint. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. snapshot(Msg, Req, Opts) -> ?event({snapshotting, {req, Req}}), ProcID = lib_process:process_id(Msg, #{}, Opts), @@ -233,6 +238,7 @@ snapshot(Msg, Req, Opts) -> }, #{ <<"path">> => <<"call">>, + <<"peer">> => genesis_wasm_peer(Opts), <<"relay-method">> => <<"POST">>, <<"relay-path">> => <<"/snapshot/", ProcID/binary>>, <<"content-type">> => <<"application/json">>, @@ -254,3 +260,10 @@ snapshot(Msg, Req, Opts) -> <<"error-details">> => Error }} end. + +genesis_wasm_peer(Opts) -> + Port = + integer_to_binary( + hb_opts:get(genesis_wasm_port, 6363, Opts) + ), + <<"http://localhost:", Port/binary>>. diff --git a/src/preloaded/vm/dev_genesis_wasm.erl b/src/preloaded/vm/dev_genesis_wasm.erl index 8d765fe47..1ced78c1e 100644 --- a/src/preloaded/vm/dev_genesis_wasm.erl +++ b/src/preloaded/vm/dev_genesis_wasm.erl @@ -12,9 +12,14 @@ -define(STATUS_TIMEOUT, 100). %% @doc Initialize the device. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Msg, _Req, _Opts) -> {ok, Msg}. %% @doc Normalize the device. +-spec normalize(#{ snapshot => #{ type => binary(), data => _, _ => _ }, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, #{ status := integer(), message := binary(), _ => _ }}. normalize(Msg, Req, Opts) -> case ensure_started(Opts) of true -> @@ -36,6 +41,11 @@ normalize(Msg, Req, Opts) -> %% @doc Genesis-wasm device compute handler. %% Normal compute execution through external CU with state persistence +-spec compute( + #{ _ => _ }, + #{ slot => integer(), type => binary(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. compute(Msg, Req, Opts) -> % Validate whether the genesis-wasm feature is enabled. case delegate_request(Msg, Req, Opts) of @@ -63,6 +73,8 @@ compute(Msg, Req, Opts) -> end. %% @doc Snapshot the state of the process via the `delegated-compute@1.0' device. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. snapshot(Msg, Req, Opts) -> delegate_request(Msg, Req, Opts). @@ -353,6 +365,11 @@ ensure_started(Opts) -> %% @doc Find either a specific checkpoint by its ID, or find the most recent %% checkpoint via GraphQL. +-spec import( + #{ _ => _ }, + #{ import => binary(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. import(Base, Req, Opts) -> PassedProcID = hb_maps:find(<<"process-id">>, Req, Opts), ProcMsg = @@ -459,7 +476,25 @@ do_import(Proc, CheckpointMessage, Opts) -> <<"snapshot">> => CheckpointMessage }, % Save the state snapshot into the store. - {ok, _} ?= dev_process_cache:write(ProcID, Slot, WithSnapshot, Opts), + PublicCheckpoint = maps:remove(<<"snapshot">>, WithSnapshot), + {ok, _} ?= + hb_cache:write_result( + [ + {ProcID, #{ <<"path">> => <<"compute">>, <<"slot">> => Slot }}, + {ProcID, #{ <<"path">> => <<"latest">> }} + ], + hb_private:reset(PublicCheckpoint), + Opts + ), + {ok, _} ?= + hb_cache:write_result( + [ + {ProcID, #{ <<"path">> => <<"restore">>, <<"slot">> => Slot }}, + {ProcID, #{ <<"path">> => <<"restore">> }} + ], + hb_private:reset(WithSnapshot), + Opts + ), % Return the normalized process message. {ok, WithSnapshot} else @@ -604,8 +639,8 @@ import_legacy_checkpoint() -> SnapshotData = hb_maps:get(<<"data">>, Snapshot, not_found, Opts), ?assert(byte_size(SnapshotData) > 0), ?assertMatch( - {ok, Slot, _} when Slot > 0, - dev_process_cache:latest(ProcID, Opts) + {hit, {ok, #{ <<"at-slot">> := Slot }}} when Slot > 0, + hb_cache:read_resolved(ProcID, #{ <<"path">> => <<"latest">> }, Opts) ), {ok, ActualSlot} = hb_ao:resolve(<>, Opts), diff --git a/src/preloaded/vm/dev_lua.erl b/src/preloaded/vm/dev_lua.erl index 17ca6d3a5..45f65c4b6 100644 --- a/src/preloaded/vm/dev_lua.erl +++ b/src/preloaded/vm/dev_lua.erl @@ -59,6 +59,11 @@ info(Base) -> %% @doc Initialize the device state, loading the script into memory if it is %% a reference. +-spec init( + #{ module => _ , 'content-type' => binary(), body => binary(), sandbox => _, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. init(Base, Req, Opts) -> ensure_initialized(Base, Req, Opts). @@ -228,6 +233,7 @@ initialize(Base, Modules, Opts) -> {ok, hb_private:set(Base, <<"state">>, State3, Opts)}. %%% @doc Return a list of all functions in the Lua environment. +-spec functions(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]} | {error, not_found}. functions(Base, _Req, Opts) -> case hb_private:get(<<"state">>, Base, Opts) of not_found -> @@ -268,6 +274,12 @@ sandbox(State, [Path | Rest], Opts) -> sandbox(NextState, Rest, Opts). %% @doc Call the Lua script with the given arguments. +-spec compute( + binary(), + _, + _, + #{ _ => _ } +) -> {ok, _} | {error, #{ status := integer(), _ => _ }}. compute(Key, RawBase, RawReq, Opts) -> ?event(debug_lua, compute_called), Req = @@ -374,6 +386,8 @@ process_response({error, Reason, Trace}, _Priv, _Opts) -> %% @doc Snapshot the Lua state from a live computation. Normalizes its `priv' %% state element, then serializes the state to a binary. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, binary()}. snapshot(Base, _Req, Opts) -> case hb_private:get(<<"state">>, Base, Opts) of not_found -> @@ -383,6 +397,8 @@ snapshot(Base, _Req, Opts) -> end. %% @doc Restore the Lua state from a snapshot, if it exists. +-spec normalize(#{ snapshot => #{ body => binary(), _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. normalize(Base, _Req, RawOpts) -> Opts = RawOpts#{ <<"hashpath">> => ignore }, case hb_private:get(<<"state">>, Base, Opts) of @@ -721,7 +737,7 @@ pure_lua_process_test() -> %% @doc Call a process whose `execution-device' is set to `lua@5.3a'. pure_lua_restore_test() -> - Opts = #{ <<"process-cache-frequency">> => 1 }, + Opts = #{}, Process = generate_lua_process("test/test.lua", Opts), {ok, _} = hb_cache:write(Process, Opts), Message = generate_test_message(Process, Opts, #{ <<"path">> => <<"inc">>}), diff --git a/src/preloaded/vm/dev_lua_test_ledgers.erl b/src/preloaded/vm/dev_lua_test_ledgers.erl index 1a8ff5b91..68241808e 100644 --- a/src/preloaded/vm/dev_lua_test_ledgers.erl +++ b/src/preloaded/vm/dev_lua_test_ledgers.erl @@ -374,13 +374,17 @@ verify_net_peer_balances(AllProcs, Opts) -> %% @doc Verify that a ledger's expectation of its balances with peer ledgers %% is consistent with the actual balances held. -verify_peer_balances(_ValidateID, ValidateProc, _AllProcs, Opts) -> +verify_peer_balances(_ValidateID, ValidateProc, NormProcs, Opts) -> Ledgers = ledgers(ValidateProc, Opts), maps:foreach( fun(PeerID, ExpectedBalance) -> ?assertEqual( ExpectedBalance, - balance(ValidateProc, PeerID, Opts) + balance( + maps:get(PeerID, NormProcs), + hb_message:id(ValidateProc, all), + Opts + ) ) end, Ledgers diff --git a/src/preloaded/vm/dev_wasi.erl b/src/preloaded/vm/dev_wasi.erl index e8433a80c..3c1373622 100644 --- a/src/preloaded/vm/dev_wasi.erl +++ b/src/preloaded/vm/dev_wasi.erl @@ -41,6 +41,7 @@ %% - Empty stdio files %% - WASI-preview-1 compatible functions for accessing the filesystem %% - File descriptors for those files. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(M1, _M2, Opts) -> ?event(running_init), MsgWithLib = @@ -77,6 +78,8 @@ stdout(M) -> %% @doc Adds a file descriptor to the state message. %path_open(M, Instance, [FDPtr, LookupFlag, PathPtr|_]) -> +-spec path_open(#{ _ => _ }, #{ args := [integer()], _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. path_open(Base, Req, Opts) -> FDs = hb_ao:get(<<"file-descriptors">>, Base, Opts), Instance = hb_private:get(<<"instance">>, Base, Opts), @@ -111,6 +114,8 @@ path_open(Base, Req, Opts) -> %% @doc WASM stdlib implementation of `fd_write', using the WASI-p1 standard %% interface. +-spec fd_write(#{ state := #{ _ => _ }, _ => _ }, #{ args := [integer()], 'func-sig' => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. fd_write(Base, Req, Opts) -> State = hb_ao:get(<<"state">>, Base, Opts), Instance = hb_private:get(<<"wasm/instance">>, State, Opts), @@ -165,6 +170,8 @@ fd_write(S, Instance, [FDnum, Ptr, Vecs, RetPtr], BytesWritten, Opts) -> ). %% @doc Read from a file using the WASI-p1 standard interface. +-spec fd_read(#{ state := #{ _ => _ }, _ => _ }, #{ args := [integer()], 'func-sig' => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. fd_read(Base, Req, Opts) -> State = hb_ao:get(<<"state">>, Base, Opts), Instance = hb_private:get(<<"wasm/instance">>, State, Opts), @@ -218,6 +225,8 @@ parse_iovec(Instance, Ptr) -> {BinPtr, Len}. %%% Misc WASI-preview-1 handlers. +-spec clock_time_get(#{ state := #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. clock_time_get(Base, _Req, Opts) -> ?event({clock_time_get, {returning, 1}}), State = hb_ao:get(<<"state">>, Base, Opts), diff --git a/src/preloaded/vm/dev_wasm.erl b/src/preloaded/vm/dev_wasm.erl index 182891871..7580327ba 100644 --- a/src/preloaded/vm/dev_wasm.erl +++ b/src/preloaded/vm/dev_wasm.erl @@ -49,6 +49,11 @@ info(_Base, _Opts) -> %% @doc Boot a WASM image on the image stated in the `process/image' field of %% the message. +-spec init( + #{ body => binary(), image => binary() | #{ _ => _ }, 'input-prefix' => binary(), _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. init(M1, _M2, Opts) -> ?event(running_init), % Where we should read initial parameters from. @@ -171,6 +176,15 @@ default_import_resolver(Base, Req, Opts) -> %% @doc Call the WASM executor with a message that has been prepared by a prior %% pass. +-spec compute( + #{ pass => integer(), function => binary(), parameters => [_], _ => _ }, + #{ body => binary() | #{ function => binary(), parameters => [_], _ => _ }, + function => binary(), + parameters => [_], + _ => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. compute(RawM1, M2, Opts) -> % Normalize the message to have an open WASM instance, but no literal `State'. % The hashpath is not updated during this process. This allows us to take @@ -259,6 +273,11 @@ compute(RawM1, M2, Opts) -> %% @doc Normalize the message to have an open WASM instance, but no literal %% `State' key. Ensure that we do not change the hashpath during this process. +-spec normalize( + #{ body => binary(), snapshot => #{ body => binary(), _ => _ }, 'device-key' => binary(), _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. normalize(RawM1, M2, Opts) -> ?event({normalize_raw_m1, RawM1}), M3 = @@ -295,6 +314,8 @@ normalize(RawM1, M2, Opts) -> {ok, hb_ao:set(M3, #{ <<"snapshot">> => unset }, Opts)}. %% @doc Serialize the WASM state to a binary. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary() }}. snapshot(M1, M2, Opts) -> ?event(snapshot, generating_snapshot), Instance = instance(M1, M2, Opts), @@ -306,6 +327,8 @@ snapshot(M1, M2, Opts) -> }. %% @doc Tear down the WASM executor. +-spec terminate(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. terminate(M1, M2, Opts) -> ?event(terminate_called_on_dev_wasm), Prefix = @@ -327,6 +350,7 @@ terminate(M1, M2, Opts) -> %% @doc Get the WASM instance from the message. Note that this function is exported %% such that other devices can use it, but it is excluded from calls from AO-Core %% resolution directly. +-spec instance(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> pid() | not_found | _. instance(M1, _M2, Opts) -> Prefix = hb_ao:get( @@ -345,6 +369,11 @@ instance(M1, _M2, Opts) -> %% 3. Resolving the adjusted-path-Req against the added-state-Base. %% 4. If it succeeds, return the new state from the message. %% 5. If it fails with `not_found', call the stub handler. +-spec import( + #{ _ => _ }, + #{ module := binary(), func := binary(), args => [_], 'func-sig' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. import(Base, Req, Opts) -> % 1. Adjust the path to the stdlib. ModName = hb_ao:get(<<"module">>, Req, Opts),