Feat/ao core vary preload

PLAN.md

@@ -0,0 +1,105 @@
+**Updated Model**
+
+Yes: `#{}` is “vary on nothing except implicit keys”, and `#{ _ => _ }` is “vary on everything, preserving unmatched keys unloaded/uncoerced.” That gives us the simple all/nothing vocabulary we need.
+
+I also agree that `map()` should collapse to `_`: it means “no AO-specific shape constraint”, so the value is accepted as-is rather than treated as an empty closed message. AO message specs should use `#{...}` when they want projection semantics.
+
+**Spec Syntax**
+
+```erlang
+-spec compute(
+    #{ already_seen => [integer()] },
+    #{ slot := integer() },
+    _
+) -> {ok, #{ results := _, _ => base }}.
+```
+
+Input rules:
+
+- `_` or `map()` means unchanged/any.
+- `#{}` means no explicit dependency.
+- `#{ key := Type }` means required, load and coerce.
+- `#{ key => Type }` means optional, load and coerce if present.
+- `#{ _ => _ }` means preserve all unmatched keys as part of the varied message.
+- Base implicitly includes `device => _`.
+- Request implicitly includes `path := _`.
+- For `add_key` handlers, AO-Core overrides request `path` with the resolved key before varying.
+
+Return rules:
+
+- `#{ _ => base }` means cache the raw result, then `set` it over the original base.
+- `#{ _ => request }` means cache the raw result, then `set` it over the original request.
+- The return spec takes precedence. We should skip runtime markers for the first implementation.
+
+**Hashpath Rule**
+
+Hashpath should cover exactly the values that influence the final returned result:
+
+```erlang
+none:
+  HashBase = VariedBase
+  HashReq  = VariedReq
+
+_ => base:
+  HashBase = OldBase
+  HashReq  = VariedReq
+
+_ => request:
+  HashBase = VariedBase
+  HashReq  = OldReq
+```
+
+So your correction is right: if the result extends the original request, the full original request participates in the final hashpath.
+
+**AO-Core Flow**
+
+1. Stage 1 normalizes `OldBase/OldReq`.
+2. Resolve key/device/function once, before cache lookup.
+3. Vary using the resolved function spec.
+4. Always run `hb_message:normalize_commitments(..., fast)` on `VariedBase/VariedReq`.
+5. Cache lookup uses `VariedBase/VariedReq`.
+6. Persistent grouping uses `VariedBase/VariedReq`.
+7. Execute on `VariedBase/VariedReq`.
+8. Normalize raw result, set its generic hashpath from `VariedBase/VariedReq`, and cache only this generic result.
+9. Notify persistent waiters with the generic result, not the caller-finalized result.
+10. Each caller finalizes locally:
+    - apply overlay with `hb_ao:set(OldBase | OldReq, Result, Opts#{ hashpath => ignore, ... })`
+    - normalize commitments fast
+    - set final hashpath using the rule above
+11. Existing spawn-worker and stream continuation run with the finalized result.
+
+The point behind step 9 is the cache-hit concern I was gesturing at earlier: if two callers share `VariedBase/VariedReq` but have different original bases/requests for overlay, the leader cannot broadcast its finalized overlay result. It must broadcast the generic cached result, and each waiter applies its own overlay/hashpath.
+
+**Minimal Patch Strategy**
+
+I agree on avoiding a context map. To keep the diff tight, I’d use the existing `Opts` temp-state pattern already used for `add_key`.
+
+Minimal shape:
+
+- Add temp keys to `?TEMP_OPTS`, for example `resolved_func` and `vary`.
+- In stage 2, do function lookup plus vary, then continue with `VariedBase/VariedReq`.
+- Stage 5 first checks `resolved_func`; if present, it skips lookup.
+- Stage 9 remains the generic hashpath step for cacheability.
+- Stage 10 still caches, but only the generic varied result.
+- Stage 11 notifies waiters with the generic result, then calls one small finalization helper.
+- Cache hits and persistent waits also call that same finalization helper before returning.
+
+This keeps the change localized and avoids threading new arguments through every resolver stage.
+
+**Trade-Offs**
+
+Moving function lookup before cache means a missing/unloadable device can no longer be masked by an old exact cache hit. I think that is acceptable because variant caching cannot be sound without knowing the actual function.
+
+The only new complexity I think is unavoidable is “generic result versus finalized result.” We can minimize it by naming exactly one helper for finalization and by keeping cache, persistent notify, and wait behavior visibly tied to the generic result.
+
+**Tests I’d Add First**
+
+- `#{}` varies to implicit-only.
+- `map()` and `_` leave message unchanged.
+- `#{ _ => _ }` preserves extras without force-loading them.
+- Return `#{ _ => base }` caches generic result and overlays current base.
+- Return `#{ _ => request }` hashes with full original request.
+- Persistent waiters sharing a varied execution finalize against their own original inputs.
+- `add_key` handler uses resolved key as request `path`.
+
+If that matches your intent, I’m ready to start cutting the minimal patch.

scripts/build-preloaded-store.escript

@@ -90,6 +90,8 @@ hb_opts_compile_opts(Ebin) ->
     end.
 
 drop_outdir([{outdir, _} | Rest]) -> drop_outdir(Rest);
+drop_outdir([{d, Name, Value} | Rest]) when is_list(Name) ->
+    [{d, list_to_atom(Name), Value} | drop_outdir(Rest)];
 drop_outdir([Opt | Rest]) -> [Opt | drop_outdir(Rest)];
 drop_outdir([]) -> [].
 

scripts/hyper-token.lua

@@ -119,11 +119,25 @@ function count_common(a, b)
     if type(a) ~= "table" then a = { a } end
     if type(b) ~= "table" then b = { b } end
 
+    -- local count = 0
+    -- for _, v in ipairs(a) do
+    --     for _, w in ipairs(b) do
+    --         if v == w then
+    --             count = count + 1
+    --         end
+    --     end
+    -- end
+
+    local seen = {}
     local count = 0
     for _, v in ipairs(a) do
-        for _, w in ipairs(b) do
-            if v == w then
-                count = count + 1
+        if not seen[v] then
+            seen[v] = true
+            for _, w in ipairs(b) do
+                if v == w then
+                    count = count + 1
+                    break
+                end
             end
         end
     end
@@ -895,4 +909,4 @@ function compute(base, assignment)
         ao.event({ "Process initialized.", { slot = assignment.slot } })
         return "ok", base
     end
-end
+end

src/core/device/hb_device_archive.erl

@@ -1,6 +1,6 @@
 %%% @doc Helpers for packaged-device implementation archives.
 -module(hb_device_archive).
--export([create/2, module_metadata/1, contents/1]).
+-export([create/2, module_metadata/1, contents/1, object_code/1]).
 -export([load/1, load/4, load_modules/1, loaded/1]).
 -export([implementation_dir/1]).
 -export([modules_match_root/2, write_resources/2]).
@@ -206,18 +206,35 @@ unsafe_resource_part(_) -> false.
 %% on-load devices fall back to ordinary Erlang loading semantics.
 load_modules(Modules) ->
     case code:atomic_load(Modules) of
-        ok -> ok;
+        ok -> remember_object_code(Modules);
         {error, Reason} ->
             case atomic_load_rejected_on_load(Reason) of
-                true -> load_modules_naturally(Modules);
+                true ->
+                    case load_modules_naturally(Modules) of
+                        ok -> remember_object_code(Modules);
+                        Error -> Error
+                    end;
                 false ->
                     case loaded(Modules) of
-                        true -> ok;
+                        true -> remember_object_code(Modules);
                         false -> {error, Reason}
                     end
             end
     end.
 
+%% @doc Return the BEAM bytes for a module loaded from a device archive.
+object_code(Module) ->
+    persistent_term:get({?MODULE, object_code, Module}, undefined).
+
+remember_object_code(Modules) ->
+    lists:foreach(
+        fun({Mod, _File, Beam}) ->
+            persistent_term:put({?MODULE, object_code, Mod}, Beam)
+        end,
+        Modules
+    ),
+    ok.
+
 %% @doc Return true if `code:atomic_load/1' rejected module on-load callbacks.
 atomic_load_rejected_on_load(Reason) when is_list(Reason) ->
     lists:any(

src/core/http/hb_client_remote.erl

@@ -94,11 +94,25 @@ upload(Msg, Opts) ->
     UploadResults = 
         lists:map(
             fun(Device) ->
-                upload(Msg, Opts, Device)
+                upload(
+                    hb_message:with_commitments(
+                        #{ <<"commitment-device">> => Device },
+                        Msg,
+                        Opts
+                    ),
+                    Opts,
+                    Device
+                )
             end,
             hb_message:commitment_devices(Msg, Opts)
         ),
-    {ok, UploadResults}.
+    case lists:filter(fun upload_failed/1, UploadResults) of
+        [] -> {ok, UploadResults};
+        Errors -> {error, Errors}
+    end.
+upload_failed({error, _}) -> true;
+upload_failed({failure, _}) -> true;
+upload_failed(_) -> false.
 upload(Msg, Opts, <<"httpsig@1.0">>) ->
     case hb_opts:get(bundler_httpsig, not_found, Opts) of
         not_found ->
@@ -107,13 +121,17 @@ upload(Msg, Opts, <<"httpsig@1.0">>) ->
             ?event({uploading_item, Msg}),
             hb_http:post(Bundler, <<"/tx">>, Msg, Opts)
     end;
-upload(Msg, Opts, _CommitmentDevice) ->
+upload(Msg, Opts, CommitmentDevice) ->
     ?event({uploading_item, Msg}),
     hb_ao:raw(
         <<"arweave@2.9">>,
         <<"tx">>,
-        #{},
-        Msg#{ <<"method">> => <<"POST">> },
+        Msg,
+        #{
+            <<"method">> => <<"POST">>,
+            <<"target">> => <<"base">>,
+            <<"commitment-device">> => CommitmentDevice
+        },
         Opts
     ).
 

src/core/http/hb_http.erl

@@ -638,9 +638,15 @@ encode_reply(Status, TABMReq, Message, Opts) ->
             end,
 			Opts
         ),
+    DefaultAcceptBundle =
+        case {Codec, hb_maps:get(<<"require-codec">>, TABMReq, not_found, Opts)} of
+            {<<"json@1.0">>, not_found} -> false;
+            {<<"json@1.0">>, _} -> true;
+            _ -> false
+        end,
     AcceptBundle =
         hb_util:atom(
-            hb_maps:get(<<"accept-bundle">>, TABMReq, false, Opts)
+            hb_maps:get(<<"accept-bundle">>, TABMReq, DefaultAcceptBundle, Opts)
         ),
     ?event(debug_http,
         {encoding_reply,
@@ -1054,6 +1060,16 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) ->
         ),
     FilterKeys = hb_opts:get(http_inbound_filter_keys, ?DEFAULT_FILTER_KEYS, Opts),
     FilteredMsg = hb_message:without_unless_signed(FilterKeys, Msg, Opts),
+    DefaultAcceptBundle =
+        case maps:get(
+            <<"require-codec">>,
+            Msg,
+            maps:get(<<"require-codec">>, PrimMsg, not_found)
+        ) of
+            <<"application/json">> -> true;
+            <<"json@1.0">> -> true;
+            _ -> maps:get(<<"accept-bundle">>, RawHeaders, false)
+        end,
     BaseMsg =
         FilteredMsg#{
             <<"method">> => Method,
@@ -1065,7 +1081,7 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) ->
                     maps:get(
                         <<"accept-bundle">>,
                         PrimMsg,
-                        maps:get(<<"accept-bundle">>, RawHeaders, false)
+                        DefaultAcceptBundle
                     )
                 ),
             <<"accept">> =>