Copycat L3 indexer and GraphQL

src/core/include/hb_store_arweave.hrl

@@ -0,0 +1,3 @@
+-define(SCOPE_PARENT, <<"parent">>).
+-define(SCOPE_OFFSET, <<"offset">>).
+

src/core/monitor/hb_event.erl

@@ -363,18 +363,8 @@ check_overload(Last, N) ->
             case erlang:process_info(self(), message_queue_len) of
                 {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH ->
                     {memory, MemorySize} = erlang:process_info(self(), memory),
-                    case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of
-                        1 ->
-                            ?debug_print(
-                                {warning,
-                                    prometheus_event_queue_overloading,
-                                    {queue, Len},
-                                    {last_event, Last},
-                                    {memory_bytes, MemorySize}
-                                }
-                            );
-                        _ -> ignored
-                    end,
+                    % If the size of this process is too large, exit such that
+                    % we can be restarted by the next caller.
                     case MemorySize of
                         MemorySize when MemorySize > ?MAX_MEMORY ->
                             ?debug_print(

src/core/monitor/hb_prometheus.erl

@@ -1,7 +1,7 @@
 %%% @doc HyperBEAM wrapper for Prometheus metrics.
 -module(hb_prometheus).
 -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]).
--export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4]).
+-export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4, set/4]).
 -define(STARTED_CACHE_KEY, {?MODULE, started}).
 
 %% @doc Ensure the Prometheus application has been started. Caches startup
@@ -118,4 +118,13 @@ dec(Type, Metrics, Labels, Value) ->
     end.
 
 do_dec(gauge, Name, Labels, Value) ->
-    prometheus_gauge:dec(Name, Labels, Value).
+    prometheus_gauge:dec(Name, Labels, Value).
+
+set(gauge, Name, Labels, Value) ->
+    case ensure_started() of
+        ok ->
+            try prometheus_gauge:set(Name, Labels, Value)
+            catch error:mfa_already_exists -> ok
+            end;
+        _ -> ok
+    end.

src/core/resolver/hb_opts.erl

@@ -20,6 +20,7 @@
 -include("include/hb.hrl").
 -include("include/hb_opts.hrl").
 -include("include/hb_arweave_nodes.hrl").
+-include("include/hb_store_arweave.hrl").
 -include("../../_build/hb_preloaded_index.hrl").
 
 -ifndef(PRELOADED_DEVICES_INDEX_MESSAGE_ID).
@@ -278,6 +279,11 @@ raw_default_message() ->
         <<"relay-http-client">> => httpc,
         %% The default codec to use for commitment signatures.
         <<"commitment-device">> => <<"httpsig@1.0">>,
+        %% Copycat-specific options.
+        <<"copycat-memory-budget">> => 6 * 1024 * 1024 * 1024,
+        <<"copycat-depth-recursion-cap">> => 6, % 2x the deepest we've seen to date
+        <<"arweave-block-workers">> => 3,
+        <<"copycat-scope">> => [?SCOPE_OFFSET, ?SCOPE_PARENT],
         %% Dev options
         <<"mode">> => debug,
         <<"profiling">> => true,

src/core/store/hb_store.erl

@@ -561,8 +561,6 @@ start_one(Store = #{ <<"store-module">> := Mod }, Req, Opts) ->
     end.
 
 call_store_start(Mod, Store, Req, Opts) ->
-    %% function_exported doesn't load the module. We need to call ensure_loaded
-    %% here since is the first time we call a function to load the module.
     code:ensure_loaded(Mod),
     case erlang:function_exported(Mod, start, 3) of
         true -> Mod:start(Store, Req, Opts);