Skip to content

impr(store_volatile): index group children in a separate ETS table #879

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shyba wants to merge 2 commits into from
Closed

impr(store_volatile): index group children in a separate ETS table #879

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
281c42d
impr(store_volatile): index group children in a separate ETS table
shyba Apr 20, 2026
b3c03bb
impr(store_volatile): use ordered_set instead
shyba Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 176 additions & 40 deletions src/hb_store_volatile.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,29 @@ start(StoreOpts = #{ <<"name">> := Name }) ->
{read_concurrency, true},
{write_concurrency, true}
]),
Parent ! {ok, #{ <<"pid">> => self(), <<"ets-table">> => Table }},
ChildrenTable = ets:new(hb_store_volatile_children, [
ordered_set,
public,
{read_concurrency, true},
{write_concurrency, true}
]),
Parent ! {
ok,
#{
<<"pid">> => self(),
<<"ets-table">> => Table,
<<"ets-children-table">> => ChildrenTable
}
},
maybe_start_ttl_timer(StoreOpts, self()),
owner_loop(StoreOpts)
end
),
receive
{ok, InstanceMessage} ->
{ok, InstanceMessage}
after 5000 ->
{error, start_timeout}
end.

%% @doc Owner loop for the ETS store. Simply waits for a stop message and exits.
Expand Down Expand Up @@ -82,19 +97,15 @@ scope(_) -> scope().

%% @doc Remove all entries from the ETS table.
reset(Opts) ->
#{ <<"ets-table">> := Table } = hb_store:find(Opts),
{Table, ChildrenTable} = tables(Opts),
ets:delete_all_objects(Table),
ets:delete_all_objects(ChildrenTable),
?event(store_volatile, {reset, {table, Table}}),
ok.

%% @doc Write a value at the key path.
write(Opts, RawKey, Value) ->
Key = hb_store:join(RawKey),
#{ <<"ets-table">> := Table } = hb_store:find(Opts),
ensure_parent_groups(Table, Key),
?event(store_volatile, {write, {key, Key}}),
ets:insert(Table, {Key, {raw, Value}}),
ok.
put_entry(Opts, RawKey, {raw, Value}).

%% @doc Read a value, following links when needed.
read(Opts, RawKey) ->
Expand Down Expand Up @@ -140,8 +151,8 @@ list(Opts, <<"/">>) ->
list(Opts, Path) ->
ResolvedPath = resolve(Opts, Path),
case lookup_entry(Opts, ResolvedPath) of
{group, Set} ->
{ok, sets:to_list(Set)};
{group, true} ->
{ok, list_group_children(Opts, ResolvedPath)};
{link, Link} ->
list(Opts, Link);
{raw, Value} when is_map(Value) ->
Expand All @@ -158,7 +169,7 @@ type(Opts, RawKey) ->
case lookup_entry(Opts, Key) of
{raw, _} ->
simple;
{group, _} ->
{group, true} ->
composite;
{link, Link} ->
type(Opts, Link);
Expand All @@ -168,22 +179,37 @@ type(Opts, RawKey) ->

%% @doc Ensure a group exists at the given path.
make_group(Opts, RawKey) ->
Key = hb_store:join(RawKey),
#{ <<"ets-table">> := Table } = hb_store:find(Opts),
ensure_dir(Table, Key),
{Table, ChildrenTable} = tables(Opts),
ensure_dir(Table, ChildrenTable, hb_store:join(RawKey)),
ok.

%% @doc Create or replace a link from New to Existing.
make_link(_, Link, Link) ->
ok;
make_link(Opts, RawExisting, RawNew) ->
Existing = hb_store:join(RawExisting),
New = hb_store:join(RawNew),
#{ <<"ets-table">> := Table } = hb_store:find(Opts),
ensure_parent_groups(Table, New),
ets:insert(Table, {New, {link, Existing}}),
put_entry(Opts, RawNew, {link, hb_store:join(RawExisting)}).

%% @doc Install an entry at Key, running parent-index maintenance or
%% overwrite cleanup as needed.
put_entry(Opts, RawKey, Entry) ->
Key = hb_store:join(RawKey),
{Table, ChildrenTable} = tables(Opts),
?event(store_volatile, {put, {key, Key}}),
case lookup_entry(Table, Key) of
nil -> ensure_parent_groups(Table, ChildrenTable, Key);
{group, true} -> delete_group_children(Table, ChildrenTable, Key);
_ -> ok
end,
ets:insert(Table, {Key, Entry}),
ok.

tables(Opts) ->
#{
<<"ets-table">> := Table,
<<"ets-children-table">> := ChildrenTable
} = hb_store:find(Opts),
{Table, ChildrenTable}.

join_path(<<>>, Next) ->
hb_store:join(Next);
join_path(CurrPath, Next) ->
Expand All @@ -200,26 +226,53 @@ lookup_entry(Table, Key) ->
Entry
end.

ensure_parent_groups(Table, Key) ->
list_group_children(Opts, GroupPath) ->
{_, ChildrenTable} = tables(Opts),
ets:select(ChildrenTable, [{{{GroupPath, '$1'}, '_'}, [], ['$1']}]).

delete_group_children(Table, ChildrenTable, GroupPath) ->
Children = ets:select(ChildrenTable, [{{{GroupPath, '$1'}, '_'}, [], ['$1']}]),
lists:foreach(
fun(Child) ->
delete_tree(
Table,
ChildrenTable,
next_group_path(GroupPath, Child)
)
end,
Children
),
ets:select_delete(ChildrenTable, [{{{GroupPath, '_'}, '_'}, [], [true]}]).

delete_tree(Table, ChildrenTable, Path) ->
case lookup_entry(Table, Path) of
{group, true} ->
delete_group_children(Table, ChildrenTable, Path);
_ ->
ok
end,
ets:delete(Table, Path).

ensure_parent_groups(Table, ChildrenTable, Key) ->
case filename:dirname(Key) of
<<".">> ->
add_group_child(Table, ?ROOT_GROUP, filename:basename(Key));
add_group_child(Table, ChildrenTable, ?ROOT_GROUP, filename:basename(Key));
ParentDir ->
ensure_dir(Table, ParentDir),
add_group_child(Table, ParentDir, filename:basename(Key))
ensure_dir(Table, ChildrenTable, ParentDir),
add_group_child(Table, ChildrenTable, ParentDir, filename:basename(Key))
end.

ensure_dir(Table, Path) ->
ensure_dir(Table, ChildrenTable, Path) ->
PathParts = hb_path:term_to_path_parts(Path),
ensure_dir(Table, ?ROOT_GROUP, PathParts).
ensure_dir(Table, ChildrenTable, ?ROOT_GROUP, PathParts).

ensure_dir(_Table, _CurrentGroup, []) ->
ensure_dir(_Table, _ChildrenTable, _CurrentGroup, []) ->
ok;
ensure_dir(Table, CurrentGroup, [Next | Rest]) ->
add_group_child(Table, CurrentGroup, Next),
ensure_dir(Table, ChildrenTable, CurrentGroup, [Next | Rest]) ->
add_group_child(Table, ChildrenTable, CurrentGroup, Next),
NextGroup = next_group_path(CurrentGroup, Next),
ensure_group(Table, NextGroup),
ensure_dir(Table, NextGroup, Rest).
ensure_dir(Table, ChildrenTable, NextGroup, Rest).

next_group_path(?ROOT_GROUP, Next) ->
hb_store:join(Next);
Expand All @@ -228,21 +281,15 @@ next_group_path(CurrentGroup, Next) ->

ensure_group(Table, GroupPath) ->
case lookup_entry(Table, GroupPath) of
{group, _} ->
{group, true} ->
ok;
_ ->
ets:insert(Table, {GroupPath, {group, sets:new()}})
ets:insert(Table, {GroupPath, {group, true}})
end.

add_group_child(Table, GroupPath, Child) ->
Set =
case lookup_entry(Table, GroupPath) of
{group, ExistingSet} ->
ExistingSet;
_ ->
sets:new()
end,
ets:insert(Table, {GroupPath, {group, sets:add_element(Child, Set)}}),
add_group_child(Table, ChildrenTable, GroupPath, Child) ->
ensure_group(Table, GroupPath),
ets:insert(ChildrenTable, {{GroupPath, Child}, true}),
ok.

%%% Tests
Expand All @@ -264,3 +311,92 @@ max_ttl_test() ->
timer:sleep(200),
?assertEqual(not_found, hb_store:read(StoreOpts, <<"a">>)),
hb_store:stop(StoreOpts).

list_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-test">>),
hb_store:reset(StoreOpts),
?assertEqual(not_found, hb_store:list(StoreOpts, <<"colors">>)),
ok = hb_store:make_group(StoreOpts, <<"colors">>),
?assertEqual({ok, []}, hb_store:list(StoreOpts, <<"colors">>)),
ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>),
ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>),
ok = hb_store:write(StoreOpts, <<"colors/green">>, <<"3">>),
ok = hb_store:write(StoreOpts, <<"colors/multi/foo">>, <<"4">>),
ok = hb_store:write(StoreOpts, <<"colors/multi/bar">>, <<"5">>),
ok = hb_store:write(StoreOpts, <<"colors/primary/red">>, <<"6">>),
ok = hb_store:write(StoreOpts, <<"colors/nested/deep/value">>, <<"7">>),
{ok, Colors} = hb_store:list(StoreOpts, <<"colors">>),
?assertEqual(
[<<"blue">>, <<"green">>, <<"multi">>, <<"nested">>, <<"primary">>, <<"red">>],
lists:sort(Colors)
),
{ok, Multi} = hb_store:list(StoreOpts, <<"colors/multi">>),
?assertEqual([<<"bar">>, <<"foo">>], lists:sort(Multi)),
{ok, Nested} = hb_store:list(StoreOpts, <<"colors/nested">>),
?assertEqual([<<"deep">>], Nested),
ok = hb_store:stop(StoreOpts).

list_dedup_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-dedup-test">>),
hb_store:reset(StoreOpts),
ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>),
ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"2">>),
ok = hb_store:make_link(StoreOpts, <<"colors/red">>, <<"colors/alias">>),
ok = hb_store:make_link(StoreOpts, <<"colors/red">>, <<"colors/alias">>),
{ok, Colors} = hb_store:list(StoreOpts, <<"colors">>),
?assertEqual([<<"alias">>, <<"red">>], lists:sort(Colors)),
ok = hb_store:stop(StoreOpts).

list_with_link_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-link-test">>),
hb_store:reset(StoreOpts),
ok = hb_store:write(StoreOpts, <<"target/one">>, <<"1">>),
ok = hb_store:write(StoreOpts, <<"target/two">>, <<"2">>),
ok = hb_store:make_link(StoreOpts, <<"target">>, <<"shortcut">>),
{ok, Shortcut} = hb_store:list(StoreOpts, <<"shortcut">>),
?assertEqual([<<"one">>, <<"two">>], lists:sort(Shortcut)),
ok = hb_store:stop(StoreOpts).

overwrite_link_to_raw_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-link-test">>),
hb_store:reset(StoreOpts),
ok = hb_store:write(StoreOpts, <<"target/one">>, <<"1">>),
ok = hb_store:make_link(StoreOpts, <<"target">>, <<"shortcut">>),
ok = hb_store:write(StoreOpts, <<"shortcut">>, <<"replacement">>),
?assertEqual({ok, <<"replacement">>}, hb_store:read(StoreOpts, <<"shortcut">>)),
?assertEqual(not_found, hb_store:list(StoreOpts, <<"shortcut">>)),
{ok, Target} = hb_store:list(StoreOpts, <<"target">>),
?assertEqual([<<"one">>], Target),
ok = hb_store:stop(StoreOpts).

overwrite_group_to_raw_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-group-test">>),
hb_store:reset(StoreOpts),
ok = hb_store:make_group(StoreOpts, <<"colors">>),
ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>),
ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>),
ok = hb_store:write(StoreOpts, <<"colors">>, <<"replacement">>),
?assertEqual({ok, <<"replacement">>}, hb_store:read(StoreOpts, <<"colors">>)),
?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/red">>)),
?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/blue">>)),
#{ <<"ets-children-table">> := CT } = hb_store:find(StoreOpts),
?assertEqual([], ets:select(CT, [{{{<<"colors">>, '_'}, '_'}, [], [true]}])),
ok = hb_store:make_group(StoreOpts, <<"colors">>),
?assertEqual({ok, []}, hb_store:list(StoreOpts, <<"colors">>)),
ok = hb_store:stop(StoreOpts).

overwrite_group_to_link_test() ->
StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-g2l-test">>),
hb_store:reset(StoreOpts),
ok = hb_store:make_group(StoreOpts, <<"colors">>),
ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>),
ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>),
ok = hb_store:write(StoreOpts, <<"target/val">>, <<"42">>),
ok = hb_store:make_link(StoreOpts, <<"target">>, <<"colors">>),
?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/red">>)),
?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/blue">>)),
#{ <<"ets-children-table">> := CT } = hb_store:find(StoreOpts),
?assertEqual([], ets:select(CT, [{{{<<"colors">>, '_'}, '_'}, [], [true]}])),
{ok, Children} = hb_store:list(StoreOpts, <<"colors">>),
?assertEqual([<<"val">>], Children),
ok = hb_store:stop(StoreOpts).