diff --git a/src/hb_store_volatile.erl b/src/hb_store_volatile.erl index 0088c8eee..cd8de07bd 100644 --- a/src/hb_store_volatile.erl +++ b/src/hb_store_volatile.erl @@ -29,7 +29,20 @@ start(StoreOpts = #{ <<"name">> := Name }) -> {read_concurrency, true}, {write_concurrency, true} ]), - Parent ! {ok, #{ <<"pid">> => self(), <<"ets-table">> => Table }}, + ChildrenTable = ets:new(hb_store_volatile_children, [ + ordered_set, + public, + {read_concurrency, true}, + {write_concurrency, true} + ]), + Parent ! { + ok, + #{ + <<"pid">> => self(), + <<"ets-table">> => Table, + <<"ets-children-table">> => ChildrenTable + } + }, maybe_start_ttl_timer(StoreOpts, self()), owner_loop(StoreOpts) end @@ -37,6 +50,8 @@ start(StoreOpts = #{ <<"name">> := Name }) -> receive {ok, InstanceMessage} -> {ok, InstanceMessage} + after 5000 -> + {error, start_timeout} end. %% @doc Owner loop for the ETS store. Simply waits for a stop message and exits. @@ -82,19 +97,15 @@ scope(_) -> scope(). %% @doc Remove all entries from the ETS table. reset(Opts) -> - #{ <<"ets-table">> := Table } = hb_store:find(Opts), + {Table, ChildrenTable} = tables(Opts), ets:delete_all_objects(Table), + ets:delete_all_objects(ChildrenTable), ?event(store_volatile, {reset, {table, Table}}), ok. %% @doc Write a value at the key path. write(Opts, RawKey, Value) -> - Key = hb_store:join(RawKey), - #{ <<"ets-table">> := Table } = hb_store:find(Opts), - ensure_parent_groups(Table, Key), - ?event(store_volatile, {write, {key, Key}}), - ets:insert(Table, {Key, {raw, Value}}), - ok. + put_entry(Opts, RawKey, {raw, Value}). %% @doc Read a value, following links when needed. read(Opts, RawKey) -> @@ -140,8 +151,8 @@ list(Opts, <<"/">>) -> list(Opts, Path) -> ResolvedPath = resolve(Opts, Path), case lookup_entry(Opts, ResolvedPath) of - {group, Set} -> - {ok, sets:to_list(Set)}; + {group, true} -> + {ok, list_group_children(Opts, ResolvedPath)}; {link, Link} -> list(Opts, Link); {raw, Value} when is_map(Value) -> @@ -158,7 +169,7 @@ type(Opts, RawKey) -> case lookup_entry(Opts, Key) of {raw, _} -> simple; - {group, _} -> + {group, true} -> composite; {link, Link} -> type(Opts, Link); @@ -168,22 +179,37 @@ type(Opts, RawKey) -> %% @doc Ensure a group exists at the given path. make_group(Opts, RawKey) -> - Key = hb_store:join(RawKey), - #{ <<"ets-table">> := Table } = hb_store:find(Opts), - ensure_dir(Table, Key), + {Table, ChildrenTable} = tables(Opts), + ensure_dir(Table, ChildrenTable, hb_store:join(RawKey)), ok. %% @doc Create or replace a link from New to Existing. make_link(_, Link, Link) -> ok; make_link(Opts, RawExisting, RawNew) -> - Existing = hb_store:join(RawExisting), - New = hb_store:join(RawNew), - #{ <<"ets-table">> := Table } = hb_store:find(Opts), - ensure_parent_groups(Table, New), - ets:insert(Table, {New, {link, Existing}}), + put_entry(Opts, RawNew, {link, hb_store:join(RawExisting)}). + +%% @doc Install an entry at Key, running parent-index maintenance or +%% overwrite cleanup as needed. +put_entry(Opts, RawKey, Entry) -> + Key = hb_store:join(RawKey), + {Table, ChildrenTable} = tables(Opts), + ?event(store_volatile, {put, {key, Key}}), + case lookup_entry(Table, Key) of + nil -> ensure_parent_groups(Table, ChildrenTable, Key); + {group, true} -> delete_group_children(Table, ChildrenTable, Key); + _ -> ok + end, + ets:insert(Table, {Key, Entry}), ok. +tables(Opts) -> + #{ + <<"ets-table">> := Table, + <<"ets-children-table">> := ChildrenTable + } = hb_store:find(Opts), + {Table, ChildrenTable}. + join_path(<<>>, Next) -> hb_store:join(Next); join_path(CurrPath, Next) -> @@ -200,26 +226,53 @@ lookup_entry(Table, Key) -> Entry end. -ensure_parent_groups(Table, Key) -> +list_group_children(Opts, GroupPath) -> + {_, ChildrenTable} = tables(Opts), + ets:select(ChildrenTable, [{{{GroupPath, '$1'}, '_'}, [], ['$1']}]). + +delete_group_children(Table, ChildrenTable, GroupPath) -> + Children = ets:select(ChildrenTable, [{{{GroupPath, '$1'}, '_'}, [], ['$1']}]), + lists:foreach( + fun(Child) -> + delete_tree( + Table, + ChildrenTable, + next_group_path(GroupPath, Child) + ) + end, + Children + ), + ets:select_delete(ChildrenTable, [{{{GroupPath, '_'}, '_'}, [], [true]}]). + +delete_tree(Table, ChildrenTable, Path) -> + case lookup_entry(Table, Path) of + {group, true} -> + delete_group_children(Table, ChildrenTable, Path); + _ -> + ok + end, + ets:delete(Table, Path). + +ensure_parent_groups(Table, ChildrenTable, Key) -> case filename:dirname(Key) of <<".">> -> - add_group_child(Table, ?ROOT_GROUP, filename:basename(Key)); + add_group_child(Table, ChildrenTable, ?ROOT_GROUP, filename:basename(Key)); ParentDir -> - ensure_dir(Table, ParentDir), - add_group_child(Table, ParentDir, filename:basename(Key)) + ensure_dir(Table, ChildrenTable, ParentDir), + add_group_child(Table, ChildrenTable, ParentDir, filename:basename(Key)) end. -ensure_dir(Table, Path) -> +ensure_dir(Table, ChildrenTable, Path) -> PathParts = hb_path:term_to_path_parts(Path), - ensure_dir(Table, ?ROOT_GROUP, PathParts). + ensure_dir(Table, ChildrenTable, ?ROOT_GROUP, PathParts). -ensure_dir(_Table, _CurrentGroup, []) -> +ensure_dir(_Table, _ChildrenTable, _CurrentGroup, []) -> ok; -ensure_dir(Table, CurrentGroup, [Next | Rest]) -> - add_group_child(Table, CurrentGroup, Next), +ensure_dir(Table, ChildrenTable, CurrentGroup, [Next | Rest]) -> + add_group_child(Table, ChildrenTable, CurrentGroup, Next), NextGroup = next_group_path(CurrentGroup, Next), ensure_group(Table, NextGroup), - ensure_dir(Table, NextGroup, Rest). + ensure_dir(Table, ChildrenTable, NextGroup, Rest). next_group_path(?ROOT_GROUP, Next) -> hb_store:join(Next); @@ -228,21 +281,15 @@ next_group_path(CurrentGroup, Next) -> ensure_group(Table, GroupPath) -> case lookup_entry(Table, GroupPath) of - {group, _} -> + {group, true} -> ok; _ -> - ets:insert(Table, {GroupPath, {group, sets:new()}}) + ets:insert(Table, {GroupPath, {group, true}}) end. -add_group_child(Table, GroupPath, Child) -> - Set = - case lookup_entry(Table, GroupPath) of - {group, ExistingSet} -> - ExistingSet; - _ -> - sets:new() - end, - ets:insert(Table, {GroupPath, {group, sets:add_element(Child, Set)}}), +add_group_child(Table, ChildrenTable, GroupPath, Child) -> + ensure_group(Table, GroupPath), + ets:insert(ChildrenTable, {{GroupPath, Child}, true}), ok. %%% Tests @@ -264,3 +311,92 @@ max_ttl_test() -> timer:sleep(200), ?assertEqual(not_found, hb_store:read(StoreOpts, <<"a">>)), hb_store:stop(StoreOpts). + +list_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-test">>), + hb_store:reset(StoreOpts), + ?assertEqual(not_found, hb_store:list(StoreOpts, <<"colors">>)), + ok = hb_store:make_group(StoreOpts, <<"colors">>), + ?assertEqual({ok, []}, hb_store:list(StoreOpts, <<"colors">>)), + ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>), + ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>), + ok = hb_store:write(StoreOpts, <<"colors/green">>, <<"3">>), + ok = hb_store:write(StoreOpts, <<"colors/multi/foo">>, <<"4">>), + ok = hb_store:write(StoreOpts, <<"colors/multi/bar">>, <<"5">>), + ok = hb_store:write(StoreOpts, <<"colors/primary/red">>, <<"6">>), + ok = hb_store:write(StoreOpts, <<"colors/nested/deep/value">>, <<"7">>), + {ok, Colors} = hb_store:list(StoreOpts, <<"colors">>), + ?assertEqual( + [<<"blue">>, <<"green">>, <<"multi">>, <<"nested">>, <<"primary">>, <<"red">>], + lists:sort(Colors) + ), + {ok, Multi} = hb_store:list(StoreOpts, <<"colors/multi">>), + ?assertEqual([<<"bar">>, <<"foo">>], lists:sort(Multi)), + {ok, Nested} = hb_store:list(StoreOpts, <<"colors/nested">>), + ?assertEqual([<<"deep">>], Nested), + ok = hb_store:stop(StoreOpts). + +list_dedup_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-dedup-test">>), + hb_store:reset(StoreOpts), + ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>), + ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"2">>), + ok = hb_store:make_link(StoreOpts, <<"colors/red">>, <<"colors/alias">>), + ok = hb_store:make_link(StoreOpts, <<"colors/red">>, <<"colors/alias">>), + {ok, Colors} = hb_store:list(StoreOpts, <<"colors">>), + ?assertEqual([<<"alias">>, <<"red">>], lists:sort(Colors)), + ok = hb_store:stop(StoreOpts). + +list_with_link_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-list-link-test">>), + hb_store:reset(StoreOpts), + ok = hb_store:write(StoreOpts, <<"target/one">>, <<"1">>), + ok = hb_store:write(StoreOpts, <<"target/two">>, <<"2">>), + ok = hb_store:make_link(StoreOpts, <<"target">>, <<"shortcut">>), + {ok, Shortcut} = hb_store:list(StoreOpts, <<"shortcut">>), + ?assertEqual([<<"one">>, <<"two">>], lists:sort(Shortcut)), + ok = hb_store:stop(StoreOpts). + +overwrite_link_to_raw_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-link-test">>), + hb_store:reset(StoreOpts), + ok = hb_store:write(StoreOpts, <<"target/one">>, <<"1">>), + ok = hb_store:make_link(StoreOpts, <<"target">>, <<"shortcut">>), + ok = hb_store:write(StoreOpts, <<"shortcut">>, <<"replacement">>), + ?assertEqual({ok, <<"replacement">>}, hb_store:read(StoreOpts, <<"shortcut">>)), + ?assertEqual(not_found, hb_store:list(StoreOpts, <<"shortcut">>)), + {ok, Target} = hb_store:list(StoreOpts, <<"target">>), + ?assertEqual([<<"one">>], Target), + ok = hb_store:stop(StoreOpts). + +overwrite_group_to_raw_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-group-test">>), + hb_store:reset(StoreOpts), + ok = hb_store:make_group(StoreOpts, <<"colors">>), + ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>), + ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>), + ok = hb_store:write(StoreOpts, <<"colors">>, <<"replacement">>), + ?assertEqual({ok, <<"replacement">>}, hb_store:read(StoreOpts, <<"colors">>)), + ?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/red">>)), + ?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/blue">>)), + #{ <<"ets-children-table">> := CT } = hb_store:find(StoreOpts), + ?assertEqual([], ets:select(CT, [{{{<<"colors">>, '_'}, '_'}, [], [true]}])), + ok = hb_store:make_group(StoreOpts, <<"colors">>), + ?assertEqual({ok, []}, hb_store:list(StoreOpts, <<"colors">>)), + ok = hb_store:stop(StoreOpts). + +overwrite_group_to_link_test() -> + StoreOpts = hb_test_utils:test_store(?MODULE, <<"ets-overwrite-g2l-test">>), + hb_store:reset(StoreOpts), + ok = hb_store:make_group(StoreOpts, <<"colors">>), + ok = hb_store:write(StoreOpts, <<"colors/red">>, <<"1">>), + ok = hb_store:write(StoreOpts, <<"colors/blue">>, <<"2">>), + ok = hb_store:write(StoreOpts, <<"target/val">>, <<"42">>), + ok = hb_store:make_link(StoreOpts, <<"target">>, <<"colors">>), + ?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/red">>)), + ?assertEqual(not_found, hb_store:read(StoreOpts, <<"colors/blue">>)), + #{ <<"ets-children-table">> := CT } = hb_store:find(StoreOpts), + ?assertEqual([], ets:select(CT, [{{{<<"colors">>, '_'}, '_'}, [], [true]}])), + {ok, Children} = hb_store:list(StoreOpts, <<"colors">>), + ?assertEqual([<<"val">>], Children), + ok = hb_store:stop(StoreOpts).