From 838aaf4964341383ce707c8fa770867912a46ea0 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 10 Feb 2016 16:31:45 -0600 Subject: [PATCH 1/5] Update to use pluggable storage API --- src/couch_mrview.erl | 9 +++++++-- src/couch_mrview_compactor.erl | 3 +-- src/couch_mrview_util.erl | 17 ++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl index 20eb4be..f9cfe29 100644 --- a/src/couch_mrview.erl +++ b/src/couch_mrview.erl @@ -413,8 +413,13 @@ all_docs_fold(Db, #mrargs{keys=undefined}=Args, Callback, UAcc) -> update_seq=UpdateSeq, args=Args }, - [Opts] = couch_mrview_util:all_docs_key_opts(Args), - {ok, Offset, FinalAcc} = couch_db:enum_docs(Db, fun map_fold/3, Acc, Opts), + [Opts1] = couch_mrview_util:all_docs_key_opts(Args), + % TODO: This is a terrible hack for now. We'll probably have + % to rewrite _all_docs to not be part of mrview and not expect + % a btree. For now non-btree's will just have to pass 0 or + % some fake reductions to get an offset. + Opts2 = [include_reductions | Opts1], + {ok, Offset, FinalAcc} = couch_db:fold_docs(Db, fun map_fold/3, Acc, Opts2), finish_fold(FinalAcc, [{total, Total}, {offset, Offset}]); all_docs_fold(Db, #mrargs{direction=Dir, keys=Keys0}=Args, Callback, UAcc) -> {ok, Info} = couch_db:get_db_info(Db), diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl index 9dba094..a9808d0 100644 --- a/src/couch_mrview_compactor.erl +++ b/src/couch_mrview_compactor.erl @@ -52,8 +52,7 @@ compact(State) -> {ok, Fd} = couch_mrview_util:open_file(CompactFName), ESt = couch_mrview_util:reset_index(Db, Fd, State), - {ok, DbReduce} = couch_btree:full_reduce(Db#db.id_tree), - Count = element(1, DbReduce), + {ok, Count} = couch_db:get_doc_count(Db), {ESt, Count} end), diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl index 934dcd3..140f515 100644 --- a/src/couch_mrview_util.erl +++ b/src/couch_mrview_util.erl @@ -161,7 +161,7 @@ extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) -> view_sig(Db, State, View, #mrargs{include_docs=true}=Args) -> BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}), UpdateSeq = couch_db:get_update_seq(Db), - PurgeSeq = couch_db:get_purge_seq(Db), + {ok, PurgeSeq} = couch_db:get_purge_seq(Db), #mrst{ seq_indexed=SeqIndexed, keyseq_indexed=KeySeqIndexed @@ -195,9 +195,10 @@ view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) -> init_state(Db, Fd, #mrst{views=Views}=State, nil) -> + {ok, PurgeSeq} = couch_db:get_purge_seq(Db), Header = #mrheader{ seq=0, - purge_seq=couch_db:get_purge_seq(Db), + purge_seq=PurgeSeq, id_btree_state=nil, log_btree_state=nil, view_states=[make_view_state(#mrview{}) || _ <- Views] @@ -232,7 +233,9 @@ init_state(Db, Fd, State, Header) -> view_states=ViewStates } = Header, - IdBtOpts = [{compression, couch_db:compression(Db)}], + IdBtOpts = [ + {compression, couch_compress:get_compression_method()} + ], {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts), {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts); @@ -252,10 +255,10 @@ init_state(Db, Fd, State, Header) -> views=Views2 }. -open_view(Db, Fd, Lang, ViewState, View) -> +open_view(_Db, Fd, Lang, ViewState, View) -> ReduceFun = make_reduce_fun(Lang, View#mrview.reduce_funs), LessFun = maybe_define_less_fun(View), - Compression = couch_db:compression(Db), + Compression = couch_compress:get_compression_method(), BTState = get_key_btree_state(ViewState), ViewBtOpts = [ {less, LessFun}, @@ -264,7 +267,7 @@ open_view(Db, Fd, Lang, ViewState, View) -> ], {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts), - BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2, + BySeqReduceFun = fun couch_bt_engine:seq_tree_reduce/2, {ok, SeqBtree} = if View#mrview.seq_indexed -> SeqBTState = get_seq_btree_state(ViewState), ViewSeqBtOpts = [{reduce, BySeqReduceFun}, @@ -316,7 +319,7 @@ get_row_count(#mrview{btree=Bt}) -> all_docs_reduce_to_count(Reductions) -> - Reduce = fun couch_db_updater:btree_by_id_reduce/2, + Reduce = fun couch_bt_engine:id_tree_reduce/2, {Count, _, _} = couch_btree:final_reduce(Reduce, Reductions), Count. From d685f9abcf9f75e9d4ee83d9b0848ccf086bf32e Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 15 Aug 2016 09:26:48 -0400 Subject: [PATCH 2/5] Update couch_mrview for new purge API Add test for view after purge requests BugzID: 68276 --- test/couch_mrview_purge_docs_tests.erl | 123 +++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 test/couch_mrview_purge_docs_tests.erl diff --git a/test/couch_mrview_purge_docs_tests.erl b/test/couch_mrview_purge_docs_tests.erl new file mode 100644 index 0000000..8537a69 --- /dev/null +++ b/test/couch_mrview_purge_docs_tests.erl @@ -0,0 +1,123 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_mrview_purge_docs_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-define(TIMEOUT, 1000). + + +setup() -> + {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map, 5), + Db. + +teardown(Db) -> + couch_db:close(Db), + couch_server:delete(Db#db.name, [?ADMIN_CTX]), + ok. + +view_purge_test_() -> + { + "Map views", + { + setup, + fun test_util:start_couch/0, fun test_util:stop_couch/1, + { + foreach, + fun setup/0, fun teardown/1, + [ + fun test_purge_single/1, + fun test_purge_multiple/1 + ] + } + } + }. + + +test_purge_single(Db) -> + Result = run_query(Db, []), + Expect = {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?_assertEqual(Expect, Result), + + FDI = couch_db:get_full_doc_info(Db, <<"1">>), + Rev = get_rev(FDI), + {ok, _, _} = couch_db:purge_docs(Db, {<<"1">>, [Rev]}), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?_assertEqual(Expect2, Result2). + + +test_purge_multiple(Db) -> + Result = run_query(Db, []), + Expect = {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?_assertEqual(Expect, Result), + + % 1st purge request + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), + Rev1 = get_rev(FDI1), + {ok, _, _} = couch_db:purge_docs(Db, {<<"1">>, [Rev1]}), + {ok, Db2} = couch_db:reopen(Db), + + % 2nd purge request + FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), + Rev2 = get_rev(FDI2), + {ok, _, _} = couch_db:purge_docs(Db2, {<<"2">>, [Rev2]}), + {ok, Db3} = couch_db:reopen(Db2), + + % 3rd purge request + FDI3 = couch_db:get_full_doc_info(Db, <<"5">>), + Rev3 = get_rev(FDI3), + {ok, _, _} = couch_db:purge_docs(Db3, {<<"5">>, [Rev3]}), + {ok, Db4} = couch_db:reopen(Db3), + + Result2 = run_query(Db4, []), + Expect2 = {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} + ]}, + ?_assertEqual(Expect2, Result2). + + +run_query(Db, Opts) -> + couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts). + + +get_rev(#full_doc_info{} = FDI) -> + #doc_info{ + revs = [#rev_info{} = PrevRev | _] + } = couch_doc:to_doc_info(FDI), + PrevRev#rev_info.rev. From 80aeafdb5a5b66ecb0d0f1d268032e4be3e76525 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 25 Nov 2016 18:06:56 -0500 Subject: [PATCH 3/5] Update tests for view after purge requests couch_db:purge_docs now accepts a list of IdRevs this changes in the tests calls this new API BugzID: 68276 --- test/couch_mrview_purge_docs_tests.erl | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/test/couch_mrview_purge_docs_tests.erl b/test/couch_mrview_purge_docs_tests.erl index 8537a69..09712ed 100644 --- a/test/couch_mrview_purge_docs_tests.erl +++ b/test/couch_mrview_purge_docs_tests.erl @@ -59,7 +59,7 @@ test_purge_single(Db) -> FDI = couch_db:get_full_doc_info(Db, <<"1">>), Rev = get_rev(FDI), - {ok, _, _} = couch_db:purge_docs(Db, {<<"1">>, [Rev]}), + {ok, {_, _}} = couch_db:purge_docs(Db, [{<<"1">>, [Rev]}]), {ok, Db2} = couch_db:reopen(Db), Result2 = run_query(Db2, []), @@ -86,24 +86,15 @@ test_purge_multiple(Db) -> ?_assertEqual(Expect, Result), % 1st purge request - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), - Rev1 = get_rev(FDI1), - {ok, _, _} = couch_db:purge_docs(Db, {<<"1">>, [Rev1]}), - {ok, Db2} = couch_db:reopen(Db), - - % 2nd purge request - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), - Rev2 = get_rev(FDI2), - {ok, _, _} = couch_db:purge_docs(Db2, {<<"2">>, [Rev2]}), - {ok, Db3} = couch_db:reopen(Db2), + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5), - % 3rd purge request - FDI3 = couch_db:get_full_doc_info(Db, <<"5">>), - Rev3 = get_rev(FDI3), - {ok, _, _} = couch_db:purge_docs(Db3, {<<"5">>, [Rev3]}), - {ok, Db4} = couch_db:reopen(Db3), + IdsRevs = [{<<"1">>, [Rev1]}, {<<"2">>, [Rev2]}, {<<"5">>, [Rev5]}], + {ok, {_, _}} = couch_db:purge_docs(Db, IdsRevs), + {ok, Db2} = couch_db:reopen(Db), - Result2 = run_query(Db4, []), + Result2 = run_query(Db2, []), Expect2 = {ok, [ {meta, [{total, 2}, {offset, 0}]}, {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, From 5b2de8d0c8781c13b0c3f39e83179de900b18491 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 8 Dec 2016 15:47:17 -0500 Subject: [PATCH 4/5] couch_db:get_purge_seq/1 returns Seq (not {ok, Seq}) BugzID: 68276 --- src/couch_mrview_util.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl index 140f515..b835380 100644 --- a/src/couch_mrview_util.erl +++ b/src/couch_mrview_util.erl @@ -161,7 +161,7 @@ extract_view(Lang, #mrargs{view_type=red}=Args, Name, [View | Rest]) -> view_sig(Db, State, View, #mrargs{include_docs=true}=Args) -> BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}), UpdateSeq = couch_db:get_update_seq(Db), - {ok, PurgeSeq} = couch_db:get_purge_seq(Db), + PurgeSeq = couch_db:get_purge_seq(Db), #mrst{ seq_indexed=SeqIndexed, keyseq_indexed=KeySeqIndexed @@ -195,7 +195,7 @@ view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) -> init_state(Db, Fd, #mrst{views=Views}=State, nil) -> - {ok, PurgeSeq} = couch_db:get_purge_seq(Db), + PurgeSeq = couch_db:get_purge_seq(Db), Header = #mrheader{ seq=0, purge_seq=PurgeSeq, From f916079dd1bd9a778cd7cb09c8e3476be07a968f Mon Sep 17 00:00:00 2001 From: jiangphcn Date: Thu, 16 Feb 2017 16:09:47 +0800 Subject: [PATCH 5/5] =?UTF-8?q?Update=20couch=5Fmrview=20for=20clustered?= =?UTF-8?q?=20purge=20improvement=20=20-=20implementation=20of=20updating?= =?UTF-8?q?=20local=20purge=20document=20=20-=20provide=20verify=5Findex?= =?UTF-8?q?=5Fexists/1=20to=20check=20whether=20=20=20=20index=20exists=20?= =?UTF-8?q?or=20not=20=20-=20clean=20up=20local=20purge=20document=20in=20?= =?UTF-8?q?cleanup=20cycle=20=20-=20add=20new=20test=20case=20and=20refine?= =?UTF-8?q?=20existing=20test=20cases=20=20-=20create=20local=20purge=20do?= =?UTF-8?q?cument=20if=20it=20doesn=E2=80=99t=20exist=20=20=20=20when=20co?= =?UTF-8?q?uch=5Fmrview=5Findex:open/2=20is=20called=20=20-=20directly=20u?= =?UTF-8?q?se=20db=20to=20clean=20up=20local=20purge=20document=20=20-=20u?= =?UTF-8?q?se=20couch=5Fdb:purge=5Fdocs/2=20with=20UUID=20=20-=20add=20err?= =?UTF-8?q?or=20handling=20for=20verify=5Findex=5Fexists/1=20=20-=20add=20?= =?UTF-8?q?check=20for=20Sig=20in=20couch=5Fmrview=5Fcleanup:run/1=20=20-?= =?UTF-8?q?=20construct=20get=5Fsignature=5Ffrom=5Ffilename/1=20=20-=20use?= =?UTF-8?q?=20utc=5Fstring/0=20provided=20by=20couch=5Futil.erl=20=20-=20u?= =?UTF-8?q?se=20ddoc=5Fcache:open/2=20to=20open=20design=20document=20=20-?= =?UTF-8?q?=20extend=20test=20from=20couch=5Fdb=20to=20fabric?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugzid: 68276 --- src/couch_mrview_cleanup.erl | 18 ++- src/couch_mrview_index.erl | 61 +++++++++ src/couch_mrview_util.erl | 22 ++++ test/couch_mrview_purge_docs_fabric_tests.erl | 93 ++++++++++++++ test/couch_mrview_purge_docs_tests.erl | 117 ++++++++++-------- 5 files changed, 257 insertions(+), 54 deletions(-) create mode 100644 test/couch_mrview_purge_docs_fabric_tests.erl diff --git a/src/couch_mrview_cleanup.erl b/src/couch_mrview_cleanup.erl index 380376d..93c9387 100644 --- a/src/couch_mrview_cleanup.erl +++ b/src/couch_mrview_cleanup.erl @@ -41,7 +41,23 @@ run(Db) -> lists:foreach(fun(FN) -> couch_log:debug("Deleting stale view file: ~s", [FN]), - couch_file:delete(RootDir, FN, [sync]) + couch_file:delete(RootDir, FN, [sync]), + Sig = couch_mrview_util:get_signature_from_filename(FN), + if length(Sig) < 16 -> ok; true -> + case re:run(Sig,"^[a-fA-F0-9]+$",[{capture, none}]) of + match -> + DocId = couch_mrview_util:get_local_purge_doc_id(Sig), + case couch_db:open_doc(Db, DocId, []) of + {ok, LocalPurgeDoc} -> + couch_db:update_doc(Db, + LocalPurgeDoc#doc{deleted=true}, [?ADMIN_CTX]); + {not_found, _} -> + ok + end; + _ -> + ok + end + end end, ToDelete), ok. diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl index 95698bc..d4142af 100644 --- a/src/couch_mrview_index.erl +++ b/src/couch_mrview_index.erl @@ -18,6 +18,7 @@ -export([start_update/3, purge/4, process_doc/3, finish_update/1, commit/1]). -export([compact/3, swap_compacted/2]). -export([index_file_exists/1]). +-export([update_local_purge_doc/2, verify_index_exists/1]). -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). @@ -134,14 +135,17 @@ open(Db, State) -> {ok, {OldSig, Header}} -> % Matching view signatures. NewSt = couch_mrview_util:init_state(Db, Fd, State, Header), + maybe_create_local_purge_doc(Db, NewSt), {ok, NewSt}; % end of upgrade code for <= 1.2.x {ok, {Sig, Header}} -> % Matching view signatures. NewSt = couch_mrview_util:init_state(Db, Fd, State, Header), + maybe_create_local_purge_doc(Db, NewSt), {ok, NewSt}; _ -> NewSt = couch_mrview_util:reset_index(Db, Fd, State), + maybe_create_local_purge_doc(Db, NewSt), {ok, NewSt} end; {error, Reason} = Error -> @@ -204,3 +208,60 @@ index_file_exists(State) -> } = State, IndexFName = couch_mrview_util:index_file(DbName, Sig), filelib:is_file(IndexFName). + + +update_local_purge_doc(Db, State) -> + Sig = couch_index_util:hexsig(get(signature, State)), + Doc = couch_doc:from_json_obj({[ + {<<"_id">>, couch_mrview_util:get_local_purge_doc_id(Sig)}, + {<<"purge_seq">>, get(purge_seq, State)}, + {<<"timestamp_utc">>, list_to_binary(couch_util:utc_string())}, + {<<"verify_module">>, <<"couch_mrview_index">>}, + {<<"verify_function">>, <<"verify_index_exists">>}, + {<<"verify_options">>, {[ + {<<"dbname">>, get(db_name, State)}, + {<<"ddoc_id">>, get(idx_name, State)}, + {<<"signature">>, Sig} + ]}}, + {<<"type">>, <<"mrview">>} + ]}), + couch_db:update_doc(Db, Doc, []). + + +verify_index_exists(Options) -> + ShardDbName = couch_mrview_util:get_value_from_options(<<"dbname">>, Options), + DDocId = couch_mrview_util:get_value_from_options(<<"ddoc_id">>, Options), + SigInLocal = couch_mrview_util:get_value_from_options(<<"signature">>, Options), + case couch_db:open_int(ShardDbName, []) of + {ok, Db} -> + try + DbName = mem3:dbname(Db#db.name), + case ddoc_cache:open(DbName, DDocId) of + {ok, DDoc} -> + {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(ShardDbName, DDoc), + couch_index_util:hexsig(IdxState#mrst.sig) == SigInLocal; + _Else -> + false + end + catch E:T -> + Stack = erlang:get_stacktrace(), + couch_log:error("Error occurs when verifying existence of ~s/~s :: ~p ~p", + [ShardDbName, DDocId, {E, T}, Stack]), + false + after + catch couch_db:close(Db) + end; + _ -> + false + end. + + +maybe_create_local_purge_doc(Db, State) -> + Sig = couch_index_util:hexsig(get(signature, State)), + LocalPurgeDocId = couch_mrview_util:get_local_purge_doc_id(Sig), + case couch_db:open_doc(Db, LocalPurgeDocId, []) of + {not_found, _Reason} -> + update_local_purge_doc(Db, State); + {ok, _LocalPurgeDoc} -> + ok + end. diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl index b835380..0368661 100644 --- a/src/couch_mrview_util.erl +++ b/src/couch_mrview_util.erl @@ -12,6 +12,8 @@ -module(couch_mrview_util). +-export([get_local_purge_doc_id/1, get_value_from_options/2]). +-export([get_signature_from_filename/1]). -export([get_view/4]). -export([ddoc_to_mrst/2, init_state/4, reset_index/3]). -export([make_header/1]). @@ -39,6 +41,26 @@ -include_lib("couch_mrview/include/couch_mrview.hrl"). +get_local_purge_doc_id(Sig) -> + list_to_binary(?LOCAL_DOC_PREFIX ++ "purge-mrview-" ++ Sig). + + +get_value_from_options(Key, Options) -> + case couch_util:get_value(Key, Options) of + undefined -> + Reason = binary_to_list(Key) ++ " must exist in Options.", + throw({bad_request, Reason}); + Value -> Value + end. + + +get_signature_from_filename(FileName) -> + FilePathList = filename:split(FileName), + [PureFN] = lists:nthtail(length(FilePathList) - 1, FilePathList), + PureFNExt = filename:extension(PureFN), + filename:basename(PureFN, PureFNExt). + + get_view(Db, DDoc, ViewName, Args0) -> ArgCheck = fun(InitState) -> Args1 = set_view_type(Args0, ViewName, InitState#mrst.views), diff --git a/test/couch_mrview_purge_docs_fabric_tests.erl b/test/couch_mrview_purge_docs_fabric_tests.erl new file mode 100644 index 0000000..8ee3f92 --- /dev/null +++ b/test/couch_mrview_purge_docs_fabric_tests.erl @@ -0,0 +1,93 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_mrview_purge_docs_fabric_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch_mrview/include/couch_mrview.hrl"). + +-define(TIMEOUT, 1000). + + +setup() -> + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [?ADMIN_CTX]), + DbName. + + +teardown(DbName) -> + ok = fabric:delete_db(DbName, [?ADMIN_CTX]). + + +view_purge_fabric_test_() -> + { + "Map views", + { + setup, + fun() -> test_util:start_couch([fabric, mem3]) end, + fun test_util:stop_couch/1, + { + foreach, + fun setup/0, fun teardown/1, + [ + fun test_purge_verify_index/1 + ] + } + } + }. + + +test_purge_verify_index(DbName) -> + ?_test(begin + Docs1 = couch_mrview_test_util:make_docs(5), + {ok, _} = fabric:update_docs(DbName, Docs1, [?ADMIN_CTX]), + {ok, _} = fabric:update_doc(DbName, couch_mrview_test_util:ddoc(map), [?ADMIN_CTX]), + + purge_docs(DbName, [<<"1">>]), + + Result2 = fabric:query_view(DbName, <<"bar">>, <<"baz">>, #mrargs{}), + Expect2 = {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + {ok, DDoc} = fabric:open_doc(DbName, <<"_design/bar">>, []), + {ok, IdxState} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc), + Sig = IdxState#mrst.sig, + HexSig = list_to_binary(couch_index_util:hexsig(Sig)), + DocId = couch_mrview_util:get_local_purge_doc_id(HexSig), + {ok, LocPurgeDoc} = fabric:open_doc(DbName, DocId, []), + {Props} = couch_doc:to_json_obj(LocPurgeDoc,[]), + {Options} = couch_util:get_value(<<"verify_options">>, Props), + ?assertEqual(true, couch_mrview_index:verify_index_exists(Options)), + + ok + end). + +get_rev(#full_doc_info{} = FDI) -> + #doc_info{ + revs = [#rev_info{} = PrevRev | _] + } = couch_doc:to_doc_info(FDI), + PrevRev#rev_info.rev. + + +purge_docs(DbName, DocIds) -> + lists:foreach(fun(DocId) -> + FDI = fabric:get_full_doc_info(DbName, DocId, []), + Rev = get_rev(FDI), + {ok, {_, [{ok, _}]}} = fabric:purge_docs(DbName, [{DocId, [Rev]}], []) + end, DocIds). diff --git a/test/couch_mrview_purge_docs_tests.erl b/test/couch_mrview_purge_docs_tests.erl index 09712ed..f7c207c 100644 --- a/test/couch_mrview_purge_docs_tests.erl +++ b/test/couch_mrview_purge_docs_tests.erl @@ -14,6 +14,7 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). +-include_lib("couch_mrview/include/couch_mrview.hrl"). -define(TIMEOUT, 1000). @@ -46,62 +47,72 @@ view_purge_test_() -> test_purge_single(Db) -> - Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?_assertEqual(Expect, Result), - - FDI = couch_db:get_full_doc_info(Db, <<"1">>), - Rev = get_rev(FDI), - {ok, {_, _}} = couch_db:purge_docs(Db, [{<<"1">>, [Rev]}]), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 4}, {offset, 0}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?_assertEqual(Expect2, Result2). + ?_test(begin + Result = run_query(Db, []), + Expect = {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI = couch_db:get_full_doc_info(Db, <<"1">>), + Rev = get_rev(FDI), + {ok, {_, _}} = couch_db:purge_docs(Db, [{<<"UUID1">>, <<"1">>, [Rev]}]), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = {ok, [ + {meta, [{total, 4}, {offset, 0}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect2, Result2), + + ok + end). test_purge_multiple(Db) -> - Result = run_query(Db, []), - Expect = {ok, [ - {meta, [{total, 5}, {offset, 0}]}, - {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, - {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, - {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} - ]}, - ?_assertEqual(Expect, Result), - - % 1st purge request - FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), - FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2), - FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5), - - IdsRevs = [{<<"1">>, [Rev1]}, {<<"2">>, [Rev2]}, {<<"5">>, [Rev5]}], - {ok, {_, _}} = couch_db:purge_docs(Db, IdsRevs), - {ok, Db2} = couch_db:reopen(Db), - - Result2 = run_query(Db2, []), - Expect2 = {ok, [ - {meta, [{total, 2}, {offset, 0}]}, - {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, - {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} - ]}, - ?_assertEqual(Expect2, Result2). - + ?_test(begin + Result = run_query(Db, []), + Expect = {ok, [ + {meta, [{total, 5}, {offset, 0}]}, + {row, [{id, <<"1">>}, {key, 1}, {value, 1}]}, + {row, [{id, <<"2">>}, {key, 2}, {value, 2}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}, + {row, [{id, <<"5">>}, {key, 5}, {value, 5}]} + ]}, + ?assertEqual(Expect, Result), + + FDI1 = couch_db:get_full_doc_info(Db, <<"1">>), Rev1 = get_rev(FDI1), + FDI2 = couch_db:get_full_doc_info(Db, <<"2">>), Rev2 = get_rev(FDI2), + FDI5 = couch_db:get_full_doc_info(Db, <<"5">>), Rev5 = get_rev(FDI5), + + IdsRevs = [ + {<<"UUID1">>, <<"1">>, [Rev1]}, + {<<"UUID2">>, <<"2">>, [Rev2]}, + {<<"UUID5">>, <<"5">>, [Rev5]} + ], + {ok, {_, _}} = couch_db:purge_docs(Db, IdsRevs), + {ok, Db2} = couch_db:reopen(Db), + + Result2 = run_query(Db2, []), + Expect2 = {ok, [ + {meta, [{total, 2}, {offset, 0}]}, + {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}, + {row, [{id, <<"4">>}, {key, 4}, {value, 4}]} + ]}, + ?assertEqual(Expect2, Result2), + + ok + end). run_query(Db, Opts) -> couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).