You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2018/03/22 21:17:09 UTC
[couchdb] branch COUCHDB-3326-clustered-purge-davisp-refactor
updated (05d9a3c -> 7a51482)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch COUCHDB-3326-clustered-purge-davisp-refactor
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 05d9a3c WIP - Updating read repair for a myriad edge cases
new 5c8f449 WIP - fix doc_open_revs always read repair
new 7a51482 WIP - read repair
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/fabric/src/fabric_doc_open.erl | 18 ++++++++++------
src/fabric/src/fabric_doc_open_revs.erl | 38 ++++++++++++++++-----------------
src/fabric/src/fabric_rpc.erl | 3 ++-
3 files changed, 32 insertions(+), 27 deletions(-)
--
To stop receiving notification emails like this one, please contact
davisp@apache.org.
[couchdb] 02/02: WIP - read repair
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7a51482229d4ec5d609422d827c33c199510734b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Mar 22 16:01:58 2018 -0500
WIP - read repair
---
src/fabric/src/fabric_doc_open.erl | 18 +++++++++++-------
src/fabric/src/fabric_doc_open_revs.erl | 31 ++++++++++++++++---------------
src/fabric/src/fabric_rpc.erl | 3 ++-
3 files changed, 29 insertions(+), 23 deletions(-)
diff --git a/src/fabric/src/fabric_doc_open.erl b/src/fabric/src/fabric_doc_open.erl
index b974880..445cfbb 100644
--- a/src/fabric/src/fabric_doc_open.erl
+++ b/src/fabric/src/fabric_doc_open.erl
@@ -25,8 +25,8 @@
r,
state,
replies,
- q_reply,
- replies_by_node=[] %[{Node, Reply}] used for checking if a doc is purged
+ node_id_revs = [],
+ q_reply
}).
@@ -84,8 +84,13 @@ handle_message({rexi_EXIT, _Reason}, Worker, Acc) ->
end;
handle_message(Reply, Worker, Acc) ->
NewReplies = fabric_util:update_counter(Reply, 1, Acc#acc.replies),
- NewNReplies = [{Worker#shard.node, Reply}|Acc#acc.replies_by_node],
- NewAcc = Acc#acc{replies = NewReplies, replies_by_node = NewNReplies},
+ NewNodeIdRevs = case Reply of
+ {ok #doc{id = Id, revs = {Pos, [Rev | _]}} ->
+ [{Worker#shard.node {Id, [{Pos, Rev}]}} | Acc#acc.node_id_revs];
+ _ ->
+ Acc#acc.node_id_revs
+ end,
+ NewAcc = Acc#acc{replies = NewReplies, node_id_revs = NewNodeIdRevs},
case is_r_met(Acc#acc.workers, NewReplies, Acc#acc.r) of
{true, QuorumReply} ->
fabric_util:cleanup(lists:delete(Worker, Acc#acc.workers)),
@@ -124,15 +129,14 @@ is_r_met(Workers, Replies, R) ->
no_more_workers
end.
-read_repair(#acc{dbname=DbName, replies=Replies, replies_by_node=NReplies0}) ->
+read_repair(#acc{dbname=DbName, replies=Replies, node_id_revs=NodeIdRevs}) ->
Docs = [Doc || {_, {{ok, #doc{}=Doc}, _}} <- Replies],
- NReplies = [{Node, Doc} || {Node, {ok, #doc{}=Doc}} <- NReplies0],
case Docs of
% omit local docs from read repair
[#doc{id = <<?LOCAL_DOC_PREFIX, _/binary>>} | _] ->
choose_reply(Docs);
[#doc{id=Id} | _] ->
- Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NReplies}],
+ Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NodeIdRevs}],
Res = fabric:update_docs(DbName, Docs, Opts),
case Res of
{ok, []} ->
diff --git a/src/fabric/src/fabric_doc_open_revs.erl b/src/fabric/src/fabric_doc_open_revs.erl
index 61e8afd..a020ba6 100644
--- a/src/fabric/src/fabric_doc_open_revs.erl
+++ b/src/fabric/src/fabric_doc_open_revs.erl
@@ -29,8 +29,8 @@
revs,
latest,
replies = [],
+ node_id_revs = [],
repair = false,
- replies_by_node=[] %[{Node, Reply}] used for read_repair
}).
go(DbName, Id, Revs, Options) ->
@@ -83,7 +83,7 @@ handle_message({ok, RawReplies}, Worker, State) ->
worker_count = WorkerCount,
workers = Workers,
replies = PrevReplies,
- replies_by_node = PrevNReplies,
+ node_id_revs = PrevNodeIdRevs,
r = R,
revs = Revs,
latest = Latest,
@@ -109,9 +109,14 @@ handle_message({ok, RawReplies}, Worker, State) ->
{NewReplies0, MinCount} = dict_replies(PrevReplies, RawReplies),
{NewReplies0, MinCount >= R, false}
end,
- NewNReplies = case Worker of
- nil -> PrevNReplies;
- _ -> [{Worker#shard.node, RawReplies}|PrevNReplies]
+ NewNodeIdRevs = if Woker == nil -> PrevNodeIdRevs; true ->
+ IdRevs = lists:foldl(fun
+ ({ok, #doc{id = Id, revs = {Pos, [Rev | _]}}}, Acc) ->
+ [{Id, {Pos, Rev}} | Acc];
+ (_, Acc) ->
+ Acc
+ end, [], RawReplies),
+ [{Worker#worker.shard, IdRevs} | PrevNodeIdRevs]
end,
Complete = (ReplyCount =:= (WorkerCount - 1)),
@@ -123,7 +128,7 @@ handle_message({ok, RawReplies}, Worker, State) ->
DbName,
IsTree,
NewReplies,
- NewNReplies,
+ NewNodeIdRevs,
ReplyCount + 1,
InRepair orelse Repair
),
@@ -131,7 +136,7 @@ handle_message({ok, RawReplies}, Worker, State) ->
false ->
{ok, State#state{
replies = NewReplies,
- replies_by_node = NewNReplies,
+ node_id_revs = NewNodeIdRevs,
reply_count = ReplyCount + 1,
workers = lists:delete(Worker, Workers),
repair = InRepair orelse Repair
@@ -188,7 +193,7 @@ dict_replies(Dict, [Reply | Rest]) ->
dict_replies(NewDict, Rest).
-maybe_read_repair(Db, IsTree, Replies, NodeReplies0, ReplyCount, DoRepair) ->
+maybe_read_repair(Db, IsTree, Replies, NodeIdRevs, ReplyCount, DoRepair) ->
Docs = case IsTree of
true -> tree_repair_docs(Replies, DoRepair);
false -> dict_repair_docs(Replies, ReplyCount)
@@ -197,11 +202,7 @@ maybe_read_repair(Db, IsTree, Replies, NodeReplies0, ReplyCount, DoRepair) ->
[] ->
ok;
_ ->
- NodeReplies = lists:foldl(fun({Node, NReplies}, Acc) ->
- NewAcc = [{Node, Doc} || {ok, Doc} <- NReplies],
- NewAcc ++ Acc
- end, [], NodeReplies0),
- erlang:spawn(fun() -> read_repair(Db, Docs, NodeReplies) end)
+ erlang:spawn(fun() -> read_repair(Db, Docs, NodeIdRevs) end)
end.
@@ -220,8 +221,8 @@ dict_repair_docs(Replies, ReplyCount) ->
end.
-read_repair(Db, Docs, NodeReplies) ->
- Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NodeReplies}],
+read_repair(Db, Docs, NodeIdRevs) ->
+ Opts = [replicated_changes, ?ADMIN_CTX, {read_repair, NodeIdRevs}],
Res = fabric:update_docs(Db, Docs, Opts),
case Res of
{ok, []} ->
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index 91fbb9e..d1d33e9 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -351,7 +351,8 @@ read_repair_filter_docs(DbName, NodeIdRevs, Docs, Options) ->
% NodeIdRevs are the list of {node(), {docid(), [rev()]}}
% tuples passed as the read_repair option to update_docs.
filter_purged_revs(Db, NodeIdRevs, Docs) ->
- Nodes = lists:usort([Node || {Node, _IdRevs} <- NodeIdRevs]),
+ Nodes0 = [Node || {Node, _IdRevs} <- NodeIdRevs, Node /= node()],
+ Nodes = lists:usort(Nodes0),
% Gather the list of {Node, PurgeSeq} pairs for all nodes
% that are present in our read repair group
--
To stop receiving notification emails like this one, please contact
davisp@apache.org.
[couchdb] 01/02: WIP - fix doc_open_revs always read repair
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch COUCHDB-3326-clustered-purge-davisp-refactor
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5c8f4499e69e4fb783249985bc215d99be12fa8f
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Mar 22 15:56:28 2018 -0500
WIP - fix doc_open_revs always read repair
---
src/fabric/src/fabric_doc_open_revs.erl | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/src/fabric/src/fabric_doc_open_revs.erl b/src/fabric/src/fabric_doc_open_revs.erl
index dbe02bf..61e8afd 100644
--- a/src/fabric/src/fabric_doc_open_revs.erl
+++ b/src/fabric/src/fabric_doc_open_revs.erl
@@ -98,14 +98,13 @@ handle_message({ok, RawReplies}, Worker, State) ->
QuorumReplies = RealReplyCount >= R,
{NewReplies, QuorumMet, Repair} = case IsTree of
true ->
- {NewReplies0, AllInternal, Repair00} =
+ {NewReplies0, AllInternal, Repair0} =
tree_replies(PrevReplies, tree_sort(RawReplies)),
- % don't set Repair=true on the first reply
- Repair0 = (ReplyCount > 0) and Repair00,
NumLeafs = couch_key_tree:count_leafs(PrevReplies),
SameNumRevs = length(RawReplies) == NumLeafs,
QMet = AllInternal andalso SameNumRevs andalso QuorumReplies,
- {NewReplies0, QMet, Repair0};
+ % Don't set repair=true on the first reply
+ {NewReplies0, QMet, (ReplyCount > 0) and Repair0};
false ->
{NewReplies0, MinCount} = dict_replies(PrevReplies, RawReplies),
{NewReplies0, MinCount >= R, false}
--
To stop receiving notification emails like this one, please contact
davisp@apache.org.