You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by mi...@apache.org on 2015/06/03 16:13:12 UTC
[1/3] fabric commit: updated refs/heads/merge-diff-from-cloudant-fork
to ece2b2e
Repository: couchdb-fabric
Updated Branches:
refs/heads/merge-diff-from-cloudant-fork [created] ece2b2ed9
Provide an access to a document info
The functions `get_doc_info/3` and `get_full_doc_info/3` were
added to API to provide an access to the records `#doc_info{}`
and `#full_doc_info{}` accordingly.
The functions are re-using `fabric_open_doc` coordinator
and consequently are the subject of the same read quorum rules
as `open_doc/3` function. However the info functions do not trigger
read repair on a not fully complete quorum.
Function `get_full_doc_info/3` accepts an option `deleted` to
allow to provide an information for a deleted document,
similar to `open_doc/3`.
FogBugz: 12933
This is a cherry-pick of:
https://github.com/cloudant/fabric/commit/c85569287ad8f86122b47775adc2ab9218db0322
Conflicts:
src/fabric_rpc.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/d7d6be85
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/d7d6be85
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/d7d6be85
Branch: refs/heads/merge-diff-from-cloudant-fork
Commit: d7d6be8535443fd3ddbe65c0a6a35534b4b98e29
Parents: d44f7fb
Author: Eric Avdey <ei...@eiri.ca>
Authored: Mon Nov 10 20:12:42 2014 -0400
Committer: Mike Wallace <mi...@apache.org>
Committed: Wed Jun 3 14:54:21 2015 +0100
----------------------------------------------------------------------
src/fabric.erl | 34 ++++++++++++++++++++--
src/fabric_doc_open.erl | 68 +++++++++++++++++++++++++++++++++++++++++---
src/fabric_rpc.erl | 10 +++++--
3 files changed, 103 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/d7d6be85/src/fabric.erl
----------------------------------------------------------------------
diff --git a/src/fabric.erl b/src/fabric.erl
index 958e63c..25205f8 100644
--- a/src/fabric.erl
+++ b/src/fabric.erl
@@ -23,8 +23,9 @@
get_security/2, get_all_security/1, get_all_security/2]).
% Documents
--export([open_doc/3, open_revs/4, get_missing_revs/2, get_missing_revs/3,
- update_doc/3, update_docs/3, purge_docs/2, att_receiver/2]).
+-export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
+ get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3,
+ purge_docs/2, att_receiver/2]).
% Views
-export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
@@ -161,7 +162,12 @@ get_all_security(DbName, Options) ->
{error, any()} |
{error, any() | any()}.
open_doc(DbName, Id, Options) ->
- fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options)).
+ case proplists:get_value(doc_info, Options) of
+ undefined ->
+ fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options));
+ Else ->
+ {error, {invalid_option, {doc_info, Else}}}
+ end.
%% @doc retrieve a collection of revisions, possible all
-spec open_revs(dbname(), docid(), [revision()] | all, [option()]) ->
@@ -172,6 +178,28 @@ open_doc(DbName, Id, Options) ->
open_revs(DbName, Id, Revs, Options) ->
fabric_doc_open_revs:go(dbname(DbName), docid(Id), Revs, opts(Options)).
+%% @doc Retrieves an information on a document with a given id
+-spec get_doc_info(dbname(), docid(), [options()]) ->
+ {ok, #doc_info{}} |
+ {not_found, missing} |
+ {timeout, any()} |
+ {error, any()} |
+ {error, any() | any()}.
+get_doc_info(DbName, Id, Options) ->
+ Options1 = [doc_info|Options],
+ fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options1)).
+
+%% @doc Retrieves a full information on a document with a given id
+-spec get_full_doc_info(dbname(), docid(), [options()]) ->
+ {ok, #full_doc_info{}} |
+ {not_found, missing | deleted} |
+ {timeout, any()} |
+ {error, any()} |
+ {error, any() | any()}.
+get_full_doc_info(DbName, Id, Options) ->
+ Options1 = [{doc_info, full}|Options],
+ fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options1)).
+
%% @equiv get_missing_revs(DbName, IdsRevs, [])
get_missing_revs(DbName, IdsRevs) ->
get_missing_revs(DbName, IdsRevs, []).
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/d7d6be85/src/fabric_doc_open.erl
----------------------------------------------------------------------
diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl
index c7d90a4..1607946 100644
--- a/src/fabric_doc_open.erl
+++ b/src/fabric_doc_open.erl
@@ -30,7 +30,12 @@
go(DbName, Id, Options) ->
- Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_doc,
+ Handler = case proplists:get_value(doc_info, Options) of
+ true -> get_doc_info;
+ full -> get_full_doc_info;
+ undefined -> open_doc
+ end,
+ Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), Handler,
[Id, [deleted|Options]]),
SuppressDeletedDoc = not lists:member(deleted, Options),
N = mem3:n(DbName),
@@ -44,11 +49,15 @@ go(DbName, Id, Options) ->
},
RexiMon = fabric_util:create_monitors(Workers),
try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of
- {ok, #acc{}=Acc} ->
+ {ok, #acc{}=Acc} when Handler =:= open_doc ->
Reply = handle_response(Acc),
format_reply(Reply, SuppressDeletedDoc);
+ {ok, #acc{state = r_not_met}} ->
+ {error, quorum_not_met};
+ {ok, #acc{q_reply = QuorumReply}} ->
+ format_reply(QuorumReply, SuppressDeletedDoc);
{timeout, #acc{workers=DefunctWorkers}} ->
- fabric_util:log_timeout(DefunctWorkers, "open_doc"),
+ fabric_util:log_timeout(DefunctWorkers, atom_to_list(Handler)),
{error, timeout};
Error ->
Error
@@ -156,12 +165,15 @@ choose_reply(Docs) ->
end, Docs),
{ok, Winner}.
+format_reply({ok, #full_doc_info{deleted=true}}, true) ->
+ {not_found, deleted};
format_reply({ok, #doc{deleted=true}}, true) ->
{not_found, deleted};
+format_reply(not_found, _) ->
+ {not_found, missing};
format_reply(Else, _) ->
Else.
-
is_r_met_test() ->
Workers0 = [],
Workers1 = [nil],
@@ -472,6 +484,54 @@ handle_response_quorum_met_test() ->
stop_meck_(),
ok.
+get_doc_info_test() ->
+ start_meck_(),
+ meck:new([mem3, rexi_monitor, fabric_util]),
+ meck:expect(twig, log, fun(_, _, _) -> ok end),
+ meck:expect(fabric, update_docs, fun(_, _, _) -> {ok, []} end),
+ meck:expect(couch_stats, increment_counter, fun(_) -> ok end),
+ meck:expect(fabric_util, submit_jobs, fun(_, _, _) -> ok end),
+ meck:expect(fabric_util, create_monitors, fun(_) -> ok end),
+ meck:expect(rexi_monitor, stop, fun(_) -> ok end),
+ meck:expect(mem3, shards, fun(_, _) -> ok end),
+ meck:expect(mem3, n, fun(_) -> 3 end),
+ meck:expect(mem3, quorum, fun(_) -> 2 end),
+
+ meck:expect(fabric_util, recv, fun(_, _, _, _) ->
+ {ok, #acc{state = r_not_met}}
+ end),
+ Rsp1 = fabric_doc_open:go("test", "one", [doc_info]),
+ ?assertEqual({error, quorum_not_met}, Rsp1),
+
+ Rsp2 = fabric_doc_open:go("test", "one", [{doc_info, full}]),
+ ?assertEqual({error, quorum_not_met}, Rsp2),
+
+ meck:expect(fabric_util, recv, fun(_, _, _, _) ->
+ {ok, #acc{state = r_met, q_reply = not_found}}
+ end),
+ MissingRsp1 = fabric_doc_open:go("test", "one", [doc_info]),
+ ?assertEqual({not_found, missing}, MissingRsp1),
+ MissingRsp2 = fabric_doc_open:go("test", "one", [{doc_info, full}]),
+ ?assertEqual({not_found, missing}, MissingRsp2),
+
+ meck:expect(fabric_util, recv, fun(_, _, _, _) ->
+ A = #doc_info{},
+ {ok, #acc{state = r_met, q_reply = {ok, A}}}
+ end),
+ {ok, Rec1} = fabric_doc_open:go("test", "one", [doc_info]),
+ ?assert(is_record(Rec1, doc_info)),
+
+ meck:expect(fabric_util, recv, fun(_, _, _, _) ->
+ A = #full_doc_info{deleted = true},
+ {ok, #acc{state = r_met, q_reply = {ok, A}}}
+ end),
+ Rsp3 = fabric_doc_open:go("test", "one", [{doc_info, full}]),
+ ?assertEqual({not_found, deleted}, Rsp3),
+ {ok, Rec2} = fabric_doc_open:go("test", "one", [{doc_info, full},deleted]),
+ ?assert(is_record(Rec2, full_doc_info)),
+
+ meck:unload([mem3, rexi_monitor, fabric_util]),
+ stop_meck_().
start_meck_() ->
meck:new([couch_log, rexi, fabric, couch_stats]).
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/d7d6be85/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index eeaebd1..057dec3 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -13,8 +13,8 @@
-module(fabric_rpc).
-export([get_db_info/1, get_doc_count/1, get_update_seq/1]).
--export([open_doc/3, open_revs/4, get_missing_revs/2, get_missing_revs/3,
- update_docs/3]).
+-export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
+ get_missing_revs/2, get_missing_revs/3, update_docs/3]).
-export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]).
-export([create_db/1, delete_db/1, reset_validation_funs/1, set_security/3,
set_revs_limit/3, create_shard_db_doc/2, delete_shard_db_doc/2]).
@@ -174,6 +174,12 @@ open_doc(DbName, DocId, Options) ->
open_revs(DbName, Id, Revs, Options) ->
with_db(DbName, Options, {couch_db, open_doc_revs, [Id, Revs, Options]}).
+get_full_doc_info(DbName, DocId, Options) ->
+ with_db(DbName, Options, {couch_db, get_full_doc_info, [DocId]}).
+
+get_doc_info(DbName, DocId, Options) ->
+ with_db(DbName, Options, {couch_db, get_doc_info, [DocId]}).
+
get_missing_revs(DbName, IdRevsList) ->
get_missing_revs(DbName, IdRevsList, []).
[2/3] fabric commit: updated refs/heads/merge-diff-from-cloudant-fork
to ece2b2e
Posted by mi...@apache.org.
Add ability to gracefully terminate existing changes feeds
Calling fabric:end_changes() will cause all current changes
feeds to gracefully exit without preventing new changes requests from
starting (unlike maintenance_mode).
BugzID: 45762
This is a cherry-pick of:
https://github.com/cloudant/fabric/commit/1b45cede8f11e209f28e3d06b9fda4cbdcd719cc
Conflicts:
src/fabric.erl
src/fabric_view_changes.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/383f8f4d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/383f8f4d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/383f8f4d
Branch: refs/heads/merge-diff-from-cloudant-fork
Commit: 383f8f4dc46737fac646a6d90ff03b64c0d94109
Parents: d7d6be8
Author: Robert Newson <rn...@apache.org>
Authored: Sun Mar 29 13:38:38 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Wed Jun 3 15:00:56 2015 +0100
----------------------------------------------------------------------
src/fabric.erl | 6 +++++-
src/fabric_view_changes.erl | 17 ++++++++++++++++-
2 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/383f8f4d/src/fabric.erl
----------------------------------------------------------------------
diff --git a/src/fabric.erl b/src/fabric.erl
index 25205f8..26b9f62 100644
--- a/src/fabric.erl
+++ b/src/fabric.erl
@@ -29,7 +29,7 @@
% Views
-export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
- query_view/6, get_view_group_info/2]).
+ query_view/6, get_view_group_info/2, end_changes/0]).
% miscellany
-export([design_docs/1, reset_validation_funs/1, cleanup_index_files/0,
@@ -362,6 +362,10 @@ query_view(DbName, DDoc, ViewName, Callback, Acc0, QueryArgs0) ->
get_view_group_info(DbName, DesignId) ->
fabric_group_info:go(dbname(DbName), design_doc(DesignId)).
+-spec end_changes() -> ok.
+end_changes() ->
+ fabric_view_changes:increment_changes_epoch().
+
%% @doc retrieve all the design docs from a database
-spec design_docs(dbname()) -> {ok, [json_obj()]}.
design_docs(DbName) ->
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/383f8f4d/src/fabric_view_changes.erl
----------------------------------------------------------------------
diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl
index 92d08e7..7e6666c 100644
--- a/src/fabric_view_changes.erl
+++ b/src/fabric_view_changes.erl
@@ -13,6 +13,7 @@
-module(fabric_view_changes).
-export([go/5, pack_seqs/1, unpack_seqs/2]).
+-export([increment_changes_epoch/0]).
%% exported for upgrade purposes.
-export([keep_sending_changes/8]).
@@ -37,6 +38,7 @@ go(DbName, Feed, Options, Callback, Acc0) when Feed == "continuous" orelse
UpdateListener = {spawn_link(fabric_db_update_listener, go,
[Parent, Ref, DbName, Timeout]),
Ref},
+ put(changes_epoch, get_changes_epoch()),
try
keep_sending_changes(
DbName,
@@ -86,8 +88,9 @@ keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, UpListen, T0)
} = Collector,
LastSeq = pack_seqs(NewSeqs),
MaintenanceMode = config:get("couchdb", "maintenance_mode"),
+ NewEpoch = get_changes_epoch() > erlang:get(changes_epoch),
if Limit > Limit2, Feed == "longpoll";
- MaintenanceMode == "true"; MaintenanceMode == "nolb" ->
+ MaintenanceMode == "true"; MaintenanceMode == "nolb"; NewEpoch ->
Callback({stop, LastSeq, pending_count(Offset)}, AccOut);
true ->
WaitForUpdate = wait_db_updated(UpListen),
@@ -459,6 +462,18 @@ validate_start_seq(DbName, Seq) ->
{error, {bad_request, Reason}}
end.
+get_changes_epoch() ->
+ case application:get_env(fabric, changes_epoch) of
+ undefined ->
+ increment_changes_epoch(),
+ get_changes_epoch();
+ {ok, Epoch} ->
+ Epoch
+ end.
+
+increment_changes_epoch() ->
+ application:set_env(fabric, changes_epoch, os:timestamp()).
+
unpack_seqs_test() ->
meck:new(mem3),
meck:new(fabric_view),
[3/3] fabric commit: updated refs/heads/merge-diff-from-cloudant-fork
to ece2b2e
Posted by mi...@apache.org.
export dbname/1
BugzID: 45853
This is a cherry-pick of:
https://github.com/cloudant/fabric/commit/818dd09e9ea645e866fc62fd1a4302ce73b2422a
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/ece2b2ed
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/ece2b2ed
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/ece2b2ed
Branch: refs/heads/merge-diff-from-cloudant-fork
Commit: ece2b2ed9114d454eea79d8788f8d0c01f172ed9
Parents: 383f8f4
Author: Robert Newson <rn...@apache.org>
Authored: Thu Apr 9 23:07:19 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Wed Jun 3 15:02:32 2015 +0100
----------------------------------------------------------------------
src/fabric.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/ece2b2ed/src/fabric.erl
----------------------------------------------------------------------
diff --git a/src/fabric.erl b/src/fabric.erl
index 26b9f62..c3a95fb 100644
--- a/src/fabric.erl
+++ b/src/fabric.erl
@@ -33,7 +33,7 @@
% miscellany
-export([design_docs/1, reset_validation_funs/1, cleanup_index_files/0,
- cleanup_index_files/1]).
+ cleanup_index_files/1, dbname/1]).
-include_lib("fabric/include/fabric.hrl").