You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/04/04 21:19:44 UTC
[01/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0 [Forced Update!]
Repository: couchdb-fabric
Updated Branches:
refs/heads/COUCHDB-3287-pluggable-storage-engines 581bd9d16 -> 6803aa03a (forced update)
Return error when workers crash
Currently, when one worker survives in fabric_open_revs, we return that
as the response. However, when all workers crash, we still return
{ok, []}. This changes the response to an error.
COUCHDB-3113
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/dd02a393
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/dd02a393
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/dd02a393
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: dd02a3938f267716e3479b7162e0b0a4f8ba3d51
Parents: f9f5681
Author: Tony Sun <to...@cloudant.com>
Authored: Wed Jan 25 13:47:00 2017 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Wed Jan 25 14:12:47 2017 -0800
----------------------------------------------------------------------
src/fabric_doc_open_revs.erl | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/dd02a393/src/fabric_doc_open_revs.erl
----------------------------------------------------------------------
diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl
index 5393f92..6c8912b 100644
--- a/src/fabric_doc_open_revs.erl
+++ b/src/fabric_doc_open_revs.erl
@@ -47,6 +47,8 @@ go(DbName, Id, Revs, Options) ->
},
RexiMon = fabric_util:create_monitors(Workers),
try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, State) of
+ {ok, []} ->
+ {error, all_workers_died};
{ok, Replies} ->
{ok, Replies};
{timeout, #state{workers=DefunctWorkers}} ->
@@ -314,6 +316,7 @@ open_doc_revs_test_() ->
check_not_found_counts_for_descendant(),
check_worker_error_skipped(),
check_quorum_only_counts_valid_responses(),
+ check_empty_list_when_no_workers_reply(),
check_not_found_replies_are_removed_when_doc_found(),
check_not_found_returned_when_one_of_docs_not_found(),
check_not_found_returned_when_doc_not_found()
@@ -491,6 +494,20 @@ check_quorum_only_counts_valid_responses() ->
end).
+check_empty_list_when_no_workers_reply() ->
+ ?_test(begin
+ S0 = state0(revs(), true),
+ Msg1 = {rexi_EXIT, reason},
+ Msg2 = {rexi_EXIT, reason},
+ Msg3 = {rexi_DOWN, nodedown, {nil, node()}, nil},
+ Expect = {stop, []},
+
+ {ok, S1} = handle_message(Msg1, w1, S0),
+ {ok, S2} = handle_message(Msg2, w2, S1),
+ ?assertEqual(Expect, handle_message(Msg3, w3, S2))
+ end).
+
+
check_not_found_replies_are_removed_when_doc_found() ->
?_test(begin
Replies = replies_to_dict([foo1(), bar1(), fooNF()]),
[03/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Use RealReplyCount to distinguish worker replies and invalid docs
We use {ok, []} in couch_db:open_doc_revs_int/3 as a return value
when the document does not exist and open_revs=all. This leads to an
incorrect all_workers_died error. We use ReplyCount and RealReplyCount
to distinguish between when no workers were actually used in a reply
versus when the document does not exist
COUCHDB-3113
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/70535eeb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/70535eeb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/70535eeb
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 70535eeb9b9c226129bdc96cbca8492fbc867cf6
Parents: cf220b2
Author: Tony Sun <to...@cloudant.com>
Authored: Fri Feb 10 11:23:07 2017 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Fri Feb 10 11:23:07 2017 -0800
----------------------------------------------------------------------
src/fabric_doc_open_revs.erl | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/70535eeb/src/fabric_doc_open_revs.erl
----------------------------------------------------------------------
diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl
index 6c8912b..096722f 100644
--- a/src/fabric_doc_open_revs.erl
+++ b/src/fabric_doc_open_revs.erl
@@ -47,7 +47,7 @@ go(DbName, Id, Revs, Options) ->
},
RexiMon = fabric_util:create_monitors(Workers),
try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, State) of
- {ok, []} ->
+ {ok, all_workers_died} ->
{error, all_workers_died};
{ok, Replies} ->
{ok, Replies};
@@ -92,8 +92,9 @@ handle_message({ok, RawReplies}, Worker, State) ->
IsTree = Revs == all orelse Latest,
% Do not count error replies when checking quorum
- QuorumReplies = ReplyCount + 1 - ReplyErrorCount >= R,
+ RealReplyCount = ReplyCount + 1 - ReplyErrorCount,
+ QuorumReplies = RealReplyCount >= R,
{NewReplies, QuorumMet, Repair} = case IsTree of
true ->
{NewReplies0, AllInternal, Repair0} =
@@ -119,7 +120,7 @@ handle_message({ok, RawReplies}, Worker, State) ->
ReplyCount + 1,
InRepair orelse Repair
),
- {stop, format_reply(IsTree, NewReplies)};
+ {stop, format_reply(IsTree, NewReplies, RealReplyCount)};
false ->
{ok, State#state{
replies = NewReplies,
@@ -219,10 +220,13 @@ read_repair(Db, Docs) ->
end.
-format_reply(true, Replies) ->
+format_reply(_, _, RealReplyCount) when RealReplyCount =< 0 ->
+ all_workers_died;
+
+format_reply(true, Replies, _) ->
tree_format_replies(Replies);
-format_reply(false, Replies) ->
+format_reply(false, Replies, _) ->
Filtered = filter_reply(Replies),
dict_format_replies(Filtered).
@@ -500,7 +504,7 @@ check_empty_list_when_no_workers_reply() ->
Msg1 = {rexi_EXIT, reason},
Msg2 = {rexi_EXIT, reason},
Msg3 = {rexi_DOWN, nodedown, {nil, node()}, nil},
- Expect = {stop, []},
+ Expect = {stop, all_workers_died},
{ok, S1} = handle_message(Msg1, w1, S0),
{ok, S2} = handle_message(Msg2, w2, S1),
[10/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Update to use new pluggable storage API
COUCHDB-3287
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/dc266ff5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/dc266ff5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/dc266ff5
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: dc266ff51b222489dd048cb96d30c90d3afebf85
Parents: c1f1501
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 10 16:38:41 2016 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:19:30 2017 -0500
----------------------------------------------------------------------
src/fabric_rpc.erl | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/dc266ff5/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index cd3bac3..556d289 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -70,7 +70,7 @@ changes(DbName, Options, StartVector, DbOptions) ->
{ok, Db} ->
StartSeq = calculate_start_seq(Db, node(), StartVector),
Enum = fun changes_enumerator/2,
- Opts = [{dir,Dir}],
+ Opts = [doc_info, {dir,Dir}],
Acc0 = #cacc{
db = Db,
seq = StartSeq,
@@ -81,7 +81,7 @@ changes(DbName, Options, StartVector, DbOptions) ->
},
try
{ok, #cacc{seq=LastSeq, pending=Pending, epochs=Epochs}} =
- couch_db:changes_since(Db, StartSeq, Enum, Opts, Acc0),
+ couch_db:fold_changes(Db, StartSeq, Enum, Acc0, Opts),
rexi:stream_last({complete, [
{seq, {LastSeq, uuid(Db), couch_db:owner_of(Epochs, LastSeq)}},
{pending, Pending}
@@ -220,7 +220,7 @@ get_missing_revs(DbName, IdRevsList, Options) ->
Ids = [Id1 || {Id1, _Revs} <- IdRevsList],
{ok, lists:zipwith(fun({Id, Revs}, FullDocInfoResult) ->
case FullDocInfoResult of
- {ok, #full_doc_info{rev_tree=RevisionTree} = FullInfo} ->
+ #full_doc_info{rev_tree=RevisionTree} = FullInfo ->
MissingRevs = couch_key_tree:find_missing(RevisionTree, Revs),
{Id, MissingRevs, possible_ancestors(FullInfo, MissingRevs)};
not_found ->
@@ -251,8 +251,7 @@ group_info(DbName, DDocId, DbOptions) ->
reset_validation_funs(DbName) ->
case get_or_create_db(DbName, []) of
{ok, Db} ->
- DbPid = couch_db:get_pid(Db),
- gen_server:cast(DbPid, {load_validation_funs, undefined});
+ couch_db:reload_validation_funs(Db);
_ ->
ok
end.
@@ -336,6 +335,8 @@ reduce_cb(complete, Acc) ->
{ok, Acc}.
+changes_enumerator(#full_doc_info{} = FDI, Acc) ->
+ changes_enumerator(couch_doc:to_doc_info(FDI), Acc);
changes_enumerator(#doc_info{id= <<"_local/", _/binary>>, high_seq=Seq}, Acc) ->
{ok, Acc#cacc{seq = Seq, pending = Acc#cacc.pending-1}};
changes_enumerator(DocInfo, Acc) ->
[07/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Merge branch 'couchdb-2992'
Closes #91
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/ce62148d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/ce62148d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/ce62148d
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: ce62148d0a4469751d8078cc223684da29b5d4a7
Parents: d22903c 3b15107
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Mar 15 22:39:31 2017 -0400
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Mar 15 22:39:31 2017 -0400
----------------------------------------------------------------------
src/fabric.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[05/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Merge remote-tracking branch 'cloudant/couchdb-3302'
Closes #89
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/d22903c0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/d22903c0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/d22903c0
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: d22903c0ceaf19a305e67b5b7322cbdc4e5e9340
Parents: 70535ee 6e9074b
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Feb 21 21:22:54 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Feb 21 21:22:54 2017 -0500
----------------------------------------------------------------------
src/fabric_rpc.erl | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
[09/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Remove public db record
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/c1f15015
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/c1f15015
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/c1f15015
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: c1f15015f6a9ee984f70ea8853fd095b702261ec
Parents: aa8cbe3
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 1 16:36:38 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:18:36 2017 -0500
----------------------------------------------------------------------
include/couch_db_tmp.hrl | 296 ------------------------------------------
src/fabric.erl | 12 +-
src/fabric_rpc.erl | 113 +++-------------
src/fabric_util.erl | 3 +-
4 files changed, 24 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/c1f15015/include/couch_db_tmp.hrl
----------------------------------------------------------------------
diff --git a/include/couch_db_tmp.hrl b/include/couch_db_tmp.hrl
deleted file mode 100644
index cd3a047..0000000
--- a/include/couch_db_tmp.hrl
+++ /dev/null
@@ -1,296 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--define(LOCAL_DOC_PREFIX, "_local/").
--define(DESIGN_DOC_PREFIX0, "_design").
--define(DESIGN_DOC_PREFIX, "_design/").
-
--define(MIN_STR, <<"">>).
--define(MAX_STR, <<255>>). % illegal utf string
-
--define(JSON_ENCODE(V), couch_util:json_encode(V)).
--define(JSON_DECODE(V), couch_util:json_decode(V)).
-
--define(b2l(V), binary_to_list(V)).
--define(l2b(V), list_to_binary(V)).
-
--define(DEFAULT_ATTACHMENT_CONTENT_TYPE, <<"application/octet-stream">>).
-
--define(LOG_DEBUG(Format, Args), couch_log:debug(Format, Args)).
--define(LOG_INFO(Format, Args), couch_log:notice(Format, Args)).
--define(LOG_ERROR(Format, Args), couch_log:error(Format, Args)).
-
--record(rev_info,
- {
- rev,
- seq = 0,
- deleted = false,
- body_sp = nil % stream pointer
- }).
-
--record(doc_info,
- {
- id = <<"">>,
- high_seq = 0,
- revs = [] % rev_info
- }).
-
--record(full_doc_info,
- {id = <<"">>,
- update_seq = 0,
- deleted = false,
- data_size = 0,
- rev_tree = []
- }).
-
--record(httpd,
- {mochi_req,
- peer,
- method,
- path_parts,
- db_url_handlers,
- user_ctx,
- req_body = undefined,
- design_url_handlers,
- auth,
- default_fun,
- url_handlers
- }).
-
-
--record(doc,
- {
- id = <<"">>,
- revs = {0, []},
-
- % the json body object.
- body = {[]},
-
- atts = [], % attachments
-
- deleted = false,
-
- % key/value tuple of meta information, provided when using special options:
- % couch_db:open_doc(Db, Id, Options).
- meta = []
- }).
-
-
--record(att,
- {
- name,
- type,
- att_len,
- disk_len, % length of the attachment in its identity form
- % (that is, without a content encoding applied to it)
- % differs from att_len when encoding /= identity
- md5= <<>>,
- revpos=0,
- data,
- encoding=identity % currently supported values are:
- % identity, gzip
- % additional values to support in the future:
- % deflate, compress
- }).
-
-
--record(user_ctx,
- {
- name=null,
- roles=[],
- handler
- }).
-
-% This should be updated anytime a header change happens that requires more
-% than filling in new defaults.
-%
-% As long the changes are limited to new header fields (with inline
-% defaults) added to the end of the record, then there is no need to increment
-% the disk revision number.
-%
-% if the disk revision is incremented, then new upgrade logic will need to be
-% added to couch_db_updater:init_db.
-
--define(LATEST_DISK_VERSION, 5).
-
--record(db_header,
- {disk_version = ?LATEST_DISK_VERSION,
- update_seq = 0,
- unused = 0,
- id_tree_state = nil,
- seq_tree_state = nil,
- local_tree_state = nil,
- purge_seq = 0,
- purged_docs = nil,
- security_ptr = nil,
- revs_limit = 1000
- }).
-
--record(db,
- {main_pid = nil,
- update_pid = nil,
- compactor_pid = nil,
- instance_start_time, % number of microsecs since jan 1 1970 as a binary string
- fd,
- fd_monitor,
- header = #db_header{},
- committed_update_seq,
- id_tree,
- seq_tree,
- local_tree,
- update_seq,
- name,
- filepath,
- validate_doc_funs = undefined,
- security = [],
- security_ptr = nil,
- user_ctx = #user_ctx{},
- waiting_delayed_commit = nil,
- revs_limit = 1000,
- fsync_options = [],
- is_sys_db = false
- }).
-
-
--record(view_query_args, {
- start_key,
- end_key,
- start_docid = ?MIN_STR,
- end_docid = ?MAX_STR,
-
- direction = fwd,
- inclusive_end=true, % aka a closed-interval
-
- limit = 10000000000, % Huge number to simplify logic
- skip = 0,
-
- group_level = 0,
-
- view_type = nil,
- include_docs = false,
- stale = false,
- multi_get = false,
- callback = nil,
- list = nil,
- keys = nil,
- sorted = true,
- extra = []
-}).
-
--record(view_fold_helper_funs, {
- reduce_count,
- passed_end,
- start_response,
- send_row
-}).
-
--record(reduce_fold_helper_funs, {
- start_response,
- send_row
-}).
-
--record(extern_resp_args, {
- code = 200,
- stop = false,
- data = <<>>,
- ctype = "application/json",
- headers = [],
- json = nil
-}).
-
--record(group, {
- sig=nil,
- dbname,
- fd=nil,
- name,
- def_lang,
- design_options=[],
- views,
- id_btree=nil,
- current_seq=0,
- purge_seq=0,
- query_server=nil,
- waiting_delayed_commit=nil,
- atts=[]
- }).
-
--record(view,
- {id_num,
- map_names=[],
- def,
- btree=nil,
- reduce_funs=[],
- dbcopies=[],
- options=[]
- }).
-
--record(index_header,
- {seq=0,
- purge_seq=0,
- id_btree_state=nil,
- view_states=nil
- }).
-
--record(http_db, {
- url,
- auth = [],
- resource = "",
- headers = [
- {"User-Agent", "CouchDB/"++couch:version()},
- {"Accept", "application/json"},
- {"Accept-Encoding", "gzip"}
- ],
- qs = [],
- method = get,
- body = nil,
- options = [
- {response_format,binary},
- {inactivity_timeout, 30000}
- ],
- retries = 10,
- pause = 500,
- conn = nil
-}).
-
-% small value used in revision trees to indicate the revision isn't stored
--define(REV_MISSING, []).
-
--record(changes_args, {
- feed = "normal",
- dir = fwd,
- since = "0",
- limit = 1000000000000000,
- style = main_only,
- heartbeat,
- timeout,
- filter,
- include_docs = false
-}).
-
--record(proc, {
- pid,
- lang,
- client = nil,
- ddoc_keys = [],
- prompt_fun,
- set_timeout_fun,
- stop_fun,
- data_fun
-}).
-
--record(leaf, {
- deleted,
- ptr,
- seq,
- size = 0,
- atts = []
-}).
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/c1f15015/src/fabric.erl
----------------------------------------------------------------------
diff --git a/src/fabric.erl b/src/fabric.erl
index f98a5c0..2b87e46 100644
--- a/src/fabric.erl
+++ b/src/fabric.erl
@@ -38,7 +38,7 @@
-include_lib("fabric/include/fabric.hrl").
--type dbname() :: (iodata() | #db{}).
+-type dbname() :: (iodata() | tuple()).
-type docid() :: iodata().
-type revision() :: {integer(), binary()}.
-type callback() :: fun((any(), any()) -> {ok | stop, any()}).
@@ -476,10 +476,12 @@ dbname(DbName) when is_list(DbName) ->
list_to_binary(DbName);
dbname(DbName) when is_binary(DbName) ->
DbName;
-dbname(#db{name=Name}) ->
- Name;
-dbname(DbName) ->
- erlang:error({illegal_database_name, DbName}).
+dbname(Db) ->
+ try
+ couch_db:name(Db)
+ catch error:badarg ->
+ erlang:error({illegal_database_name, Db})
+ end.
name(Thing) ->
couch_util:to_binary(Thing).
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/c1f15015/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index a3f9bde..cd3bac3 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -38,7 +38,8 @@
}).
%% rpc endpoints
-%% call to with_db will supply your M:F with a #db{} and then remaining args
+%% call to with_db will supply your M:F with a Db instance
+%% and then remaining args
%% @equiv changes(DbName, Args, StartSeq, [])
changes(DbName, Args, StartSeq) ->
@@ -76,13 +77,13 @@ changes(DbName, Options, StartVector, DbOptions) ->
args = Args,
options = Options,
pending = couch_db:count_changes_since(Db, StartSeq),
- epochs = get_epochs(Db)
+ epochs = couch_db:get_epochs(Db)
},
try
{ok, #cacc{seq=LastSeq, pending=Pending, epochs=Epochs}} =
couch_db:changes_since(Db, StartSeq, Enum, Opts, Acc0),
rexi:stream_last({complete, [
- {seq, {LastSeq, uuid(Db), owner_of(LastSeq, Epochs)}},
+ {seq, {LastSeq, uuid(Db), couch_db:owner_of(Epochs, LastSeq)}},
{pending, Pending}
]})
after
@@ -225,7 +226,7 @@ get_missing_revs(DbName, IdRevsList, Options) ->
not_found ->
{Id, Revs, []}
end
- end, IdRevsList, couch_btree:lookup(Db#db.id_tree, Ids))};
+ end, IdRevsList, couch_db:get_full_doc_infos(Db, Ids))};
Error ->
Error
end).
@@ -249,8 +250,9 @@ group_info(DbName, DDocId, DbOptions) ->
reset_validation_funs(DbName) ->
case get_or_create_db(DbName, []) of
- {ok, #db{main_pid = Pid}} ->
- gen_server:cast(Pid, {load_validation_funs, undefined});
+ {ok, Db} ->
+ DbPid = couch_db:get_pid(Db),
+ gen_server:cast(DbPid, {load_validation_funs, undefined});
_ ->
ok
end.
@@ -356,7 +358,7 @@ changes_enumerator(DocInfo, Acc) ->
Opts = if Conflicts -> [conflicts | DocOptions]; true -> DocOptions end,
ChangesRow = {change, [
{pending, Pending-1},
- {seq, {Seq, uuid(Db), owner_of(Seq, Epochs)}},
+ {seq, {Seq, uuid(Db), couch_db:owner_of(Epochs, Seq)}},
{id, Id},
{changes, Results},
{deleted, Del} |
@@ -454,78 +456,17 @@ set_io_priority(DbName, Options) ->
ok
end.
-calculate_start_seq(_Db, _Node, Seq) when is_integer(Seq) ->
- Seq;
-calculate_start_seq(Db, Node, {Seq, Uuid}) ->
- % Treat the current node as the epoch node
- calculate_start_seq(Db, Node, {Seq, Uuid, Node});
-calculate_start_seq(Db, _Node, {Seq, Uuid, EpochNode}) ->
- case is_prefix(Uuid, couch_db:get_uuid(Db)) of
- true ->
- case is_owner(EpochNode, Seq, couch_db:get_epochs(Db)) of
- true -> Seq;
- false -> 0
- end;
- false ->
- %% The file was rebuilt, most likely in a different
- %% order, so rewind.
- 0
- end;
-calculate_start_seq(Db, _Node, {replace, OriginalNode, Uuid, Seq}) ->
- case is_prefix(Uuid, couch_db:get_uuid(Db)) of
- true ->
- start_seq(get_epochs(Db), OriginalNode, Seq);
- false ->
+calculate_start_seq(Db, Node, Seq) ->
+ case couch_db:calculate_start_seq(Db, Node, Seq) of
+ N when is_integer(N) ->
+ N;
+ {replace, OriginalNode, Uuid, OriginalSeq} ->
%% Scan history looking for an entry with
%% * target_node == TargetNode
%% * target_uuid == TargetUUID
%% * target_seq =< TargetSeq
%% If such an entry is found, stream from associated source_seq
- mem3_rep:find_source_seq(Db, OriginalNode, Uuid, Seq)
- end.
-
-is_prefix(Pattern, Subject) ->
- binary:longest_common_prefix([Pattern, Subject]) == size(Pattern).
-
-is_owner(Node, Seq, Epochs) ->
- validate_epochs(Epochs),
- Node =:= owner_of(Seq, Epochs).
-
-owner_of(_Seq, []) ->
- undefined;
-owner_of(Seq, [{EpochNode, EpochSeq} | _Rest]) when Seq > EpochSeq ->
- EpochNode;
-owner_of(Seq, [_ | Rest]) ->
- owner_of(Seq, Rest).
-
-get_epochs(Db) ->
- Epochs = couch_db:get_epochs(Db),
- validate_epochs(Epochs),
- Epochs.
-
-start_seq([{OrigNode, EpochSeq} | _], OrigNode, Seq) when Seq > EpochSeq ->
- %% OrigNode is the owner of the Seq so we can safely stream from there
- Seq;
-start_seq([{_, NewSeq}, {OrigNode, _} | _], OrigNode, Seq) when Seq > NewSeq ->
- %% We transferred this file before Seq was written on OrigNode, so we need
- %% to stream from the beginning of the next epoch. Note that it is _not_
- %% necessary for the current node to own the epoch beginning at NewSeq
- NewSeq;
-start_seq([_ | Rest], OrigNode, Seq) ->
- start_seq(Rest, OrigNode, Seq);
-start_seq([], OrigNode, Seq) ->
- erlang:error({epoch_mismatch, OrigNode, Seq}).
-
-validate_epochs(Epochs) ->
- %% Assert uniqueness.
- case length(Epochs) == length(lists:ukeysort(2, Epochs)) of
- true -> ok;
- false -> erlang:error(duplicate_epoch)
- end,
- %% Assert order.
- case Epochs == lists:sort(fun({_, A}, {_, B}) -> B =< A end, Epochs) of
- true -> ok;
- false -> erlang:error(epoch_order)
+ mem3_rep:find_source_seq(Db, OriginalNode, Uuid, OriginalSeq)
end.
uuid(Db) ->
@@ -538,30 +479,6 @@ uuid_prefix_len() ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-calculate_start_seq_test() ->
- %% uuid mismatch is always a rewind.
- Hdr1 = couch_db_header:new(),
- Hdr2 = couch_db_header:set(Hdr1, [{epochs, [{node1, 1}]}, {uuid, <<"uuid1">>}]),
- ?assertEqual(0, calculate_start_seq(#db{header=Hdr2}, node1, {1, <<"uuid2">>})),
- %% uuid matches and seq is owned by node.
- Hdr3 = couch_db_header:set(Hdr2, [{epochs, [{node1, 1}]}]),
- ?assertEqual(2, calculate_start_seq(#db{header=Hdr3}, node1, {2, <<"uuid1">>})),
- %% uuids match but seq is not owned by node.
- Hdr4 = couch_db_header:set(Hdr2, [{epochs, [{node2, 2}, {node1, 1}]}]),
- ?assertEqual(0, calculate_start_seq(#db{header=Hdr4}, node1, {3, <<"uuid1">>})),
- %% return integer if we didn't get a vector.
- ?assertEqual(4, calculate_start_seq(#db{}, foo, 4)).
-
-is_owner_test() ->
- ?assertNot(is_owner(foo, 1, [])),
- ?assertNot(is_owner(foo, 1, [{foo, 1}])),
- ?assert(is_owner(foo, 2, [{foo, 1}])),
- ?assert(is_owner(foo, 50, [{bar, 100}, {foo, 1}])),
- ?assert(is_owner(foo, 50, [{baz, 200}, {bar, 100}, {foo, 1}])),
- ?assert(is_owner(bar, 150, [{baz, 200}, {bar, 100}, {foo, 1}])),
- ?assertError(duplicate_epoch, is_owner(foo, 1, [{foo, 1}, {bar, 1}])),
- ?assertError(epoch_order, is_owner(foo, 1, [{foo, 100}, {bar, 200}])).
-
maybe_filtered_json_doc_no_filter_test() ->
Body = {[{<<"a">>, 1}]},
Doc = #doc{id = <<"1">>, revs = {1, [<<"r1">>]}, body = Body},
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/c1f15015/src/fabric_util.erl
----------------------------------------------------------------------
diff --git a/src/fabric_util.erl b/src/fabric_util.erl
index 7e3f23e..c5aef03 100644
--- a/src/fabric_util.erl
+++ b/src/fabric_util.erl
@@ -302,7 +302,8 @@ path_ends_with(Path, Suffix) ->
fake_db(DbName, Opts) ->
{SecProps} = fabric:get_security(DbName), % as admin
UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
- #db{name = DbName, security = SecProps, user_ctx = UserCtx}.
+ {ok, Db} = couch_db:clustered_db(DbName, UserCtx, SecProps),
+ Db.
%% test function
kv(Item, Count) ->
[06/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Allow limiting maximum document body size
Update doc function to check and validate document body sizes
Main implementation is in PR:
https://github.com/apache/couchdb-couch/pull/235
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/3b15107d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/3b15107d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/3b15107d
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 3b15107df83a16a26dbc6c06a1a080437cb558b8
Parents: d22903c
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 14 02:59:31 2017 -0400
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 14 02:59:31 2017 -0400
----------------------------------------------------------------------
src/fabric.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/3b15107d/src/fabric.erl
----------------------------------------------------------------------
diff --git a/src/fabric.erl b/src/fabric.erl
index 05eed7e..f98a5c0 100644
--- a/src/fabric.erl
+++ b/src/fabric.erl
@@ -497,7 +497,7 @@ docs(Docs) ->
doc(#doc{} = Doc) ->
Doc;
doc({_} = Doc) ->
- couch_doc:from_json_obj(Doc);
+ couch_doc:from_json_obj_validate(Doc);
doc(Doc) ->
erlang:error({illegal_doc_format, Doc}).
[04/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Prevent attachment upload from timing out during update_docs fabric call
Currently if an attachment was large enough or the connection was slow enough
such that it took more than fabric.request_timeout = 60000 milliseconds, the
fabric request would time out during attachment data transfer from coordinator
node to other nodes and the whole request would fail.
This was most evident when replicating database with large attachments.
The fix is to periodically send `attachment_chunk_received` to coordinator to
prevent the timeout.
COUCHDB-3302
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/6e9074bc
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/6e9074bc
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/6e9074bc
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 6e9074bc8778e00471d96191319ac67d6c78c05a
Parents: 70535ee
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Feb 21 17:46:57 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Feb 21 17:46:57 2017 -0500
----------------------------------------------------------------------
src/fabric_rpc.erl | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/6e9074bc/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index 679a305..80b110a 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -422,6 +422,7 @@ make_att_reader({follows, Parser, Ref}) ->
Parser ! {get_bytes, Ref, self()},
receive
{bytes, Ref, Bytes} ->
+ rexi:reply(attachment_chunk_received),
Bytes;
{'DOWN', ParserRef, _, _, Reason} ->
throw({mp_parser_died, Reason})
[02/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Add Else Clause For Embed Doc
When open_doc or open_revs return an error, we set the doc value
to be an error message. This way we account for errors rather than
transform_row throwing a function_clause.
COUCHDB-3109
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/cf220b2e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/cf220b2e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/cf220b2e
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: cf220b2e927093e3bd6f409b4ca9f7b1be0a04a3
Parents: dd02a39
Author: Tony Sun <to...@cloudant.com>
Authored: Tue Jan 31 20:47:13 2017 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Thu Feb 9 11:27:47 2017 -0800
----------------------------------------------------------------------
src/fabric_view.erl | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/cf220b2e/src/fabric_view.erl
----------------------------------------------------------------------
diff --git a/src/fabric_view.erl b/src/fabric_view.erl
index 01291a9..45262e4 100644
--- a/src/fabric_view.erl
+++ b/src/fabric_view.erl
@@ -172,7 +172,9 @@ possibly_embed_doc(#collector{db_name=DbName, query_args=Args},
{ok, NewDoc} ->
Row#view_row{doc=couch_doc:to_json_obj(NewDoc,[])};
{not_found, _} ->
- Row#view_row{doc=null}
+ Row#view_row{doc=null};
+ Else ->
+ Row#view_row{doc={error, Else}}
end;
Rev0 ->
Rev = couch_doc:parse_rev(Rev0),
@@ -180,7 +182,9 @@ possibly_embed_doc(#collector{db_name=DbName, query_args=Args},
{ok, [{ok, NewDoc}]} ->
Row#view_row{doc=couch_doc:to_json_obj(NewDoc,[])};
{ok, [{{not_found, _}, Rev}]} ->
- Row#view_row{doc=null}
+ Row#view_row{doc=null};
+ Else ->
+ Row#view_row{doc={error, Else}}
end
end) end),
receive {'DOWN',Ref,process,Pid, Resp} ->
[08/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Revert "Handle no_pass message when limit is 0"
This partially reverts commit 998cf2. Only the path src/fabric_rpc.erl has
been reverted, which is 1 out of 2 files changed in the original commit.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/aa8cbe38
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/aa8cbe38
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/aa8cbe38
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: aa8cbe3836622709c568b8541a02d126ac6af35b
Parents: ce62148
Author: ILYA Khlopotov <ii...@apache.org>
Authored: Thu Jan 26 10:19:27 2017 -0800
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:18:36 2017 -0500
----------------------------------------------------------------------
src/fabric_rpc.erl | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/aa8cbe38/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index 80b110a..a3f9bde 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -351,9 +351,7 @@ changes_enumerator(DocInfo, Acc) ->
#doc_info{id=Id, high_seq=Seq, revs=[#rev_info{deleted=Del}|_]} = DocInfo,
case [X || X <- couch_changes:filter(Db, DocInfo, Filter), X /= null] of
[] ->
- ChangesRow = {no_pass, [
- {pending, Pending-1},
- {seq, Seq}]};
+ ChangesRow = {no_pass, Seq};
Results ->
Opts = if Conflicts -> [conflicts | DocOptions]; true -> DocOptions end,
ChangesRow = {change, [
[11/11] fabric commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 6803aa0
Posted by da...@apache.org.
Pass the storage engine option to RPC workers
COUCHDB-3287
Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/6803aa03
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/6803aa03
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/6803aa03
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 6803aa03a940e68f41037de72b602ca1c1d3c5b0
Parents: dc266ff
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Apr 6 11:06:51 2016 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:19:30 2017 -0500
----------------------------------------------------------------------
src/fabric_db_create.erl | 29 ++++++++++++++++++-----------
src/fabric_rpc.erl | 6 +++---
2 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/6803aa03/src/fabric_db_create.erl
----------------------------------------------------------------------
diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl
index a7f4ed9..3685398 100644
--- a/src/fabric_db_create.erl
+++ b/src/fabric_db_create.erl
@@ -28,7 +28,7 @@ go(DbName, Options) ->
{error, file_exists};
false ->
{Shards, Doc} = generate_shard_map(DbName, Options),
- CreateShardResult = create_shard_files(Shards),
+ CreateShardResult = create_shard_files(Shards, Options),
case CreateShardResult of
enametoolong ->
{error, {database_name_too_long, DbName}};
@@ -64,12 +64,12 @@ generate_shard_map(DbName, Options) ->
% the DB already exists, and may have a different Suffix
ok;
{not_found, _} ->
- Doc = make_document(Shards, Suffix)
+ Doc = make_document(Shards, Suffix, Options)
end,
{Shards, Doc}.
-create_shard_files(Shards) ->
- Workers = fabric_util:submit_jobs(Shards, create_db, []),
+create_shard_files(Shards, Options) ->
+ Workers = fabric_util:submit_jobs(Shards, create_db, [Options]),
RexiMon = fabric_util:create_monitors(Shards),
try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Workers) of
{error, file_exists} ->
@@ -155,7 +155,7 @@ maybe_stop(W, Counters) ->
end
end.
-make_document([#shard{dbname=DbName}|_] = Shards, Suffix) ->
+make_document([#shard{dbname=DbName}|_] = Shards, Suffix, Options) ->
{RawOut, ByNodeOut, ByRangeOut} =
lists:foldl(fun(#shard{node=N, range=[B,E]}, {Raw, ByNode, ByRange}) ->
Range = ?l2b([couch_util:to_hex(<<B:32/integer>>), "-",
@@ -164,12 +164,19 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix) ->
{[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode),
orddict:append(Range, Node, ByRange)}
end, {[], [], []}, Shards),
- #doc{id=DbName, body = {[
- {<<"shard_suffix">>, Suffix},
- {<<"changelog">>, lists:sort(RawOut)},
- {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}},
- {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}}
- ]}}.
+ EngineProp = case couch_util:get_value(engine, Options) of
+ E when is_binary(E) -> [{<<"engine">>, E}];
+ _ -> []
+ end,
+ #doc{
+ id = DbName,
+ body = {[
+ {<<"shard_suffix">>, Suffix},
+ {<<"changelog">>, lists:sort(RawOut)},
+ {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}},
+ {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}}
+ ] ++ EngineProp}
+ }.
db_exists(DbName) -> is_list(catch mem3:shards(DbName)).
http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/6803aa03/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index 556d289..9840c9f 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -16,7 +16,7 @@
-export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
get_missing_revs/2, get_missing_revs/3, update_docs/3]).
-export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]).
--export([create_db/1, delete_db/1, reset_validation_funs/1, set_security/3,
+-export([create_db/2, delete_db/1, reset_validation_funs/1, set_security/3,
set_revs_limit/3, create_shard_db_doc/2, delete_shard_db_doc/2]).
-export([get_all_security/2, open_shard/2]).
-export([compact/1, compact/2]).
@@ -144,8 +144,8 @@ fix_skip_and_limit(Args) ->
#mrargs{skip=Skip, limit=Limit}=Args,
Args#mrargs{skip=0, limit=Skip+Limit}.
-create_db(DbName) ->
- rexi:reply(case couch_server:create(DbName, []) of
+create_db(DbName, Options) ->
+ rexi:reply(case couch_server:create(DbName, Options) of
{ok, _} ->
ok;
Error ->