You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2014/05/06 18:33:21 UTC
[1/4] couch-mrview commit: updated refs/heads/master to 01cff34
Repository: couchdb-couch-mrview
Updated Branches:
refs/heads/master [created] 01cff34df
Merge remote-tracking branch 'origin/import-master'
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/86de6ccf
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/86de6ccf
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/86de6ccf
Branch: refs/heads/master
Commit: 86de6ccffb3b851b31c609f4f58e4f89c9517e8c
Parents: 3a093fa deec893
Author: Robert Newson <rn...@apache.org>
Authored: Tue May 6 16:56:00 2014 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Tue May 6 16:56:00 2014 +0100
----------------------------------------------------------------------
include/couch_mrview.hrl | 1 +
src/couch_mrview_compactor.erl | 12 ++-
src/couch_mrview_http.erl | 81 +++++++++++++++++--
src/couch_mrview_index.erl | 21 +++++
src/couch_mrview_show.erl | 7 +-
src/couch_mrview_updater.erl | 46 +++++------
src/couch_mrview_util.erl | 153 ++++++++++++++++++++++++++++--------
7 files changed, 252 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --cc include/couch_mrview.hrl
index 6a0dfd0,e4ec66d..33dd061
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@@ -70,9 -70,9 +70,10 @@@
skip = 0,
group_level = 0,
stale = false,
+ multi_get = false,
inclusive_end = true,
include_docs = false,
+ doc_options = [],
update_seq=false,
conflicts,
callback,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_http.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_index.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_show.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --cc src/couch_mrview_updater.erl
index 99aedd0,9604ea9..7aee110
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@@ -140,44 -156,16 +156,18 @@@ map_docs(Parent, State0) -
end.
- compute_map_results(#mrst{qserver = Qs}, Dequeued) ->
- % Run all the non deleted docs through the view engine and
- % then pass the results on to the writer process.
- DocFun = fun
- ({nil, Seq, _}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), AccDel, AccNotDel};
- ({Id, Seq, deleted}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), [{Id, []} | AccDel], AccNotDel};
- ({_Id, Seq, Doc}, {SeqAcc, AccDel, AccNotDel}) ->
- {erlang:max(Seq, SeqAcc), AccDel, [Doc | AccNotDel]}
- end,
- FoldFun = fun(Docs, Acc) ->
- lists:foldl(DocFun, Acc, Docs)
- end,
- {MaxSeq, DeletedResults, Docs} =
- lists:foldl(FoldFun, {0, [], []}, Dequeued),
- {ok, MapResultList} = couch_query_servers:map_docs_raw(Qs, Docs),
- NotDeletedResults = lists:zipwith(
- fun(#doc{id = Id}, MapResults) -> {Id, MapResults} end,
- Docs,
- MapResultList),
- AllMapResults = DeletedResults ++ NotDeletedResults,
- update_task(length(AllMapResults)),
- {ok, {MaxSeq, AllMapResults}}.
-
-
write_results(Parent, State) ->
- case couch_work_queue:dequeue(State#mrst.write_queue) of
- closed ->
+ case accumulate_writes(State, State#mrst.write_queue, nil) of
+ stop ->
Parent ! {new_state, State};
- {ok, Info} ->
- EmptyKVs = [{V#mrview.id_num, []} || V <- State#mrst.views],
- {Seq, ViewKVs, DocIdKeys} = merge_results(Info, 0, EmptyKVs, []),
+ {Go, {Seq, ViewKVs, DocIdKeys}} ->
NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys),
- send_partial(NewState#mrst.partial_resp_pid, NewState),
- write_results(Parent, NewState)
+ if Go == stop ->
+ Parent ! {new_state, NewState};
+ true ->
+ send_partial(NewState#mrst.partial_resp_pid, NewState),
+ write_results(Parent, NewState)
+ end
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/86de6ccf/src/couch_mrview_util.erl
----------------------------------------------------------------------
[3/4] couch-mrview commit: updated refs/heads/master to 01cff34
Posted by rn...@apache.org.
Fix reference to renamed tree
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/fd525897
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/fd525897
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/fd525897
Branch: refs/heads/master
Commit: fd52589769216ca4c7e759927d21e10fd6969f79
Parents: a816272
Author: Robert Newson <rn...@apache.org>
Authored: Tue May 6 17:03:27 2014 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Tue May 6 17:03:27 2014 +0100
----------------------------------------------------------------------
src/couch_mrview_compactor.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/fd525897/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 6cd8ae9..f8293bd 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -48,7 +48,7 @@ compact(State) ->
{ok, Fd} = couch_mrview_util:open_file(CompactFName),
ESt = couch_mrview_util:reset_index(Db, Fd, State),
- {ok, DbReduce} = couch_btree:full_reduce(Db#db.fulldocinfo_by_id_btree),
+ {ok, DbReduce} = couch_btree:full_reduce(Db#db.id_tree),
Count = element(1, DbReduce),
{ESt, Count}
[4/4] couch-mrview commit: updated refs/heads/master to 01cff34
Posted by rn...@apache.org.
remove refcounting introduced by new upgrade path
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/01cff34d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/01cff34d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/01cff34d
Branch: refs/heads/master
Commit: 01cff34df1cf6aa56ffd713f28968ee778d265ae
Parents: fd52589
Author: Robert Newson <rn...@apache.org>
Authored: Tue May 6 17:12:10 2014 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Tue May 6 17:12:46 2014 +0100
----------------------------------------------------------------------
src/couch_mrview_index.erl | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/01cff34d/src/couch_mrview_index.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index a455e0a..5baee7c 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -96,8 +96,7 @@ open(Db, State) ->
{ok, {OldSig, Header}} ->
% Matching view signatures.
NewSt = couch_mrview_util:init_state(Db, Fd, State, Header),
- {ok, RefCounter} = couch_ref_counter:start([Fd]),
- {ok, NewSt#mrst{refc=RefCounter}};
+ {ok, NewSt#mrst{fd_monitor=erlang:monitor(process, Fd)}};
% end of upgrade code for <= 1.2.x
{ok, {Sig, Header}} ->
% Matching view signatures.
[2/4] couch-mrview commit: updated refs/heads/master to 01cff34
Posted by rn...@apache.org.
Merge remote-tracking branch 'origin/1993-bigcouch-couch-mrview'
Conflicts:
src/couch_mrview_http.erl
src/couch_mrview_util.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/a8162720
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/a8162720
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/a8162720
Branch: refs/heads/master
Commit: a81627207e4f1e84a96164536dfe85292c88603a
Parents: 86de6cc c186e47
Author: Robert Newson <rn...@apache.org>
Authored: Tue May 6 16:57:46 2014 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Tue May 6 16:57:46 2014 +0100
----------------------------------------------------------------------
include/couch_mrview.hrl | 20 ++++
src/couch_mrview.erl | 4 +
src/couch_mrview_http.erl | 228 +++++++++++++++++++++++++++--------------
src/couch_mrview_show.erl | 28 ++---
src/couch_mrview_util.erl | 30 ++++++
5 files changed, 218 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a8162720/include/couch_mrview.hrl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a8162720/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --cc src/couch_mrview_http.erl
index 302efde,8735187..fc0a143
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@@ -129,14 -130,11 +146,10 @@@ all_docs_req(Req, Db, Keys) -
do_all_docs_req(Req, Db, Keys)
end.
-
do_all_docs_req(Req, Db, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
- ETag = couch_httpd:make_etag(Sig),
- case couch_httpd:etag_match(Req, ETag) of
- true -> throw({etag_match, ETag});
- false -> {ok, Acc0#vacc{etag=ETag}}
- end
+ check_view_etag(Sig, Acc0, Req)
end,
Args = Args0#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
@@@ -154,35 -146,11 +167,31 @@@
_ -> {ok, Resp}
end.
+is_admin(Db) ->
+ case catch couch_db:check_is_admin(Db) of
+ {unauthorized, _} ->
+ false;
+ ok ->
+ true
+ end.
+
+
+% admin users always get all fields
+get_view_callback(_, _, true) ->
+ fun view_cb/2;
+% if we are operating on the users db and we aren't
+% admin, filter the view
+get_view_callback(_DbName, _DbName, false) ->
+ fun filtered_view_cb/2;
+% non _users databases get all fields
+get_view_callback(_, _, _) ->
+ fun view_cb/2.
+
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
- ETag = couch_httpd:make_etag(Sig),
- case couch_httpd:etag_match(Req, ETag) of
- true -> throw({etag_match, ETag});
- false -> {ok, Acc0#vacc{etag=ETag}}
- end
+ check_view_etag(Sig, Acc0, Req)
end,
Args = Args0#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
@@@ -195,24 -163,42 +204,55 @@@
end.
+filtered_view_cb({row, Row0}, Acc) ->
+ Row1 = lists:map(fun({doc, null}) ->
+ {doc, null};
+ ({doc, Body}) ->
+ Doc = couch_users_db:strip_non_public_fields(#doc{body=Body}),
+ {doc, Doc#doc.body};
+ (KV) ->
+ KV
+ end, Row0),
+ view_cb({row, Row1}, Acc);
+filtered_view_cb(Obj, Acc) ->
+ view_cb(Obj, Acc).
+
+ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
+ Args0 = parse_params(Req, undefined),
+ {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0),
+ ArgQueries = lists:map(fun({Query}) ->
+ QueryArg = parse_params(Query, undefined, Args1),
+ couch_mrview_util:validate_args(QueryArg)
+ end, Queries),
+ {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
+ VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
+ %% TODO: proper calculation of etag
+ Etag = couch_uuids:new(),
+ Headers = [{"ETag", Etag}],
+ FirstChunk = "{\"results\":[",
+ {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, 200, Headers, FirstChunk),
+ VAcc1 = VAcc0#vacc{resp=Resp0},
+ VAcc2 = lists:foldl(fun(Args, Acc0) ->
+ {ok, Acc1} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, Acc0),
+ Acc1
+ end, VAcc1, ArgQueries),
+ {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, VAcc2#vacc{resp=Resp2}}
+ end),
+ case is_record(Resp2, vacc) of
+ true -> {ok, Resp2#vacc.resp};
+ _ -> {ok, Resp2}
+ end.
+
view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
- Headers = [{"ETag", Acc#vacc.etag}],
- {ok, Resp} = couch_httpd:start_json_response(Acc#vacc.req, 200, Headers),
% Map function starting
+ Headers = [{"ETag", Acc#vacc.etag}],
+ {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ view_cb({meta, Meta}, Acc#vacc{resp=Resp, should_close=true});
+ view_cb({meta, Meta}, #vacc{resp=Resp}=Acc) ->
+ % Sending metadata
Parts = case couch_util:get_value(total, Meta) of
undefined -> [];
Total -> [io_lib:format("\"total_rows\":~p", [Total])]
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a8162720/src/couch_mrview_show.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/a8162720/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --cc src/couch_mrview_util.erl
index fc3ebfe,815bf08..cbba5ac
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@@ -24,7 -24,9 +24,10 @@@
-export([calculate_data_size/2]).
-export([validate_args/1]).
-export([maybe_load_doc/3, maybe_load_doc/4]).
+-export([maybe_update_index_file/1]).
+ -export([extract_view/4, extract_view_reduce/1]).
+ -export([get_view_keys/1, get_view_queries/1]).
+ -export([set_view_type/3]).
-define(MOD, couch_mrview_index).
@@@ -714,88 -713,29 +717,115 @@@ mrverror(Mesg) -
throw({query_parse_error, Mesg}).
+%% Updates 1.2.x or earlier view files to 1.3.x or later view files
+%% transparently, the first time the 1.2.x view file is opened by
+%% 1.3.x or later.
+%%
+%% Here's how it works:
+%%
+%% Before opening a view index,
+%% If no matching index file is found in the new location:
+%% calculate the <= 1.2.x view signature
+%% if a file with that signature lives in the old location
+%% rename it to the new location with the new signature in the name.
+%% Then proceed to open the view index as usual.
+%% After opening, read its header.
+%%
+%% If the header matches the <= 1.2.x style #index_header record:
+%% upgrade the header to the new #mrheader record
+%% The next time the view is used, the new header is used.
+%%
+%% If we crash after the rename, but before the header upgrade,
+%% the header upgrade is done on the next view opening.
+%%
+%% If we crash between upgrading to the new header and writing
+%% that header to disk, we start with the old header again,
+%% do the upgrade and write to disk.
+
+maybe_update_index_file(State) ->
+ DbName = State#mrst.db_name,
+ NewIndexFile = index_file(DbName, State#mrst.sig),
+ % open in read-only mode so we don't create
+ % the file if it doesn't exist.
+ case file:open(NewIndexFile, [read, raw]) of
+ {ok, Fd_Read} ->
+ % the new index file exists, there is nothing to do here.
+ file:close(Fd_Read);
+ _Error ->
+ update_index_file(State)
+ end.
+
+update_index_file(State) ->
+ Sig = sig_vsn_12x(State),
+ DbName = State#mrst.db_name,
+ FileName = couch_index_util:hexsig(Sig) ++ ".view",
+ IndexFile = couch_index_util:index_file("", DbName, FileName),
+
+ % If we have an old index, rename it to the new position.
+ case file:read_file_info(IndexFile) of
+ {ok, _FileInfo} ->
+ % Crash if the rename fails for any reason.
+ % If the target exists, e.g. the next request will find the
+ % new file and we are good. We might need to catch this
+ % further up to avoid a full server crash.
+ ?LOG_INFO("Attempting to update legacy view index file.", []),
+ NewIndexFile = index_file(DbName, State#mrst.sig),
+ ok = filelib:ensure_dir(NewIndexFile),
+ ok = file:rename(IndexFile, NewIndexFile),
+ ?LOG_INFO("Successfully updated legacy view index file.", []),
+ Sig;
+ _ ->
+ % Ignore missing index file
+ ok
+ end.
+
+sig_vsn_12x(State) ->
+ ViewInfo = [old_view_format(V) || V <- State#mrst.views],
+ SigData = case State#mrst.lib of
+ {[]} ->
+ {ViewInfo, State#mrst.language, State#mrst.design_opts};
+ _ ->
+ {ViewInfo, State#mrst.language, State#mrst.design_opts,
+ couch_index_util:sort_lib(State#mrst.lib)}
+ end,
+ couch_util:md5(term_to_binary(SigData)).
+
+old_view_format(View) ->
+{
+ view,
+ View#mrview.id_num,
+ View#mrview.map_names,
+ View#mrview.def,
+ View#mrview.btree,
+ View#mrview.reduce_funs,
+ View#mrview.options
+}.
+
+%% End of <= 1.2.x upgrade code.
++
+ extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs=Reds}}, _Ref}) ->
+ {_Name, FunSrc} = lists:nth(N, Reds),
+ FunSrc.
+
+
+ get_view_keys({Props}) ->
+ case couch_util:get_value(<<"keys">>, Props) of
+ undefined ->
+ ?LOG_DEBUG("POST with no keys member.", []),
+ undefined;
+ Keys when is_list(Keys) ->
+ Keys;
+ _ ->
+ throw({bad_request, "`keys` member must be a array."})
+ end.
+
+
+ get_view_queries({Props}) ->
+ case couch_util:get_value(<<"queries">>, Props) of
+ undefined ->
+ undefined;
+ Queries when is_list(Queries) ->
+ Queries;
+ _ ->
+ throw({bad_request, "`queries` member must be a array."})
+ end.