You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2014/02/14 21:38:05 UTC
couch-mrview commit: updated refs/heads/1993-bigcouch-couch-mrview to
22ec9d9
Updated Branches:
refs/heads/1993-bigcouch-couch-mrview f980f2503 -> 22ec9d906 (forced update)
WIP: support multi query views
COUCHDB-523
COUCHDB-1993
This is a functional implementation of multi query views on the single
node interface. This is a WIP commit as the implementation is a bit
awkward in some places. In particular, this uses the
couch_mrview_http:view_cb/2 for the entirety of the http response,
which is nice, but it requires introducing state to indicate when
we've reached the last view query, to know when we can complete the
chunked response. Is this better than having multi_query_view/5 finish
the final http chunk and end the response? Pros and cons to either
approach. Once I get some feedback on approach there I'll update
fabric to support this as well.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/22ec9d90
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/22ec9d90
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/22ec9d90
Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: 22ec9d906251010eced428765d24ba883e511b91
Parents: 11eb1b3
Author: Russell Branca <ch...@apache.org>
Authored: Fri Feb 14 11:58:18 2014 -0800
Committer: Russell Branca <ch...@apache.org>
Committed: Fri Feb 14 12:38:08 2014 -0800
----------------------------------------------------------------------
include/couch_mrview.hrl | 3 +-
src/couch_mrview_http.erl | 151 ++++++++++++++++++++++++++++++++---------
src/couch_mrview_show.erl | 2 +-
3 files changed, 122 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index ce1547f..8791249 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -86,5 +86,6 @@
req,
resp,
prepend,
- etag
+ etag,
+ multi_query
}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 71c3845..87a56b0 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -19,7 +19,7 @@
handle_info_req/3,
handle_compact_req/3,
handle_cleanup_req/2,
- parse_qs/2,
+ parse_params/2,
view_cb/2,
row_to_json/1,
row_to_json/2,
@@ -49,9 +49,24 @@ handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
design_doc_view(Req, Db, DDoc, ViewName, undefined);
handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) ->
[_, _, _, _, ViewName] = Req#httpd.path_parts,
- Keys = get_view_keys(couch_httpd:json_body_obj(Req)),
- couch_stats_collector:increment({httpd, view_reads}),
- design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ Props = couch_httpd:json_body_obj(Req),
+ Keys = get_view_keys(Props),
+ Queries = get_view_queries(Props),
+ case {Queries, Keys} of
+ {Queries, undefined} when is_list(Queries) ->
+ [couch_stats_collector:increment({httpd, view_reads}) || _I <- Queries],
+ multi_query_view(Req, Db, DDoc, ViewName, Queries);
+ {undefined, Keys} when is_list(Keys) ->
+ couch_stats_collector:increment({httpd, view_reads}),
+ design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ {undefined, undefined} ->
+ throw({
+ bad_request,
+ "POST body must contain `keys` or `queries` field"
+ });
+ {_, _} ->
+ throw({bad_request, "`keys` and `queries` are mutually exclusive"})
+ end;
handle_view_req(Req, _Db, _DDoc) ->
couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
@@ -85,7 +100,7 @@ handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) ->
ok = couch_mrview:compact(Db, DDoc),
couch_httpd:send_json(Req, 202, {[{ok, true}]});
handle_compact_req(Req, _Db, _DDoc) ->
- couch_httpd:send_method_not_allowd(Req, "POST").
+ couch_httpd:send_method_not_allowed(Req, "POST").
handle_cleanup_req(#httpd{method='POST'}=Req, Db) ->
@@ -113,7 +128,7 @@ all_docs_req(Req, Db, Keys) ->
do_all_docs_req(Req, Db, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -129,7 +144,7 @@ do_all_docs_req(Req, Db, Keys) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -144,10 +159,58 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end.
-view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
+multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
+ Args0 = parse_params(Req, undefined),
+ %% ETagFun = fun(Sig, Acc0) ->
+ %% check_view_etag(Sig, Acc0, Req)
+ %% end,
+ %% Args = Args0#mrargs{preflight_fun=ETagFun},
+ %% Args2 = couch_util:with_db(
+ %% DbName,
+ %% fun(WDb) ->
+ %% {ok, _VInfo, _Sig, Args1} =
+ %% couch_mrview_util:get_view(WDb, DDoc, ViewName, Args0),
+ %% Args1
+ %% end
+ %% ),
+ {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0),
+ ArgQueries = lists:map(fun({Query}) ->
+ QueryArg = parse_params(Query, undefined, Args1),
+ couch_mrview_util:validate_args(QueryArg)
+ end, Queries),
+ {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
+ VAcc0 = #vacc{db=Db, req=Req, multi_query=true},
+ Headers = [{"ETag", VAcc0#vacc.etag}],
+ FirstChunk = "{\"results\":[",
+ {ok, Resp0} = chttpd:start_delayed_json_response(VAcc0#vacc.req, 200, Headers, FirstChunk),
+ VAcc1 = VAcc0#vacc{resp=Resp0},
+ VAcc2 = lists:foldl(fun(Args, Acc0) ->
+ {ok, Acc1} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, Acc0),
+ Acc1
+ end, VAcc1, ArgQueries),
+ {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
+ {ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
+ {ok, VAcc2#vacc{resp=Resp2}}
+ end),
+ case is_record(Resp2, vacc) of
+ true -> {ok, Resp2#vacc.resp};
+ _ -> {ok, Resp2}
+ end.
+
+
+view_cb({meta, Meta}, #vacc{resp=undefined, multi_query=undefined}=Acc) ->
+ % Map function starting
Headers = [{"ETag", Acc#vacc.etag}],
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
- % Map function starting
+ view_cb({meta, Meta}, Acc#vacc{resp=Resp});
+view_cb({meta, Meta}, #vacc{resp=Resp, prepend=Prepend0}=Acc) ->
+ % Sending metadata
+ Prepend = case Prepend0 of
+ undefined ->
+ "";
+ _ ->
+ Prepend0
+ end,
Parts = case couch_util:get_value(total, Meta) of
undefined -> [];
Total -> [io_lib:format("\"total_rows\":~p", [Total])]
@@ -158,7 +221,7 @@ view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
undefined -> [];
UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
end ++ ["\"rows\":["],
- Chunk = lists:flatten("{" ++ string:join(Parts, ",") ++ "\r\n"),
+ Chunk = lists:flatten(Prepend ++ "{" ++ string:join(Parts, ",") ++ "\r\n"),
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
{ok, Acc#vacc{resp=Resp1, prepend=""}};
view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
@@ -167,7 +230,7 @@ view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
Chunk = ["{\"rows\":[\r\n", row_to_json(Row)],
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, #vacc{resp=Resp1, prepend=",\r\n"}};
+ {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
view_cb({row, Row}, Acc) ->
% Adding another row
Chunk = [Acc#vacc.prepend, row_to_json(Row)],
@@ -177,6 +240,10 @@ view_cb(complete, #vacc{resp=undefined}=Acc) ->
% Nothing in view
{ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
{ok, Acc#vacc{resp=Resp}};
+view_cb(complete, #vacc{resp=Resp0, multi_query=true}=Acc) ->
+ % Finish current query of a multiquery view output
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "\r\n]}"),
+ {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
view_cb(complete, #vacc{resp=Resp}=Acc) ->
% Finish view output
chttpd:send_delayed_chunk(Resp, "\r\n]}"),
@@ -223,47 +290,67 @@ get_view_keys({Props}) ->
end.
-parse_qs(Req, Keys) ->
- Args = #mrargs{keys=Keys},
+get_view_queries({Props}) ->
+ case couch_util:get_value(<<"queries">>, Props) of
+ undefined ->
+ undefined;
+ Queries when is_list(Queries) ->
+ Queries;
+ _ ->
+ throw({bad_request, "`queries` member must be a array."})
+ end.
+
+
+parse_params(#httpd{}=Req, Keys) ->
+ parse_params(couch_httpd:qs(Req), Keys);
+parse_params(Props, Keys) ->
+ Args = #mrargs{},
+ parse_params(Props, Keys, Args).
+
+
+parse_params(Props, Keys, #mrargs{}=Args0) ->
+ Args = Args0#mrargs{keys=Keys},
lists:foldl(fun({K, V}, Acc) ->
- parse_qs(K, V, Acc)
- end, Args, couch_httpd:qs(Req)).
+ parse_param(K, V, Acc)
+ end, Args, Props).
-parse_qs(Key, Val, Args) ->
+parse_param(Key, Val, Args) when is_binary(Key) ->
+ parse_param(binary_to_list(Key), Val, Args);
+parse_param(Key, Val, Args) ->
case Key of
"" ->
Args;
"reduce" ->
Args#mrargs{reduce=parse_boolean(Val)};
"key" ->
- JsonKey = ?JSON_DECODE(Val),
+ JsonKey = parse_json(Val),
Args#mrargs{start_key=JsonKey, end_key=JsonKey};
"keys" ->
- Args#mrargs{keys=?JSON_DECODE(Val)};
+ Args#mrargs{keys=parse_json(Val)};
"startkey" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"start_key" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"startkey_docid" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"start_key_doc_id" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"endkey" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"end_key" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"endkey_docid" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"end_key_doc_id" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"limit" ->
Args#mrargs{limit=parse_pos_int(Val)};
"count" ->
throw({query_parse_error, <<"QS param `count` is not `limit`">>});
- "stale" when Val == "ok" ->
+ "stale" when Val == "ok" orelse Val == <<"ok">> ->
Args#mrargs{stale=ok};
- "stale" when Val == "update_after" ->
+ "stale" when Val == "update_after" orelse Val == "update_after" ->
Args#mrargs{stale=update_after};
"stale" ->
throw({query_parse_error, <<"Invalid value for `stale`.">>});
@@ -290,12 +377,12 @@ parse_qs(Key, Val, Args) ->
"conflicts" ->
Args#mrargs{conflicts=parse_boolean(Val)};
"list" ->
- Args#mrargs{list=list_to_binary(Val)};
+ Args#mrargs{list=couch_util:to_binary(Val)};
"callback" ->
- Args#mrargs{callback=list_to_binary(Val)};
+ Args#mrargs{callback=couch_util:to_binary(Val)};
_ ->
- BKey = list_to_binary(Key),
- BVal = list_to_binary(Val),
+ BKey = couch_util:to_binary(Key),
+ BVal = couch_util:to_binary(Val),
Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]}
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/22ec9d90/src/couch_mrview_show.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_show.erl b/src/couch_mrview_show.erl
index 1be96d5..0b42da5 100644
--- a/src/couch_mrview_show.erl
+++ b/src/couch_mrview_show.erl
@@ -193,7 +193,7 @@ handle_view_list_req(Req, _Db, _DDoc) ->
handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
- Args0 = couch_mrview_http:parse_qs(Req, Keys),
+ Args0 = couch_mrview_http:parse_params(Req, Keys),
ETagFun = fun(BaseSig, Acc0) ->
UserCtx = Req#httpd.user_ctx,
Name = UserCtx#user_ctx.name,