You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2014/02/14 20:58:18 UTC
[3/3] couch-mrview commit: updated
refs/heads/1993-bigcouch-couch-mrview to f980f25
WIP: support multi query views
COUCHDB-523
COUCHDB-1993
This is a functional implementation of multi query views on the single
node interface. This is a WIP commit as the implementation is a bit
awkward in some places. In particular, this uses the
couch_mrview_http:view_cb/2 for the entirety of the http response,
which is nice, but it requires introducing state to indicate when
we've reached the last view query, to know when we can complete the
chunked response. Is this better than having multi_query_view/5 finish
the final http chunk and end the response? Pros and cons to either
approach. Once I get some feedback on approach there I'll update
fabric to support this as well.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/f980f250
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/f980f250
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/f980f250
Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: f980f2503a54a1168efe30a13632ef79e34f6982
Parents: 11eb1b3
Author: Russell Branca <ch...@apache.org>
Authored: Fri Feb 14 11:58:18 2014 -0800
Committer: Russell Branca <ch...@apache.org>
Committed: Fri Feb 14 11:58:18 2014 -0800
----------------------------------------------------------------------
include/couch_mrview.hrl | 4 +-
src/couch_mrview_http.erl | 177 +++++++++++++++++++++++++++++++++--------
src/couch_mrview_show.erl | 2 +-
3 files changed, 149 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index ce1547f..003094e 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -86,5 +86,7 @@
req,
resp,
prepend,
- etag
+ etag,
+ multi_query,
+ last_row
}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 71c3845..0f4b36c 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -19,7 +19,7 @@
handle_info_req/3,
handle_compact_req/3,
handle_cleanup_req/2,
- parse_qs/2,
+ parse_params/2,
view_cb/2,
row_to_json/1,
row_to_json/2,
@@ -49,9 +49,24 @@ handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
design_doc_view(Req, Db, DDoc, ViewName, undefined);
handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) ->
[_, _, _, _, ViewName] = Req#httpd.path_parts,
- Keys = get_view_keys(couch_httpd:json_body_obj(Req)),
- couch_stats_collector:increment({httpd, view_reads}),
- design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ Props = couch_httpd:json_body_obj(Req),
+ Keys = get_view_keys(Props),
+ Queries = get_view_queries(Props),
+ case {Queries, Keys} of
+ {Queries, undefined} when is_list(Queries) ->
+ [couch_stats_collector:increment({httpd, view_reads}) || _I <- Queries],
+ multi_query_view(Req, Db, DDoc, ViewName, Queries);
+ {undefined, Keys} when is_list(Keys) ->
+ couch_stats_collector:increment({httpd, view_reads}),
+ design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ {undefined, undefined} ->
+ throw({
+ bad_request,
+ "POST body must contain `keys` or `queries` field"
+ });
+ {_, _} ->
+ throw({bad_request, "`keys` and `queries` are mutually exclusive"})
+ end;
handle_view_req(Req, _Db, _DDoc) ->
couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
@@ -85,7 +100,7 @@ handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) ->
ok = couch_mrview:compact(Db, DDoc),
couch_httpd:send_json(Req, 202, {[{ok, true}]});
handle_compact_req(Req, _Db, _DDoc) ->
- couch_httpd:send_method_not_allowd(Req, "POST").
+ couch_httpd:send_method_not_allowed(Req, "POST").
handle_cleanup_req(#httpd{method='POST'}=Req, Db) ->
@@ -113,7 +128,7 @@ all_docs_req(Req, Db, Keys) ->
do_all_docs_req(Req, Db, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -129,7 +144,7 @@ do_all_docs_req(Req, Db, Keys) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -144,10 +159,76 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end.
-view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
+multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
+ Args0 = parse_params(Req, undefined),
+ %% ETagFun = fun(Sig, Acc0) ->
+ %% check_view_etag(Sig, Acc0, Req)
+ %% end,
+ %% Args = Args0#mrargs{preflight_fun=ETagFun},
+ %% Args2 = couch_util:with_db(
+ %% DbName,
+ %% fun(WDb) ->
+ %% {ok, _VInfo, _Sig, Args1} =
+ %% couch_mrview_util:get_view(WDb, DDoc, ViewName, Args0),
+ %% Args1
+ %% end
+ %% ),
+ {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0),
+ ArgQueries = lists:map(fun({Query}) ->
+ QueryArg = parse_params(Query, undefined, Args1),
+ couch_mrview_util:validate_args(QueryArg)
+ end, Queries),
+ {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
+ %% lists:foreach
+ VAcc0 = #vacc{db=Db, req=Req, multi_query=true},
+ VAcc1 = lists:foldl(fun(Args, {Acc0, N}) ->
+ LastRow = (N-1 =< 0),
+ Acc1 = Acc0#vacc{last_row=LastRow},
+ {ok, Acc2} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun mq_view_cb/2, Acc1),
+ {Acc2, N-1}
+ end, {VAcc0, length(ArgQueries)}, ArgQueries),
+ {ok, VAcc1}
+ end),
+ case is_record(Resp2, vacc) of
+ true -> {ok, Resp2#vacc.resp};
+ _ -> {ok, Resp2}
+ end.
+
+
+mq_view_cb(Msg, Acc) ->
+ view_cb(Msg, Acc).
+
+
+maybe_prepend(#vacc{multi_query=MultiQuery}) ->
+ case MultiQuery of
+ true ->
+ ",\r\n";
+ _ ->
+ ""
+ end.
+
+
+view_cb({meta, Meta}, #vacc{resp=undefined, multi_query=MultiQuery}=Acc) ->
Headers = [{"ETag", Acc#vacc.etag}],
- {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ {ok, Resp0} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ Resp = case MultiQuery of
+ true ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "{\"results\":[\r\n"),
+ Resp1;
+ _ ->
+ Resp0
+ end,
+ %% Prepend = maybe_prepend(Acc),
+ %% view_cb({meta, Meta}, Acc#vacc{resp=Resp, prepend=Prepend});
+ view_cb({meta, Meta}, Acc#vacc{resp=Resp});
+view_cb({meta, Meta}, #vacc{resp=Resp, prepend=Prepend0}=Acc) ->
% Map function starting
+ Prepend = case Prepend0 of
+ undefined ->
+ "";
+ _ ->
+ Prepend0
+ end,
Parts = case couch_util:get_value(total, Meta) of
undefined -> [];
Total -> [io_lib:format("\"total_rows\":~p", [Total])]
@@ -158,7 +239,7 @@ view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
undefined -> [];
UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
end ++ ["\"rows\":["],
- Chunk = lists:flatten("{" ++ string:join(Parts, ",") ++ "\r\n"),
+ Chunk = lists:flatten(Prepend ++ "{" ++ string:join(Parts, ",") ++ "\r\n"),
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
{ok, Acc#vacc{resp=Resp1, prepend=""}};
view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
@@ -167,7 +248,7 @@ view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
Chunk = ["{\"rows\":[\r\n", row_to_json(Row)],
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, #vacc{resp=Resp1, prepend=",\r\n"}};
+ {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
view_cb({row, Row}, Acc) ->
% Adding another row
Chunk = [Acc#vacc.prepend, row_to_json(Row)],
@@ -177,6 +258,18 @@ view_cb(complete, #vacc{resp=undefined}=Acc) ->
% Nothing in view
{ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
{ok, Acc#vacc{resp=Resp}};
+view_cb(complete, #vacc{resp=Resp0, multi_query=true, last_row=LastRow}=Acc) ->
+ % Finish current query of a multiquery view output
+ Prepend = maybe_prepend(Acc),
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "\r\n]}"),
+ {ok, Resp3} = case LastRow of
+ true ->
+ {ok, Resp2} = chttpd:send_delayed_chunk(Resp1, "\r\n]}"),
+ chttpd:end_delayed_json_response(Resp2);
+ _ ->
+ {ok, Resp1}
+ end,
+ {ok, Acc#vacc{resp=Resp3, prepend=Prepend}};
view_cb(complete, #vacc{resp=Resp}=Acc) ->
% Finish view output
chttpd:send_delayed_chunk(Resp, "\r\n]}"),
@@ -223,47 +316,67 @@ get_view_keys({Props}) ->
end.
-parse_qs(Req, Keys) ->
- Args = #mrargs{keys=Keys},
+get_view_queries({Props}) ->
+ case couch_util:get_value(<<"queries">>, Props) of
+ undefined ->
+ undefined;
+ Queries when is_list(Queries) ->
+ Queries;
+ _ ->
+ throw({bad_request, "`queries` member must be a array."})
+ end.
+
+
+parse_params(#httpd{}=Req, Keys) ->
+ parse_params(couch_httpd:qs(Req), Keys);
+parse_params(Props, Keys) ->
+ Args = #mrargs{},
+ parse_params(Props, Keys, Args).
+
+
+parse_params(Props, Keys, #mrargs{}=Args0) ->
+ Args = Args0#mrargs{keys=Keys},
lists:foldl(fun({K, V}, Acc) ->
- parse_qs(K, V, Acc)
- end, Args, couch_httpd:qs(Req)).
+ parse_param(K, V, Acc)
+ end, Args, Props).
-parse_qs(Key, Val, Args) ->
+parse_param(Key, Val, Args) when is_binary(Key) ->
+ parse_param(binary_to_list(Key), Val, Args);
+parse_param(Key, Val, Args) ->
case Key of
"" ->
Args;
"reduce" ->
Args#mrargs{reduce=parse_boolean(Val)};
"key" ->
- JsonKey = ?JSON_DECODE(Val),
+ JsonKey = parse_json(Val),
Args#mrargs{start_key=JsonKey, end_key=JsonKey};
"keys" ->
- Args#mrargs{keys=?JSON_DECODE(Val)};
+ Args#mrargs{keys=parse_json(Val)};
"startkey" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"start_key" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"startkey_docid" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"start_key_doc_id" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"endkey" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"end_key" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"endkey_docid" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"end_key_doc_id" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"limit" ->
Args#mrargs{limit=parse_pos_int(Val)};
"count" ->
throw({query_parse_error, <<"QS param `count` is not `limit`">>});
- "stale" when Val == "ok" ->
+ "stale" when Val == "ok" orelse Val == <<"ok">> ->
Args#mrargs{stale=ok};
- "stale" when Val == "update_after" ->
+ "stale" when Val == "update_after" orelse Val == "update_after" ->
Args#mrargs{stale=update_after};
"stale" ->
throw({query_parse_error, <<"Invalid value for `stale`.">>});
@@ -290,12 +403,12 @@ parse_qs(Key, Val, Args) ->
"conflicts" ->
Args#mrargs{conflicts=parse_boolean(Val)};
"list" ->
- Args#mrargs{list=list_to_binary(Val)};
+ Args#mrargs{list=couch_util:to_binary(Val)};
"callback" ->
- Args#mrargs{callback=list_to_binary(Val)};
+ Args#mrargs{callback=couch_util:to_binary(Val)};
_ ->
- BKey = list_to_binary(Key),
- BVal = list_to_binary(Val),
+ BKey = couch_util:to_binary(Key),
+ BVal = couch_util:to_binary(Val),
Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]}
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/src/couch_mrview_show.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_show.erl b/src/couch_mrview_show.erl
index 1be96d5..0b42da5 100644
--- a/src/couch_mrview_show.erl
+++ b/src/couch_mrview_show.erl
@@ -193,7 +193,7 @@ handle_view_list_req(Req, _Db, _DDoc) ->
handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
- Args0 = couch_mrview_http:parse_qs(Req, Keys),
+ Args0 = couch_mrview_http:parse_params(Req, Keys),
ETagFun = fun(BaseSig, Acc0) ->
UserCtx = Req#httpd.user_ctx,
Name = UserCtx#user_ctx.name,