You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ch...@apache.org on 2014/02/14 20:58:16 UTC
[1/3] couch-mrview commit: updated
refs/heads/1993-bigcouch-couch-mrview to f980f25
Updated Branches:
refs/heads/1993-bigcouch-couch-mrview bf513f0bd -> f980f2503
Bring in RCouch parse updates
This brings in updates from 53b8373 that allow the parse functions to handle
previously parsed JSON bodies. In particular this is used for parsing query
params in the request body.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/dae60784
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/dae60784
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/dae60784
Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: dae60784f87910e03bc26fbee77a39d1cb600e25
Parents: bf513f0
Author: Russell Branca <ch...@gmail.com>
Authored: Thu Feb 13 12:32:38 2014 -0800
Committer: Russell Branca <ch...@gmail.com>
Committed: Thu Feb 13 12:32:38 2014 -0800
----------------------------------------------------------------------
src/couch_mrview_http.erl | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/dae60784/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 1277c6a..944b1ac 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -26,6 +26,9 @@
check_view_etag/3
]).
+-export([parse_boolean/1,
+ parse_int/1,
+ parse_pos_int/1]).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -297,6 +300,10 @@ parse_qs(Key, Val, Args) ->
end.
+parse_boolean(true) ->
+ true;
+parse_boolean(false) ->
+ false;
parse_boolean(Val) ->
case string:to_lower(Val) of
"true" -> true;
@@ -307,6 +314,8 @@ parse_boolean(Val) ->
end.
+parse_int(Val) when is_integer(Val) ->
+ Val;
parse_int(Val) ->
case (catch list_to_integer(Val)) of
IntVal when is_integer(IntVal) ->
[2/3] couch-mrview commit: updated
refs/heads/1993-bigcouch-couch-mrview to f980f25
Posted by ch...@apache.org.
Bring in RCouch's parse_json function
This brings in the parse_json function from couch-chttpd 70409b20 which will
not reparse binaries that have already been JSON decoded.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/11eb1b3c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/11eb1b3c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/11eb1b3c
Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: 11eb1b3ce04b8a922d4ebff07c2cb708e2e7f1ba
Parents: dae6078
Author: Russell Branca <ch...@gmail.com>
Authored: Thu Feb 13 12:34:47 2014 -0800
Committer: Russell Branca <ch...@gmail.com>
Committed: Thu Feb 13 12:34:47 2014 -0800
----------------------------------------------------------------------
src/couch_mrview_http.erl | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/11eb1b3c/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 944b1ac..71c3845 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -343,3 +343,9 @@ check_view_etag(Sig, Acc0, Req) ->
true -> throw({etag_match, ETag});
false -> {ok, Acc0#vacc{etag=ETag}}
end.
+
+
+parse_json(V) when is_list(V) ->
+ ?JSON_DECODE(V);
+parse_json(V) ->
+ V.
[3/3] couch-mrview commit: updated
refs/heads/1993-bigcouch-couch-mrview to f980f25
Posted by ch...@apache.org.
WIP: support multi query views
COUCHDB-523
COUCHDB-1993
This is a functional implementation of multi query views on the single
node interface. This is a WIP commit as the implementation is a bit
awkward in some places. In particular, this uses the
couch_mrview_http:view_cb/2 for the entirety of the http response,
which is nice, but it requires introducing state to indicate when
we've reached the last view query, to know when we can complete the
chunked response. Is this better than having multi_query_view/5 finish
the final http chunk and end the response? Pros and cons to either
approach. Once I get some feedback on approach there I'll update
fabric to support this as well.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/f980f250
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/f980f250
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/f980f250
Branch: refs/heads/1993-bigcouch-couch-mrview
Commit: f980f2503a54a1168efe30a13632ef79e34f6982
Parents: 11eb1b3
Author: Russell Branca <ch...@apache.org>
Authored: Fri Feb 14 11:58:18 2014 -0800
Committer: Russell Branca <ch...@apache.org>
Committed: Fri Feb 14 11:58:18 2014 -0800
----------------------------------------------------------------------
include/couch_mrview.hrl | 4 +-
src/couch_mrview_http.erl | 177 +++++++++++++++++++++++++++++++++--------
src/couch_mrview_show.erl | 2 +-
3 files changed, 149 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index ce1547f..003094e 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -86,5 +86,7 @@
req,
resp,
prepend,
- etag
+ etag,
+ multi_query,
+ last_row
}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 71c3845..0f4b36c 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -19,7 +19,7 @@
handle_info_req/3,
handle_compact_req/3,
handle_cleanup_req/2,
- parse_qs/2,
+ parse_params/2,
view_cb/2,
row_to_json/1,
row_to_json/2,
@@ -49,9 +49,24 @@ handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
design_doc_view(Req, Db, DDoc, ViewName, undefined);
handle_view_req(#httpd{method='POST'}=Req, Db, DDoc) ->
[_, _, _, _, ViewName] = Req#httpd.path_parts,
- Keys = get_view_keys(couch_httpd:json_body_obj(Req)),
- couch_stats_collector:increment({httpd, view_reads}),
- design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ Props = couch_httpd:json_body_obj(Req),
+ Keys = get_view_keys(Props),
+ Queries = get_view_queries(Props),
+ case {Queries, Keys} of
+ {Queries, undefined} when is_list(Queries) ->
+ [couch_stats_collector:increment({httpd, view_reads}) || _I <- Queries],
+ multi_query_view(Req, Db, DDoc, ViewName, Queries);
+ {undefined, Keys} when is_list(Keys) ->
+ couch_stats_collector:increment({httpd, view_reads}),
+ design_doc_view(Req, Db, DDoc, ViewName, Keys);
+ {undefined, undefined} ->
+ throw({
+ bad_request,
+ "POST body must contain `keys` or `queries` field"
+ });
+ {_, _} ->
+ throw({bad_request, "`keys` and `queries` are mutually exclusive"})
+ end;
handle_view_req(Req, _Db, _DDoc) ->
couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
@@ -85,7 +100,7 @@ handle_compact_req(#httpd{method='POST'}=Req, Db, DDoc) ->
ok = couch_mrview:compact(Db, DDoc),
couch_httpd:send_json(Req, 202, {[{ok, true}]});
handle_compact_req(Req, _Db, _DDoc) ->
- couch_httpd:send_method_not_allowd(Req, "POST").
+ couch_httpd:send_method_not_allowed(Req, "POST").
handle_cleanup_req(#httpd{method='POST'}=Req, Db) ->
@@ -113,7 +128,7 @@ all_docs_req(Req, Db, Keys) ->
do_all_docs_req(Req, Db, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -129,7 +144,7 @@ do_all_docs_req(Req, Db, Keys) ->
design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
- Args0 = parse_qs(Req, Keys),
+ Args0 = parse_params(Req, Keys),
ETagFun = fun(Sig, Acc0) ->
check_view_etag(Sig, Acc0, Req)
end,
@@ -144,10 +159,76 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end.
-view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
+multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
+ Args0 = parse_params(Req, undefined),
+ %% ETagFun = fun(Sig, Acc0) ->
+ %% check_view_etag(Sig, Acc0, Req)
+ %% end,
+ %% Args = Args0#mrargs{preflight_fun=ETagFun},
+ %% Args2 = couch_util:with_db(
+ %% DbName,
+ %% fun(WDb) ->
+ %% {ok, _VInfo, _Sig, Args1} =
+ %% couch_mrview_util:get_view(WDb, DDoc, ViewName, Args0),
+ %% Args1
+ %% end
+ %% ),
+ {ok, _, _, Args1} = couch_mrview_util:get_view(Db, DDoc, ViewName, Args0),
+ ArgQueries = lists:map(fun({Query}) ->
+ QueryArg = parse_params(Query, undefined, Args1),
+ couch_mrview_util:validate_args(QueryArg)
+ end, Queries),
+ {ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
+ %% lists:foreach
+ VAcc0 = #vacc{db=Db, req=Req, multi_query=true},
+ VAcc1 = lists:foldl(fun(Args, {Acc0, N}) ->
+ LastRow = (N-1 =< 0),
+ Acc1 = Acc0#vacc{last_row=LastRow},
+ {ok, Acc2} = couch_mrview:query_view(Db, DDoc, ViewName, Args, fun mq_view_cb/2, Acc1),
+ {Acc2, N-1}
+ end, {VAcc0, length(ArgQueries)}, ArgQueries),
+ {ok, VAcc1}
+ end),
+ case is_record(Resp2, vacc) of
+ true -> {ok, Resp2#vacc.resp};
+ _ -> {ok, Resp2}
+ end.
+
+
+mq_view_cb(Msg, Acc) ->
+ view_cb(Msg, Acc).
+
+
+maybe_prepend(#vacc{multi_query=MultiQuery}) ->
+ case MultiQuery of
+ true ->
+ ",\r\n";
+ _ ->
+ ""
+ end.
+
+
+view_cb({meta, Meta}, #vacc{resp=undefined, multi_query=MultiQuery}=Acc) ->
Headers = [{"ETag", Acc#vacc.etag}],
- {ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ {ok, Resp0} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
+ Resp = case MultiQuery of
+ true ->
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "{\"results\":[\r\n"),
+ Resp1;
+ _ ->
+ Resp0
+ end,
+ %% Prepend = maybe_prepend(Acc),
+ %% view_cb({meta, Meta}, Acc#vacc{resp=Resp, prepend=Prepend});
+ view_cb({meta, Meta}, Acc#vacc{resp=Resp});
+view_cb({meta, Meta}, #vacc{resp=Resp, prepend=Prepend0}=Acc) ->
% Map function starting
+ Prepend = case Prepend0 of
+ undefined ->
+ "";
+ _ ->
+ Prepend0
+ end,
Parts = case couch_util:get_value(total, Meta) of
undefined -> [];
Total -> [io_lib:format("\"total_rows\":~p", [Total])]
@@ -158,7 +239,7 @@ view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
undefined -> [];
UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
end ++ ["\"rows\":["],
- Chunk = lists:flatten("{" ++ string:join(Parts, ",") ++ "\r\n"),
+ Chunk = lists:flatten(Prepend ++ "{" ++ string:join(Parts, ",") ++ "\r\n"),
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
{ok, Acc#vacc{resp=Resp1, prepend=""}};
view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
@@ -167,7 +248,7 @@ view_cb({row, Row}, #vacc{resp=undefined}=Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
Chunk = ["{\"rows\":[\r\n", row_to_json(Row)],
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, #vacc{resp=Resp1, prepend=",\r\n"}};
+ {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}};
view_cb({row, Row}, Acc) ->
% Adding another row
Chunk = [Acc#vacc.prepend, row_to_json(Row)],
@@ -177,6 +258,18 @@ view_cb(complete, #vacc{resp=undefined}=Acc) ->
% Nothing in view
{ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
{ok, Acc#vacc{resp=Resp}};
+view_cb(complete, #vacc{resp=Resp0, multi_query=true, last_row=LastRow}=Acc) ->
+ % Finish current query of a multiquery view output
+ Prepend = maybe_prepend(Acc),
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, "\r\n]}"),
+ {ok, Resp3} = case LastRow of
+ true ->
+ {ok, Resp2} = chttpd:send_delayed_chunk(Resp1, "\r\n]}"),
+ chttpd:end_delayed_json_response(Resp2);
+ _ ->
+ {ok, Resp1}
+ end,
+ {ok, Acc#vacc{resp=Resp3, prepend=Prepend}};
view_cb(complete, #vacc{resp=Resp}=Acc) ->
% Finish view output
chttpd:send_delayed_chunk(Resp, "\r\n]}"),
@@ -223,47 +316,67 @@ get_view_keys({Props}) ->
end.
-parse_qs(Req, Keys) ->
- Args = #mrargs{keys=Keys},
+get_view_queries({Props}) ->
+ case couch_util:get_value(<<"queries">>, Props) of
+ undefined ->
+ undefined;
+ Queries when is_list(Queries) ->
+ Queries;
+ _ ->
+ throw({bad_request, "`queries` member must be a array."})
+ end.
+
+
+parse_params(#httpd{}=Req, Keys) ->
+ parse_params(couch_httpd:qs(Req), Keys);
+parse_params(Props, Keys) ->
+ Args = #mrargs{},
+ parse_params(Props, Keys, Args).
+
+
+parse_params(Props, Keys, #mrargs{}=Args0) ->
+ Args = Args0#mrargs{keys=Keys},
lists:foldl(fun({K, V}, Acc) ->
- parse_qs(K, V, Acc)
- end, Args, couch_httpd:qs(Req)).
+ parse_param(K, V, Acc)
+ end, Args, Props).
-parse_qs(Key, Val, Args) ->
+parse_param(Key, Val, Args) when is_binary(Key) ->
+ parse_param(binary_to_list(Key), Val, Args);
+parse_param(Key, Val, Args) ->
case Key of
"" ->
Args;
"reduce" ->
Args#mrargs{reduce=parse_boolean(Val)};
"key" ->
- JsonKey = ?JSON_DECODE(Val),
+ JsonKey = parse_json(Val),
Args#mrargs{start_key=JsonKey, end_key=JsonKey};
"keys" ->
- Args#mrargs{keys=?JSON_DECODE(Val)};
+ Args#mrargs{keys=parse_json(Val)};
"startkey" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"start_key" ->
- Args#mrargs{start_key=?JSON_DECODE(Val)};
+ Args#mrargs{start_key=parse_json(Val)};
"startkey_docid" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"start_key_doc_id" ->
- Args#mrargs{start_key_docid=list_to_binary(Val)};
+ Args#mrargs{start_key_docid=couch_util:to_binary(Val)};
"endkey" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"end_key" ->
- Args#mrargs{end_key=?JSON_DECODE(Val)};
+ Args#mrargs{end_key=parse_json(Val)};
"endkey_docid" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"end_key_doc_id" ->
- Args#mrargs{end_key_docid=list_to_binary(Val)};
+ Args#mrargs{end_key_docid=couch_util:to_binary(Val)};
"limit" ->
Args#mrargs{limit=parse_pos_int(Val)};
"count" ->
throw({query_parse_error, <<"QS param `count` is not `limit`">>});
- "stale" when Val == "ok" ->
+ "stale" when Val == "ok" orelse Val == <<"ok">> ->
Args#mrargs{stale=ok};
- "stale" when Val == "update_after" ->
+ "stale" when Val == "update_after" orelse Val == "update_after" ->
Args#mrargs{stale=update_after};
"stale" ->
throw({query_parse_error, <<"Invalid value for `stale`.">>});
@@ -290,12 +403,12 @@ parse_qs(Key, Val, Args) ->
"conflicts" ->
Args#mrargs{conflicts=parse_boolean(Val)};
"list" ->
- Args#mrargs{list=list_to_binary(Val)};
+ Args#mrargs{list=couch_util:to_binary(Val)};
"callback" ->
- Args#mrargs{callback=list_to_binary(Val)};
+ Args#mrargs{callback=couch_util:to_binary(Val)};
_ ->
- BKey = list_to_binary(Key),
- BVal = list_to_binary(Val),
+ BKey = couch_util:to_binary(Key),
+ BVal = couch_util:to_binary(Val),
Args#mrargs{extra=[{BKey, BVal} | Args#mrargs.extra]}
end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/f980f250/src/couch_mrview_show.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_show.erl b/src/couch_mrview_show.erl
index 1be96d5..0b42da5 100644
--- a/src/couch_mrview_show.erl
+++ b/src/couch_mrview_show.erl
@@ -193,7 +193,7 @@ handle_view_list_req(Req, _Db, _DDoc) ->
handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
- Args0 = couch_mrview_http:parse_qs(Req, Keys),
+ Args0 = couch_mrview_http:parse_params(Req, Keys),
ETagFun = fun(BaseSig, Acc0) ->
UserCtx = Req#httpd.user_ctx,
Name = UserCtx#user_ctx.name,