You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by jc...@apache.org on 2009/05/05 00:06:01 UTC
svn commit: r771466 - in /couchdb/trunk: etc/couchdb/ share/server/
share/www/script/test/ src/couchdb/
Author: jchris
Date: Mon May 4 22:06:01 2009
New Revision: 771466
URL: http://svn.apache.org/viewvc?rev=771466&view=rev
Log:
reduce_limit error is thrown when the reduce function output is not small enough compared to the input. Errors can be switched off using the config API.
Modified:
couchdb/trunk/etc/couchdb/default.ini.tpl.in
couchdb/trunk/share/server/loop.js
couchdb/trunk/share/server/state.js
couchdb/trunk/share/server/views.js
couchdb/trunk/share/www/script/test/design_docs.js
couchdb/trunk/share/www/script/test/view_errors.js
couchdb/trunk/src/couchdb/couch_query_servers.erl
couchdb/trunk/src/couchdb/couch_view_group.erl
Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Mon May 4 22:06:01 2009
@@ -26,6 +26,9 @@
[query_servers]
javascript = %bindir%/%couchjs_command_name% %localdatadir%/server/main.js
+[query_server_config]
+reduce_limit = true
+
; enable external as an httpd handler, then link it with commands here.
; note, this api is still under consideration.
; [external]
Modified: couchdb/trunk/share/server/loop.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/loop.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/loop.js (original)
+++ couchdb/trunk/share/server/loop.js Mon May 4 22:06:01 2009
@@ -28,7 +28,7 @@
//
// Responses are json values followed by a new line ("\n")
-var cmd, cmdkey;
+var line, cmd, cmdkey;
var dispatch = {
"reset" : State.reset,
@@ -43,7 +43,9 @@
"list_tail" : Render.listTail
};
-while (cmd = eval(readline())) {
+while (line = eval(readline())) {
+ cmd = eval(line)
+ line_length = line.length
try {
cmdkey = cmd.shift();
if (dispatch[cmdkey]) {
Modified: couchdb/trunk/share/server/state.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/state.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/state.js (original)
+++ couchdb/trunk/share/server/state.js Mon May 4 22:06:01 2009
@@ -13,12 +13,14 @@
// globals used by other modules and functions
var funs = []; // holds functions used for computation
var funsrc = []; // holds function source for debug info
+var query_config = {};
var State = (function() {
return {
- reset : function() {
+ reset : function(config) {
// clear the globals and run gc
funs = [];
funsrc = [];
+ query_config = config;
gc();
print("true"); // indicates success
},
Modified: couchdb/trunk/share/server/views.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/views.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/views.js (original)
+++ couchdb/trunk/share/server/views.js Mon May 4 22:06:01 2009
@@ -46,7 +46,17 @@
reductions[i] = null;
}
}
- print("[true," + toJSON(reductions) + "]");
+ var reduce_line = toJSON(reductions);
+ var reduce_length = reduce_line.length;
+ if (query_config && query_config.reduce_limit &&
+ reduce_length > 200 && ((reduce_length * 2) > line.length)) {
+ throw {
+ error:"reduce_overflow_error",
+ reason: "Reduce output must shrink more rapidly. Current output: '"+reduce_line+"'"
+ };
+ } else {
+ print("[true," + reduce_line + "]");
+ }
};
return {
Modified: couchdb/trunk/share/www/script/test/design_docs.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/design_docs.js (original)
+++ couchdb/trunk/share/www/script/test/design_docs.js Mon May 4 22:06:01 2009
@@ -16,6 +16,12 @@
db.createDb();
if (debug) debugger;
+ run_on_modified_server(
+ [{section: "query_server_config",
+ key: "reduce_limit",
+ value: "false"}],
+function() {
+
var numDocs = 500;
function makebigstring(power) {
@@ -104,4 +110,5 @@
restartServer();
T(db.open(designDoc._id) == null);
T(db.view("test/no_docs") == null);
+});
};
Modified: couchdb/trunk/share/www/script/test/view_errors.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/view_errors.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/view_errors.js (original)
+++ couchdb/trunk/share/www/script/test/view_errors.js Mon May 4 22:06:01 2009
@@ -75,13 +75,26 @@
_id:"_design/test",
language: "javascript",
views: {
- no_reduce: {map:"function(doc) {emit(doc._id, null);}"},
- with_reduce: {map:"function (doc) {emit(doc.integer, doc.integer)};",
- reduce:"function (keys, values) { return sum(values); };"},
+ "no_reduce": {map:"function(doc) {emit(doc._id, null);}"},
+ "with_reduce": {
+ map:"function (doc) {emit(doc.integer, doc.integer)};",
+ reduce:"function (keys, values) { return sum(values); };"}
}
};
T(db.save(designDoc).ok);
+ var designDoc2 = {
+ _id:"_design/testbig",
+ language: "javascript",
+ views: {
+ "reduce_too_big" : {
+ map:"function (doc) {emit(doc.integer, doc.integer)};",
+ reduce:"function (keys, values) { var chars = []; for (var i=0; i < 1000; i++) {chars.push('wazzap');};return chars; };"}
+ }
+ };
+ T(db.save(designDoc2).ok);
+
+
try {
db.view("test/no_reduce", {group: true});
T(0 == 1);
@@ -117,4 +130,11 @@
result = JSON.parse(xhr.responseText);
T(result.error == "bad_request");
T(result.reason == "`keys` member must be a array.");
+
+ // if the reduce grows to fast, throw an overflow error
+ var path = "/test_suite_db/_design/testbig/_view/reduce_too_big";
+ xhr = CouchDB.request("GET", path);
+ T(xhr.status == 500);
+ result = JSON.parse(xhr.responseText);
+ T(result.error == "reduce_overflow_error");
};
Modified: couchdb/trunk/src/couchdb/couch_query_servers.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_query_servers.erl?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_query_servers.erl (original)
+++ couchdb/trunk/src/couchdb/couch_query_servers.erl Mon May 4 22:06:01 2009
@@ -85,23 +85,27 @@
rereduce(Lang, RedSrcs, ReducedValues) ->
Pid = get_os_process(Lang),
Grouped = group_reductions_results(ReducedValues),
- Results = lists:zipwith(
+ Results = try lists:zipwith(
fun(FunSrc, Values) ->
[true, [Result]] =
couch_os_process:prompt(Pid, [<<"rereduce">>, [FunSrc], Values]),
Result
- end, RedSrcs, Grouped),
-
- ok = ret_os_process(Lang, Pid),
+ end, RedSrcs, Grouped)
+ after
+ ok = ret_os_process(Lang, Pid)
+ end,
{ok, Results}.
reduce(_Lang, [], _KVs) ->
{ok, []};
reduce(Lang, RedSrcs, KVs) ->
Pid = get_os_process(Lang),
- [true, Results] = couch_os_process:prompt(Pid,
- [<<"reduce">>, RedSrcs, KVs]),
- ok = ret_os_process(Lang, Pid),
+ Results = try couch_os_process:prompt(Pid,
+ [<<"reduce">>, RedSrcs, KVs]) of
+ [true, Reductions] -> Reductions
+ after
+ ok = ret_os_process(Lang, Pid)
+ end,
{ok, Results}.
validate_doc_update(Lang, FunSrc, EditDoc, DiskDoc, Ctx) ->
@@ -209,7 +213,8 @@
add_value(PidLangs, Pid, Lang),
rem_from_list(Pids, Lang, Pid),
add_to_list(InUse, Lang, Pid),
- true = couch_os_process:prompt(Pid, [<<"reset">>]),
+ QueryConfig = get_query_server_config(),
+ true = couch_os_process:prompt(Pid, [<<"reset">>, QueryConfig]),
{reply, Pid, Server};
_ ->
{ok, Pid} = new_process(Langs, Lang),
@@ -249,6 +254,11 @@
% Private API
+get_query_server_config() ->
+ ReduceLimit = list_to_atom(
+ couch_config:get("query_server_config","reduce_limit","true")),
+ {[{<<"reduce_limit">>, ReduceLimit}]}.
+
new_process(Langs, Lang) ->
Proc =
case ets:lookup(Langs, Lang) of
Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Mon May 4 22:06:01 2009
@@ -42,9 +42,9 @@
{ok, Group, RefCounter} ->
couch_ref_counter:add(RefCounter),
{ok, Group};
- Else ->
- ?LOG_DEBUG("get_updated_group replied with _Else ~p", [Else]),
- Else
+ Error ->
+ ?LOG_DEBUG("request_group Error ~p", [Error]),
+ throw(Error)
end.
@@ -261,6 +261,10 @@
handle_info({'EXIT', _FromPid, normal}, State) ->
{noreply, State};
+handle_info({'EXIT', FromPid, {{nocatch, Reason}, Trace}}, State) ->
+ ?LOG_DEBUG("Uncaught throw() in linked pid: ~p", [{FromPid, Reason}]),
+ {stop, Reason, State};
+
handle_info({'EXIT', FromPid, Reason}, State) ->
?LOG_DEBUG("Exit from linked pid: ~p", [{FromPid, Reason}]),
{stop, Reason, State};