You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2017/10/09 19:42:09 UTC
[couchdb] branch improve-reduce-limit-errors updated (8d33cd7 ->
b030a86)
This is an automated email from the ASF dual-hosted git repository.
jan pushed a change to branch improve-reduce-limit-errors
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
omit 8d33cd7 Return reduce overflow errors to the client
add 1091b5a Implement attachment size limits
add aa14e5d Query operator tests for multiple index types (#800)
add 04e4cfa Avoid duplicate index selection in Mango
add bc43efb Use unittest assert in index selection tests
add 41e2984 Use unittest assertions in mango index tests
add 302126b Return 400 when no index can fulfil a sort
add cf00dc2 Log unhandled mango errors
add ef8a934 Do not crash when free space cannot be calculated (#803)
add d9e2940 Provide a more accurate size check for max_document_size limit
add c622e17 Don't reset logs when JS tests restart the server
add ed6ec66 Add selector support for json indexes (#808)
add 369b442 Catch invalid base64 in inline attachments
add 190ee30 Merge pull request #817 from cloudant/issue-784-fix-invalid-base64-att-crash
add e590cad Support setting cookie domain for auth
add cacc774 Add unit tests for cookie domain
add c531a13 Fix replication ID parsing in URL paths
add dae81be Fix json index selection (#816)
add e474da0 Run mango tests with make check (#786)
add 00df0de Rename selector to partialfilterselector in indexes (#818)
add 8d1c704 Avoid decompressing just to calculate external size
add 1eaf178 Do not buffer rexi messages to disconnected nodes
add 3b8b9a3 Make stats interval into config parameter (#830)
add 32ccd1b Add cluster info to db_info (#837)
add 7267f92 Reduce replicator.retries_per_request value from 10 to 5
add 7c3cf50 Reorganize exports from couch_db.erl
add 20a1021 Move calculate_start_seq and owner_of
add 35fcd7e Update couch_server to not use the db record
add 98eda03 Add a test helper for creating fake db records
add 5530204 Avoid bad match on really old databases
add aee57bf Remove public access to the db record
add cb610bf Add clause for mixed cluster upgrades
add d5a261b Allow for mixed db record definitions
add c07f3f5 Handle attachments downgrades in a mixed cluster environment
add 355f0fc fix tests to be compatible for both python2 and python3 (#839)
add 30fcd7b Add convenience remsh bash script
add 2684561 Clean up replicator logs
add b756b77 Replace replication start multi-line log statement
add 3e6432c Remove bashisms in remsh script
add 7c49f25 fix remsh refactoring bug
add d04038a Merge pull request #847 from apache/add-remsh
add 9751b06 Update meck to latest version 0.8.8
add 796e1b0 Merge branch 'master' into master
add 0eb7677 Support setting cookie domain for AuthSession cookie
add 262c527 add spidermonkey to README-DEV instructions for mac
add 8b8de97 add sphinx_rtd_theme
add 59effcf Merge pull request #853 from apache/fix/docs-mac-spidermonkey
add 6bff5fb Replace deprecated crypto:rand_bytes call
add a84f41a Whitelist system DB names as valid _dbs docids
add 6fed9cd Merge pull request #862 from apache/858-whitelist-shard-map-docids
add 4963f66 Correct result count in Mango execution stats (#867)
add 75984da Handle deprecated random module
add 405c80a Bump khash, b64, ioq deps
add 4c61d10 Fix attachments tests for 20.0 compatibility
add c26ce7f Unit tests now pass on 20.0. Eenable it in rebar.config and in Travis
new b030a86 Return reduce overflow errors to the client
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8d33cd7)
\
N -- N -- N refs/heads/improve-reduce-limit-errors (b030a86)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 3 +-
Makefile | 1 +
README-DEV.rst | 35 ++-
configure | 2 +-
dev/run | 8 +-
rebar.config.script | 12 +-
rel/overlay/bin/remsh | 76 +++++
rel/overlay/etc/default.ini | 22 +-
src/chttpd/src/chttpd.erl | 2 +
src/chttpd/src/chttpd_db.erl | 27 +-
src/chttpd/src/chttpd_external.erl | 22 +-
src/chttpd/src/chttpd_show.erl | 3 +-
.../test/chttpd_db_attachment_size_tests.erl | 206 ++++++++++++++
src/chttpd/test/chttpd_db_doc_size_tests.erl | 2 +-
src/couch/include/couch_db.hrl | 34 +--
src/couch/rebar.config.script | 11 +-
src/couch/src/couch_att.erl | 93 ++++++-
src/couch/src/couch_auth_cache.erl | 14 +-
src/couch/src/couch_changes.erl | 56 ++--
src/couch/src/couch_compaction_daemon.erl | 48 ++--
src/couch/src/couch_compress.erl | 14 +
src/couch/src/couch_db.erl | 307 +++++++++++++++++++--
src/couch/src/couch_db_int.hrl | 93 +++++++
src/couch/src/couch_db_plugin.erl | 6 +-
src/couch/src/couch_db_updater.erl | 10 +-
src/couch/src/couch_debug.erl | 2 +-
src/couch/src/couch_doc.erl | 122 +++++---
src/couch/src/couch_ejson_size.erl | 99 +++++++
src/couch/src/couch_httpd.erl | 4 +
src/couch/src/couch_httpd_auth.erl | 9 +-
src/couch/src/couch_httpd_db.erl | 48 ++--
src/couch/src/couch_lru.erl | 9 +-
src/couch/src/couch_multidb_changes.erl | 2 +-
.../src/couch_rand.erl} | 53 ++--
src/couch/src/couch_server.erl | 135 ++++-----
.../rexi.hrl => couch/src/couch_server_int.hrl} | 17 +-
src/couch/src/couch_users_db.erl | 8 +-
src/couch/src/couch_util.erl | 18 +-
src/couch/src/test_util.erl | 14 +
src/couch/test/couch_auth_cache_tests.erl | 2 +-
src/couch/test/couch_btree_tests.erl | 8 +-
src/couch/test/couch_changes_tests.erl | 2 +-
src/couch/test/couch_compress_tests.erl | 11 +
src/couch/test/couch_db_plugin_tests.erl | 13 +-
src/couch/test/couch_doc_json_tests.erl | 71 +++++
src/couch/test/couch_doc_tests.erl | 17 +-
src/couch/test/couch_ejson_size_tests.erl | 72 +++++
src/couch/test/couch_file_tests.erl | 4 +-
src/couch/test/couch_server_tests.erl | 11 +-
src/couch/test/couchdb_attachments_tests.erl | 136 ++++++++-
src/couch/test/couchdb_compaction_daemon_tests.erl | 4 +-
src/couch/test/couchdb_cookie_domain_tests.erl | 77 ++++++
src/couch/test/couchdb_file_compression_tests.erl | 2 +-
src/couch/test/couchdb_views_tests.erl | 25 +-
src/couch_index/src/couch_index_server.erl | 15 +-
src/couch_index/src/couch_index_util.erl | 2 +-
.../test/couch_index_compaction_tests.erl | 3 +-
.../test/couch_index_ddoc_updated_tests.erl | 2 +-
src/couch_log/test/couch_log_test.erl | 3 +-
src/couch_mrview/src/couch_mrview.erl | 18 +-
src/couch_mrview/src/couch_mrview_compactor.erl | 3 +-
src/couch_mrview/src/couch_mrview_http.erl | 10 +-
src/couch_mrview/src/couch_mrview_show.erl | 16 +-
.../test/couch_mrview_all_docs_tests.erl | 2 +-
.../test/couch_mrview_changes_since_tests.erl | 2 +-
.../test/couch_mrview_collation_tests.erl | 2 +-
.../test/couch_mrview_compact_tests.erl | 2 +-
.../test/couch_mrview_ddoc_updated_tests.erl | 2 +-
.../test/couch_mrview_ddoc_validation_tests.erl | 2 +-
.../test/couch_mrview_index_changes_tests.erl | 2 +-
.../test/couch_mrview_index_info_tests.erl | 2 +-
.../test/couch_mrview_local_docs_tests.erl | 2 +-
.../test/couch_mrview_map_views_tests.erl | 2 +-
.../test/couch_mrview_red_views_tests.erl | 2 +-
.../src/couch_replicator_api_wrap.erl | 31 ++-
.../src/couch_replicator_api_wrap.hrl | 2 +-
.../src/couch_replicator_clustering.erl | 116 +++-----
.../src/couch_replicator_doc_processor.erl | 14 +-
src/couch_replicator/src/couch_replicator_docs.erl | 21 +-
src/couch_replicator/src/couch_replicator_ids.erl | 16 +-
.../src/couch_replicator_scheduler_job.erl | 203 +++++++++++---
.../src/couch_replicator_utils.erl | 37 ++-
.../src/couch_replicator_worker.erl | 56 ++--
... => couch_replicator_attachments_too_large.erl} | 36 ++-
.../test/couch_replicator_compact_tests.erl | 27 +-
src/couch_stats/src/couch_stats.app.src | 4 +-
src/couch_stats/src/couch_stats.erl | 6 +-
.../src/couch_stats.hrl} | 5 +-
src/couch_stats/src/couch_stats_aggregator.erl | 11 +-
src/ddoc_cache/test/ddoc_cache_lru_test.erl | 2 +-
src/ddoc_cache/test/ddoc_cache_refresh_test.erl | 4 +-
src/fabric/include/couch_db_tmp.hrl | 296 --------------------
src/fabric/rebar.config | 2 +-
src/fabric/src/fabric.erl | 12 +-
src/fabric/src/fabric_db_info.erl | 48 +++-
src/fabric/src/fabric_doc_attachments.erl | 28 +-
src/fabric/src/fabric_doc_update.erl | 4 +-
src/fabric/src/fabric_rpc.erl | 123 ++-------
src/fabric/src/fabric_util.erl | 3 +-
src/mango/src/mango_crud.erl | 2 +-
src/mango/src/mango_cursor.erl | 31 ++-
src/mango/src/mango_cursor_text.erl | 7 +-
src/mango/src/mango_cursor_view.erl | 13 +-
src/mango/src/mango_error.erl | 6 +-
src/mango/src/mango_httpd.erl | 33 +--
src/mango/src/mango_idx.erl | 74 ++++-
src/mango/src/mango_idx_text.erl | 17 +-
src/mango/src/mango_idx_view.erl | 17 +-
src/mango/src/mango_native_proc.erl | 49 +++-
src/mango/src/mango_selector.erl | 110 +++++++-
src/mango/test/01-index-crud-test.py | 21 +-
src/mango/test/02-basic-find-test.py | 14 +-
src/mango/test/03-operator-test.py | 78 +++++-
src/mango/test/05-index-selection-test.py | 116 ++++++--
src/mango/test/06-basic-text-test.py | 3 -
src/mango/test/07-text-custom-field-list-test.py | 4 +-
src/mango/test/08-text-limit-test.py | 6 +-
src/mango/test/09-text-sort-test.py | 4 +-
src/mango/test/12-use-correct-index.py | 19 +-
src/mango/test/14-json-pagination.py | 4 +-
src/mango/test/15-execution-stats-test.py | 4 +
src/mango/test/16-index-selectors.py | 237 ++++++++++++++++
src/mango/test/mango.py | 29 +-
src/mango/test/user_docs.py | 29 +-
src/mem3/src/mem3.erl | 17 +-
src/mem3/src/mem3_cluster.erl | 161 +++++++++++
src/mem3/src/mem3_httpd.erl | 4 +-
src/mem3/src/mem3_nodes.erl | 10 +-
src/mem3/src/mem3_rep.erl | 15 +-
src/mem3/src/mem3_rpc.erl | 4 +-
src/mem3/src/mem3_shards.erl | 10 +-
src/mem3/test/mem3_cluster_test.erl | 133 +++++++++
src/rexi/src/rexi_server_mon.erl | 84 ++++--
test/javascript/tests/attachments.js | 18 ++
134 files changed, 3414 insertions(+), 1272 deletions(-)
create mode 100755 rel/overlay/bin/remsh
create mode 100644 src/chttpd/test/chttpd_db_attachment_size_tests.erl
create mode 100644 src/couch/src/couch_db_int.hrl
create mode 100644 src/couch/src/couch_ejson_size.erl
copy src/{ddoc_cache/src/ddoc_cache_entry_validation_funs.erl => couch/src/couch_rand.erl} (51%)
copy src/{rexi/include/rexi.hrl => couch/src/couch_server_int.hrl} (83%)
create mode 100644 src/couch/test/couch_ejson_size_tests.erl
create mode 100755 src/couch/test/couchdb_cookie_domain_tests.erl
copy src/couch_replicator/test/{couch_replicator_id_too_long_tests.erl => couch_replicator_attachments_too_large.erl} (73%)
copy src/{couch_replicator/src/couch_replicator_scheduler.hrl => couch_stats/src/couch_stats.hrl} (89%)
delete mode 100644 src/fabric/include/couch_db_tmp.hrl
create mode 100644 src/mango/test/16-index-selectors.py
create mode 100644 src/mem3/src/mem3_cluster.erl
create mode 100644 src/mem3/test/mem3_cluster_test.erl
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].
[couchdb] 01/01: Return reduce overflow errors to the client
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jan pushed a commit to branch improve-reduce-limit-errors
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b030a8648214d88411191dd74f65ee4edc0ed7a2
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Sep 7 14:37:36 2017 -0500
Return reduce overflow errors to the client
This changes the reduce overflow error to return an error to the client
rather than blowing up the view build. This allows views that have a
single bad reduce to build while not crushing the server's RAM usage.
---
src/couch/src/couch_query_servers.erl | 29 ++++++++++++++++++++++++-----
src/couch_mrview/src/couch_mrview_http.erl | 6 +++++-
src/fabric/src/fabric_util.erl | 2 --
src/fabric/src/fabric_view.erl | 2 ++
test/javascript/tests/view_errors.js | 4 ++--
5 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl
index 63b0e39..4928eea 100644
--- a/src/couch/src/couch_query_servers.erl
+++ b/src/couch/src/couch_query_servers.erl
@@ -125,20 +125,39 @@ os_reduce(Lang, OsRedSrcs, KVs) ->
Proc = get_os_process(Lang),
OsResults = try proc_prompt(Proc, [<<"reduce">>, OsRedSrcs, KVs]) of
[true, Reductions] -> Reductions
+ catch
+ throw:{reduce_overflow_error, Msg} ->
+ [{[{reduce_overflow_error, Msg}]} || _ <- OsRedSrcs]
after
ok = ret_os_process(Proc)
end,
{ok, OsResults}.
os_rereduce(Lang, OsRedSrcs, KVs) ->
- Proc = get_os_process(Lang),
- try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of
- [true, [Reduction]] -> Reduction
- after
- ok = ret_os_process(Proc)
+ case get_overflow_error(KVs) of
+ undefined ->
+ Proc = get_os_process(Lang),
+ try proc_prompt(Proc, [<<"rereduce">>, OsRedSrcs, KVs]) of
+ [true, [Reduction]] -> Reduction
+ catch
+ throw:{reduce_overflow_error, Msg} ->
+ {[{reduce_overflow_error, Msg}]}
+ after
+ ok = ret_os_process(Proc)
+ end;
+ Error ->
+ Error
end.
+get_overflow_error([]) ->
+ undefined;
+get_overflow_error([{[{reduce_overflow_error, _}]} = Error | _]) ->
+ Error;
+get_overflow_error([_ | Rest]) ->
+ get_overflow_error(Rest).
+
+
builtin_reduce(_Re, [], _KVs, Acc) ->
{ok, lists:reverse(Acc)};
builtin_reduce(Re, [<<"_sum",_/binary>>|BuiltinReds], KVs, Acc) ->
diff --git a/src/couch_mrview/src/couch_mrview_http.erl b/src/couch_mrview/src/couch_mrview_http.erl
index 9ad50ee..004caef 100644
--- a/src/couch_mrview/src/couch_mrview_http.erl
+++ b/src/couch_mrview/src/couch_mrview_http.erl
@@ -440,7 +440,11 @@ row_to_json(error, Row) ->
% match prior behavior.
Key = couch_util:get_value(key, Row),
Val = couch_util:get_value(value, Row),
- Obj = {[{key, Key}, {error, Val}]},
+ Reason = couch_util:get_value(reason, Row),
+ ReasonProp = if Reason == undefined -> []; true ->
+ [{reason, Reason}]
+ end,
+ Obj = {[{key, Key}, {error, Val}] ++ ReasonProp},
?JSON_ENCODE(Obj);
row_to_json(Id0, Row) ->
Id = case Id0 of
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index bf3f023..49f4c89 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -203,8 +203,6 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) ->
rexi_monitor:stop(Mon)
end.
-error_info({{<<"reduce_overflow_error">>, _} = Error, _Stack}) ->
- Error;
error_info({{timeout, _} = Error, _Stack}) ->
Error;
error_info({{Error, Reason}, Stack}) ->
diff --git a/src/fabric/src/fabric_view.erl b/src/fabric/src/fabric_view.erl
index 45262e4..dd0fcfd 100644
--- a/src/fabric/src/fabric_view.erl
+++ b/src/fabric/src/fabric_view.erl
@@ -258,6 +258,8 @@ find_next_key([], _, _, _) ->
find_next_key([Key|Rest], _, _, _) ->
{Key, Rest}.
+transform_row(#view_row{value={[{reduce_overflow_error, Msg}]}}) ->
+ {row, [{key,null}, {id,error}, {value,reduce_overflow_error}, {reason,Msg}]};
transform_row(#view_row{key=Key, id=reduced, value=Value}) ->
{row, [{key,Key}, {value,Value}]};
transform_row(#view_row{key=Key, id=undefined}) ->
diff --git a/test/javascript/tests/view_errors.js b/test/javascript/tests/view_errors.js
index b53a3c7..0d9cd79 100644
--- a/test/javascript/tests/view_errors.js
+++ b/test/javascript/tests/view_errors.js
@@ -174,9 +174,9 @@ couchTests.view_errors = function(debug) {
// if the reduce grows to fast, throw an overflow error
var path = "/" + db_name + "/_design/testbig/_view/reduce_too_big";
xhr = CouchDB.request("GET", path);
- T(xhr.status == 500);
+ T(xhr.status == 200);
result = JSON.parse(xhr.responseText);
- T(result.error == "reduce_overflow_error");
+ T(result.rows[0].error == "reduce_overflow_error");
try {
db.query(function() {emit(null, null)}, null, {startkey: 2, endkey:1});
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.