You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/01/02 21:18:57 UTC
[couchdb] branch feature/database-partition-limits updated (06f4445
-> f0efadb)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch feature/database-partition-limits
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 06f4445 Enforce partition size limits
discard cb79853 Use an accumulator when merging revision trees
discard 3c8e0c6 Add Elixir tests for database partitions
discard 56c1404 Support partitioned queries in Mango
discard 739c262 Use index names when testing index selection
discard b90c6c5 Optimize offset/limit for partition queries
discard 546894f Optimize all_docs queries in a single partition
discard f7b0822 Implement partitioned views
discard 2db6577 Implement `couch_db:get_partition_info/2`
discard 50de080 Implement partitioned dbs
discard 1f569b9 Implement configurable hash functions
discard 381bb0a Validate design document options more strictly
discard 0d7e38f Pass the DB record to index validation functions
discard 9649dba Implement `fabric_util:open_cluster_db`
discard 90b5eee Improve `couch_db:clustered_db` flexibility
discard 7f9d910 Add PSE API to store opaque properties
add e97f029 Remove shim couch_replicator_manager module
add c54bea3 Fix elixir test formatting
add 90c0a6f Improve all_docs_test robustness
add add9fae Do not automatically fail tests if quorum conditions unmet
add ea20abc Merge pull request #1829 from cloudant/elixir-test-improvements
add cb10e48 Suppress variable exported from 'case' compiler warnings
add f519b26 Suppress unused variable and type compiler warnings
add b9b333b Suppress crypto and random compiler warnings
add 4cc9565 Suppress unused function compiler warnings
add f29cdb4 Suppress misc compiler warnings
add 899b49c Reduce number of behaviour undefined compiler warnings
add 4e75964 Suppress export-related compiler warnings
add 26cee91 Merge pull request #1798 from cloudant/suppress-compiler-warnings
add 7be0ab8 Remove explicit modules list from .app.src files
add c347470 Remove obsolete travis files
add 21359d2 Change minimum supported Erlang version to OTP 19
add db3b365 Merge pull request #1833 from cloudant/minimum-erlang-otp-19
add f708c05 happy new year (#1838)
add f3a2eb2 Add PSE API to store opaque properties
add ed9c97d Improve `couch_db:clustered_db` flexibility
add 9819cfa Implement `fabric_util:open_cluster_db`
add 71a1057 Pass the DB record to index validation functions
add 946b942 Validate design document options more strictly
add 01715aa Implement configurable hash functions
add 0c97d71 Implement partitioned dbs
add c6befe6 Implement `couch_db:get_partition_info/2`
add ad6b455 Implement partitioned views
add 7ec5717 Optimize all_docs queries in a single partition
add d6213aa Optimize offset/limit for partition queries
add 1c83f62 Use index names when testing index selection
add fbce2c0 Support partitioned queries in Mango
add b98a7c6 Add Elixir tests for database partitions
new f578ac2 Use an accumulator when merging revision trees
new f0efadb Enforce partition size limits
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (06f4445)
\
N -- N -- N refs/heads/feature/database-partition-limits (f0efadb)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 2 -
LICENSE | 2 +-
NOTICE | 2 +-
rebar.config.script | 6 +-
src/chttpd/.travis.yml | 23 ---
src/chttpd/src/chttpd.app.src | 13 --
src/chttpd/src/chttpd_sup.erl | 2 +-
src/couch/.travis.yml | 23 ---
src/couch/rebar.config.script | 4 -
src/couch/src/couch.erl | 6 +-
src/couch/src/couch_btree.erl | 7 +-
src/couch/src/couch_compaction_daemon.erl | 10 +-
src/couch/src/couch_db.erl | 2 +-
src/couch/src/couch_debug.erl | 2 +-
src/couch/src/couch_rand.erl | 30 ----
src/couch/src/couch_server.erl | 28 ++--
src/couch/src/couch_util.erl | 9 --
src/couch/src/couch_uuids.erl | 2 +-
src/couch/test/couch_changes_tests.erl | 176 ++++++++++-----------
src/couch/test/couch_doc_json_tests.erl | 2 +
src/couch/test/couch_file_tests.erl | 2 +-
src/couch/test/couch_flags_tests.erl | 16 +-
src/couch/test/couch_key_tree_prop_tests.erl | 1 -
src/couch/test/couch_util_tests.erl | 2 +-
src/couch/test/couchdb_mrview_tests.erl | 4 +-
src/couch/test/couchdb_vhosts_tests.erl | 4 -
src/couch/test/json_stream_parse_tests.erl | 2 +-
src/couch_epi/.travis.yml | 34 ----
src/couch_epi/src/couch_epi.erl | 3 -
src/couch_epi/src/couch_epi_sup.erl | 107 +------------
src/couch_epi/test/couch_epi_basic_test.erl | 137 ++++++++++++++++
src/couch_index/.travis.yml | 43 -----
src/couch_index/src/couch_index.app.src | 4 -
src/couch_log/.travis.yml | 43 -----
src/couch_log/src/couch_log_writer_file.erl | 4 +-
src/couch_log/src/couch_log_writer_syslog.erl | 5 +-
src/couch_log/test/couch_log_test_util.erl | 11 +-
src/couch_mrview/.travis.yml | 23 ---
src/couch_mrview/src/couch_mrview.app.src | 10 --
src/couch_mrview/src/couch_mrview_test_util.erl | 1 +
.../test/couch_mrview_index_changes_tests.erl | 4 +-
src/couch_pse_tests/src/cpse_test_attachments.erl | 1 +
src/couch_pse_tests/src/cpse_test_compaction.erl | 1 +
src/couch_pse_tests/src/cpse_test_fold_changes.erl | 17 +-
src/couch_pse_tests/src/cpse_test_fold_docs.erl | 1 +
.../src/cpse_test_fold_purge_infos.erl | 1 +
.../src/cpse_test_get_set_props.erl | 1 +
.../src/cpse_test_open_close_delete.erl | 1 +
src/couch_pse_tests/src/cpse_test_purge_docs.erl | 1 +
src/couch_pse_tests/src/cpse_test_purge_seqs.erl | 1 +
.../src/cpse_test_read_write_docs.erl | 1 +
src/couch_pse_tests/src/cpse_test_ref_counting.erl | 1 +
src/couch_pse_tests/src/cpse_util.erl | 1 +
src/couch_replicator/.travis.yml | 44 ------
.../src/couch_replicator_manager.erl | 29 ----
.../test/couch_replicator_compact_tests.erl | 2 +-
.../test/couch_replicator_connection_tests.erl | 2 +-
.../test/couch_replicator_modules_load_tests.erl | 45 ------
src/couch_tests/test/couch_tests_app_tests.erl | 2 +-
src/ddoc_cache/test/ddoc_cache_tutil.erl | 11 +-
src/fabric/.travis.yml | 23 ---
src/fabric/src/fabric.app.src | 23 ---
src/fabric/src/fabric_dict.erl | 1 +
src/fabric/src/fabric_doc_update.erl | 2 +-
src/global_changes/.travis.yml | 23 ---
src/mango/.travis.yml | 29 ----
src/mem3/src/mem3.app.src | 14 --
src/mem3/src/mem3.erl | 14 +-
src/mem3/src/mem3_rep.erl | 1 -
src/mem3/src/mem3_sync_event_listener.erl | 2 +-
src/mem3/test/mem3_seeds_test.erl | 13 +-
src/rexi/src/rexi.app.src | 10 --
test/elixir/lib/couch/db_test.ex | 6 +-
test/elixir/test/all_docs_test.exs | 40 +++--
test/elixir/test/cluster_with_quorum_test.exs | 18 ++-
test/elixir/test/cluster_without_quorum_test.exs | 18 ++-
test/elixir/test/rewrite_test.exs | 6 +-
test/elixir/test/security_validation_test.exs | 6 +-
78 files changed, 411 insertions(+), 812 deletions(-)
delete mode 100644 src/chttpd/.travis.yml
delete mode 100644 src/couch/.travis.yml
delete mode 100644 src/couch_epi/.travis.yml
create mode 100644 src/couch_epi/test/couch_epi_basic_test.erl
delete mode 100644 src/couch_index/.travis.yml
delete mode 100644 src/couch_log/.travis.yml
delete mode 100644 src/couch_mrview/.travis.yml
delete mode 100644 src/couch_replicator/.travis.yml
delete mode 100644 src/couch_replicator/src/couch_replicator_manager.erl
delete mode 100644 src/couch_replicator/test/couch_replicator_modules_load_tests.erl
delete mode 100644 src/fabric/.travis.yml
delete mode 100644 src/global_changes/.travis.yml
delete mode 100644 src/mango/.travis.yml
[couchdb] 02/02: Enforce partition size limits
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature/database-partition-limits
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f0efadb031f0211d192627344848a88a2acdf053
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 14 11:06:03 2018 -0600
Enforce partition size limits
This limit helps prevent users from inadvertently misusing partitions by
refusing to add documents when the size of a partition exceeds 10GiB.
Co-authored-by: Robert Newson <rn...@apache.org>
---
rel/overlay/etc/default.ini | 5 +++
src/chttpd/src/chttpd.erl | 3 ++
src/couch/src/couch_db_updater.erl | 81 ++++++++++++++++++++++++++++++++++++--
3 files changed, 85 insertions(+), 4 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index a77add4..ae9d313 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -64,6 +64,11 @@ default_engine = couch
; move deleted databases/shards there instead. You can then manually delete
; these files later, as desired.
;enable_database_recovery = false
+;
+; Set the maximum size allowed for a partition. This helps users avoid
+; inadvertently abusing partitions resulting in hot shards. The default
+; is 10GiB. A value of 0 or less will disable partition size checks.
+;max_partition_size = 10737418240
[couchdb_engines]
; The keys in this section are the filename extension that
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 2f241cd..6558b1e 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -873,6 +873,9 @@ error_info(conflict) ->
{409, <<"conflict">>, <<"Document update conflict.">>};
error_info({conflict, _}) ->
{409, <<"conflict">>, <<"Document update conflict.">>};
+error_info({partition_overflow, DocId}) ->
+ Descr = <<"'", DocId/binary, "' exceeds partition limit">>,
+ {403, <<"partition_overflow">>, Descr};
error_info({{not_found, missing}, {_, _}}) ->
{409, <<"not_found">>, <<"missing_rev">>};
error_info({forbidden, Error, Msg}) ->
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 95508e2..00fee90 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -21,6 +21,7 @@
-include("couch_db_int.hrl").
-define(IDLE_LIMIT_DEFAULT, 61000).
+-define(DEFAULT_MAX_PARTITION_SIZE, 16#280000000). % 10 GiB
-record(merge_acc, {
@@ -28,7 +29,8 @@
merge_conflicts,
add_infos = [],
rem_seqs = [],
- cur_seq
+ cur_seq,
+ full_partitions = []
}).
@@ -466,13 +468,22 @@ merge_rev_trees([], [], Acc) ->
merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) ->
#merge_acc{
revs_limit = Limit,
- merge_conflicts = MergeConflicts
+ merge_conflicts = MergeConflicts,
+ full_partitions = FullPartitions
} = Acc,
% Track doc ids so we can debug large revision trees
erlang:put(last_id_merged, OldDocInfo#full_doc_info.id),
NewDocInfo0 = lists:foldl(fun({Client, NewDoc}, OldInfoAcc) ->
- merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts)
+ NewInfo = merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts),
+ case is_overflowed(NewInfo, OldInfoAcc, FullPartitions) of
+ true when not MergeConflicts ->
+ DocId = NewInfo#doc.id,
+ send_result(Client, NewDoc, {partition_overflow, DocId}),
+ OldInfoAcc;
+ false ->
+ NewInfo
+ end
end, OldDocInfo, NewDocs),
NewDocInfo1 = maybe_stem_full_doc_info(NewDocInfo0, Limit),
% When MergeConflicts is false, we updated #full_doc_info.deleted on every
@@ -595,6 +606,16 @@ merge_rev_tree(OldInfo, NewDoc, _Client, true) ->
{NewTree, _} = couch_key_tree:merge(OldTree, NewTree0),
OldInfo#full_doc_info{rev_tree = NewTree}.
+is_overflowed(_New, _Old, []) ->
+ false;
+is_overflowed(Old, Old, _FullPartitions) ->
+ false;
+is_overflowed(New, Old, FullPartitions) ->
+ Partition = couch_partition:from_docid(New#full_doc_info.id),
+ NewSize = estimate_size(New),
+ OldSize = estimate_size(Old),
+ lists:member(Partition, FullPartitions) andalso NewSize > OldSize.
+
maybe_stem_full_doc_info(#full_doc_info{rev_tree = Tree} = Info, Limit) ->
case config:get_boolean("couchdb", "stem_interactive_updates", true) of
true ->
@@ -617,13 +638,31 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts, FullCommit) ->
(Id, not_found) ->
#full_doc_info{id=Id}
end, Ids, OldDocLookups),
+
+ %% Get the list of full partitions
+ FullPartitions = case couch_db:is_partitioned(Db) of
+ true ->
+ case max_partition_size() of
+ N when N =< 0 ->
+ [];
+ Max ->
+ Partitions = lists:usort(lists:map(fun(Id) ->
+ couch_partition:from_docid(Id)
+ end, Ids)),
+ [P || P <- Partitions, partition_size(Db, P) >= Max]
+ end;
+ false ->
+ []
+ end,
+
% Merge the new docs into the revision trees.
AccIn = #merge_acc{
revs_limit = RevsLimit,
merge_conflicts = MergeConflicts,
add_infos = [],
rem_seqs = [],
- cur_seq = UpdateSeq
+ cur_seq = UpdateSeq,
+ full_partitions = FullPartitions
},
{ok, AccOut} = merge_rev_trees(DocsList, OldDocInfos, AccIn),
#merge_acc{
@@ -685,6 +724,40 @@ increment_local_doc_revs(#doc{revs = {0, [RevStr | _]}} = Doc) ->
increment_local_doc_revs(#doc{}) ->
{error, <<"Invalid rev format">>}.
+max_partition_size() ->
+ config:get_integer("couchdb", "max_partition_size",
+ ?DEFAULT_MAX_PARTITION_SIZE).
+
+partition_size(Db, Partition) ->
+ {ok, Info} = couch_db:get_partition_info(Db, Partition),
+ Sizes = couch_util:get_value(sizes, Info),
+ couch_util:get_value(external, Sizes).
+
+estimate_size(#full_doc_info{} = FDI) ->
+ #full_doc_info{rev_tree = RevTree} = FDI,
+ Fun = fun
+ (_Rev, Value, leaf, SizesAcc) ->
+ case Value of
+ #doc{} = Doc ->
+ ExternalSize = get_meta_body_size(Value#doc.meta),
+ {size_info, AttSizeInfo} =
+ lists:keyfind(size_info, 1, Doc#doc.meta),
+ Leaf = #leaf{
+ sizes = #size_info{
+ external = ExternalSize
+ },
+ atts = AttSizeInfo
+ },
+ add_sizes(leaf, Leaf, SizesAcc);
+ #leaf{} ->
+ add_sizes(leaf, Value, SizesAcc)
+ end;
+ (_Rev, _Value, branch, SizesAcc) ->
+ SizesAcc
+ end,
+ {_, FinalES, FinalAtts} = couch_key_tree:fold(Fun, {0, 0, []}, RevTree),
+ TotalAttSize = lists:foldl(fun({_, S}, A) -> S + A end, 0, FinalAtts),
+ FinalES + TotalAttSize.
purge_docs(Db, []) ->
{ok, Db, []};
[couchdb] 01/02: Use an accumulator when merging revision trees
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature/database-partition-limits
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f578ac211559642b642141df67270e5ce6c67adb
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Dec 14 10:31:02 2018 -0600
Use an accumulator when merging revision trees
This cleans up the `couch_db_updater:merge_rev_trees/7` to instead use
an accumulator argument.
---
src/couch/src/couch_db_updater.erl | 57 ++++++++++++++++++++++++++++----------
1 file changed, 43 insertions(+), 14 deletions(-)
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index c0974aa..95508e2 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -23,6 +23,15 @@
-define(IDLE_LIMIT_DEFAULT, 61000).
+-record(merge_acc, {
+ revs_limit,
+ merge_conflicts,
+ add_infos = [],
+ rem_seqs = [],
+ cur_seq
+}).
+
+
init({Engine, DbName, FilePath, Options0}) ->
erlang:put(io_priority, {db_update, DbName}),
update_idle_limit_from_config(),
@@ -450,11 +459,18 @@ doc_tag(#doc{meta=Meta}) ->
Else -> throw({invalid_doc_tag, Else})
end.
-merge_rev_trees(_Limit, _Merge, [], [], AccNewInfos, AccRemoveSeqs, AccSeq) ->
- {ok, lists:reverse(AccNewInfos), AccRemoveSeqs, AccSeq};
-merge_rev_trees(Limit, MergeConflicts, [NewDocs|RestDocsList],
- [OldDocInfo|RestOldInfo], AccNewInfos, AccRemoveSeqs, AccSeq) ->
- erlang:put(last_id_merged, OldDocInfo#full_doc_info.id), % for debugging
+merge_rev_trees([], [], Acc) ->
+ {ok, Acc#merge_acc{
+ add_infos = lists:reverse(Acc#merge_acc.add_infos)
+ }};
+merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo | RestOldInfo], Acc) ->
+ #merge_acc{
+ revs_limit = Limit,
+ merge_conflicts = MergeConflicts
+ } = Acc,
+
+ % Track doc ids so we can debug large revision trees
+ erlang:put(last_id_merged, OldDocInfo#full_doc_info.id),
NewDocInfo0 = lists:foldl(fun({Client, NewDoc}, OldInfoAcc) ->
merge_rev_tree(OldInfoAcc, NewDoc, Client, MergeConflicts)
end, OldDocInfo, NewDocs),
@@ -475,22 +491,25 @@ merge_rev_trees(Limit, MergeConflicts, [NewDocs|RestDocsList],
end,
if NewDocInfo2 == OldDocInfo ->
% nothing changed
- merge_rev_trees(Limit, MergeConflicts, RestDocsList, RestOldInfo,
- AccNewInfos, AccRemoveSeqs, AccSeq);
+ merge_rev_trees(RestDocsList, RestOldInfo, Acc);
true ->
% We have updated the document, give it a new update_seq. Its
% important to note that the update_seq on OldDocInfo should
% be identical to the value on NewDocInfo1.
OldSeq = OldDocInfo#full_doc_info.update_seq,
NewDocInfo3 = NewDocInfo2#full_doc_info{
- update_seq = AccSeq + 1
+ update_seq = Acc#merge_acc.cur_seq + 1
},
RemoveSeqs = case OldSeq of
- 0 -> AccRemoveSeqs;
- _ -> [OldSeq | AccRemoveSeqs]
+ 0 -> Acc#merge_acc.rem_seqs;
+ _ -> [OldSeq | Acc#merge_acc.rem_seqs]
end,
- merge_rev_trees(Limit, MergeConflicts, RestDocsList, RestOldInfo,
- [NewDocInfo3|AccNewInfos], RemoveSeqs, AccSeq+1)
+ NewAcc = Acc#merge_acc{
+ add_infos = [NewDocInfo3 | Acc#merge_acc.add_infos],
+ rem_seqs = RemoveSeqs,
+ cur_seq = Acc#merge_acc.cur_seq + 1
+ },
+ merge_rev_trees(RestDocsList, RestOldInfo, NewAcc)
end.
merge_rev_tree(OldInfo, NewDoc, Client, false)
@@ -599,8 +618,18 @@ update_docs_int(Db, DocsList, LocalDocs, MergeConflicts, FullCommit) ->
#full_doc_info{id=Id}
end, Ids, OldDocLookups),
% Merge the new docs into the revision trees.
- {ok, NewFullDocInfos, RemSeqs, _} = merge_rev_trees(RevsLimit,
- MergeConflicts, DocsList, OldDocInfos, [], [], UpdateSeq),
+ AccIn = #merge_acc{
+ revs_limit = RevsLimit,
+ merge_conflicts = MergeConflicts,
+ add_infos = [],
+ rem_seqs = [],
+ cur_seq = UpdateSeq
+ },
+ {ok, AccOut} = merge_rev_trees(DocsList, OldDocInfos, AccIn),
+ #merge_acc{
+ add_infos = NewFullDocInfos,
+ rem_seqs = RemSeqs
+ } = AccOut,
% Write out the document summaries (the bodies are stored in the nodes of
% the trees, the attachments are already written to disk)