You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2017/07/15 18:04:08 UTC
[couchdb] branch 3430-external-size-views updated (19a827b ->
da5e714)
This is an automated email from the ASF dual-hosted git repository.
tonysun83 pushed a change to branch 3430-external-size-views
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
omit 19a827b change external reds name
omit 0f150fb update compression comparison tests
omit ab6db5f use 0 as return value for no external size value
omit f95cd61 calculate data_size correctly
add 2f539e0 Add X-Frame-Options (#582)
add c7f7dcf Fix "not_mocked" error in compaction tests
add 7105971 Refactor compaction tests setup and teardown
add 9ea4ad8 Address a race on compacting process exit
add d23b26d Merge pull request #614 from cloudant/fix-compaction_daemon_tests
add 571a2fc Ensure replicator _active_tasks entry reports recent pending changes value
add 67048ab Add a soak-eunit target to Makefile
add 06772ad Update CONTRIBUTING.md to reflect monorepo
add a5b8795 Merge pull request #625 from apache/contributing-md-update
add 409ea97 Also enable node decom using string "true"
add 7589340 bump docs dep
add 736e732 Fix mismatch between MAX_DBS_OPEN and default.ini
add dbe6cf7 Fix eunit timeout option for compression tests
add 7f54516 Remove yet another invalid test case (ddoc_cache related)
add 858088e Make map functions distinct with each invocation
add cc42357 feat: enable compaction daemon by default
add 8b127f4 test: expect compaction daemon to be off
add ff32a15 Fix couch_replicator_changes_reader:process_change
add 5b329ac Merge pull request #648 from cloudant/fixup-for-fix-replicator-progress-reporting-2
add 3b23d21 Remove some sleeps from change feed test.
add add912a Bump jiffy dep
add a461c44 really disable compaction daemon for JS tests
add 7885d80 wipe all databases on test exit
add a1b5e13 disable unstable stats.js test
add 3195578 Pass db open options to fabric_view_map
add 3e4da60 Merge pull request #645 from cloudant/pass-user_ctx-in-fabric_view_map
add 83f085d Fix Windows build
add da7aa54 Disable compaction daemon on eunit run couch startups
add 13a6e1f Merge branch 'master' of https://github.com/apache/couchdb
add 017d76f Fix Windows release builds
add 34b803a Remove deprecated OAuth 1.0 implementation
add cc668ba Make couch_event_sup:stop/1 synchronous
add 45c573b Require server or db admin user for db/_compact and db/_view_cleanup endpoints following rnewson's suggestion
add 3b509ce make sure to also run security check for design doc compaction
add 6d2ece5 Add tests for db admin/db member compaction.
add 6ff73aa Fix test
add 247b985 Require admin or db admin for compact requests
add 348accf admin context in couch tests no longer necessary for compaction with the new approach
add 1deeac0 fix whitespace crimes
add b95dbb3 Update couch_native_proc to respect idle timeouts
add 94f2907 Fix couchdb_os_proc_pool eunit timeouts
add 764168c Prevent a terrible race condition
add 6586102 Avoid a race when restarting an index updater
add 11afc8b Fix timeout in couchdb_views_test for couchdb_1283
add d909cd2 Pass UserCtx to fabric's all_docs from mango query
add 6d9673c Send user_ctx to query_view
add 022925a Send user_ctx to fabric's open_doc
add 779f00e Add tests for queries against _users db
add f77a78f Bump ets_lru for flaky test fix
add 677e748 Change recommended install path to /opt/couchdb
add 03f2657 Improve UNIX dependency installation (Node.JS, Sphinx)
add 23127c1 Make replication filter fetch error for _replicate return a 404
add a866f6c Fix replication Javascript test
add d5a0d99 Remove unused TIMEOUT macro from os daemons module
add 789f75d Fix flaky os daemons tests
add dce6e34 Use Ejson Body Instead of Compressed Body for External size (#606)
new da5e714 calculate data_size correctly
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (19a827b)
\
N -- N -- N refs/heads/3430-external-size-views (da5e714)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CONTRIBUTING.md | 8 +-
INSTALL.Unix.md | 62 ++-
LICENSE | 25 -
Makefile | 6 +
Makefile.win | 8 +-
NOTICE | 4 -
build-aux/print-committerlist.sh | 2 +-
configure.ps1 | 2 +-
dev/run | 4 +-
license.skip | 2 -
rebar.config.script | 10 +-
rel/overlay/etc/default.ini | 38 +-
rel/reltool.config | 2 -
src/chttpd/src/chttpd.erl | 5 +-
src/chttpd/src/chttpd_auth_request.erl | 34 ++
src/chttpd/src/chttpd_show.erl | 3 +-
src/chttpd/src/chttpd_view.erl | 10 +-
src/chttpd/src/chttpd_xframe_options.erl | 99 ++++
src/chttpd/test/chttpd_security_tests.erl | 186 ++++++++
src/chttpd/test/chttpd_xframe_test.erl | 84 ++++
src/couch/include/couch_db.hrl | 1 +
src/couch/include/couch_js_functions.hrl | 18 -
src/couch/src/couch.app.src | 1 -
src/couch/src/couch.erl | 1 -
src/couch/src/couch_db.erl | 6 +-
src/couch/src/couch_db_updater.erl | 22 +-
src/couch/src/couch_event_sup.erl | 10 +-
src/couch/src/couch_httpd.erl | 8 +-
src/couch/src/couch_httpd_handlers.erl | 1 -
src/couch/src/couch_httpd_oauth.erl | 391 ----------------
src/couch/src/couch_native_process.erl | 46 +-
src/couch/src/couch_os_daemons.erl | 1 -
src/couch/src/couch_server.erl | 2 +-
src/couch/src/test_util.erl | 2 +-
src/couch/test/chttpd_endpoints_tests.erl | 1 -
src/couch/test/couch_changes_tests.erl | 37 +-
src/couch/test/couchdb_auth_tests.erl | 2 +-
src/couch/test/couchdb_compaction_daemon_tests.erl | 57 ++-
src/couch/test/couchdb_file_compression_tests.erl | 39 +-
src/couch/test/couchdb_os_daemons_tests.erl | 3 +-
src/couch/test/couchdb_os_proc_pool.erl | 6 +-
src/couch/test/couchdb_vhosts_tests.erl | 139 ------
src/couch/test/couchdb_views_tests.erl | 27 ++
src/couch/test/fixtures/os_daemon_configer.escript | 1 -
src/couch_index/src/couch_index.erl | 49 +-
src/couch_index/src/couch_index_updater.erl | 14 +-
.../src/couch_replicator_changes_reader.erl | 25 +-
.../src/couch_replicator_scheduler_job.erl | 16 +-
.../src/couch_replicator_worker.erl | 4 +
src/fabric/src/fabric.erl | 23 +-
src/fabric/src/fabric_view_map.erl | 11 +-
src/mango/src/mango_cursor_view.erl | 9 +-
src/mango/src/mango_httpd.erl | 3 +-
src/mango/src/mango_util.erl | 6 +-
src/mango/test/13-users-db-find-test.py | 83 ++++
src/mango/test/mango.py | 11 +
src/mango/test/user_docs.py | 36 ++
src/mem3/src/mem3.erl | 35 +-
test/javascript/couch.js | 3 +
test/javascript/oauth.js | 511 ---------------------
test/javascript/run | 1 -
test/javascript/tests/config.js | 3 -
test/javascript/tests/delayed_commits.js | 6 +
test/javascript/tests/design_docs.js | 16 -
test/javascript/tests/oauth_users_db.js | 168 -------
test/javascript/tests/proxyauth.js | 2 +
test/javascript/tests/reader_acl.js | 7 +-
test/javascript/tests/replication.js | 30 +-
test/javascript/tests/replicator_db_security.js | 28 --
test/javascript/tests/rev_stemming.js | 2 +
test/javascript/tests/rewrite.js | 5 +-
test/javascript/tests/rewrite_js.js | 3 +
test/javascript/tests/security_validation.js | 5 +
test/javascript/tests/stats.js | 25 +
74 files changed, 1044 insertions(+), 1512 deletions(-)
create mode 100644 src/chttpd/src/chttpd_xframe_options.erl
create mode 100644 src/chttpd/test/chttpd_security_tests.erl
create mode 100644 src/chttpd/test/chttpd_xframe_test.erl
delete mode 100644 src/couch/src/couch_httpd_oauth.erl
create mode 100644 src/mango/test/13-users-db-find-test.py
delete mode 100644 test/javascript/oauth.js
delete mode 100644 test/javascript/tests/oauth_users_db.js
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].
[couchdb] 01/01: calculate data_size correctly
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tonysun83 pushed a commit to branch 3430-external-size-views
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit da5e7146af87c15c8bdc1a110b3f885d9bf2c4da
Author: Tony Sun <to...@cloudant.com>
AuthorDate: Mon Jun 12 09:26:49 2017 -0700
calculate data_size correctly
Previously, we were calculating the ExternalSize for views by summing
up all the nodes in the btree. Furthermore, this was the compressed
size. Now we modify the reduce function to return an ExternalSize for
uncompressed values in the KVList.
PR: https://github.com/apache/couchdb/pull/608
COUCHDB-3430
---
src/couch/test/couchdb_file_compression_tests.erl | 12 ++++++
src/couch_mrview/src/couch_mrview_util.erl | 47 +++++++++++++++++------
2 files changed, 48 insertions(+), 11 deletions(-)
diff --git a/src/couch/test/couchdb_file_compression_tests.erl b/src/couch/test/couchdb_file_compression_tests.erl
index 41d0556..fa2c226 100644
--- a/src/couch/test/couchdb_file_compression_tests.erl
+++ b/src/couch/test/couchdb_file_compression_tests.erl
@@ -125,6 +125,7 @@ compare_compression_methods(DbName) ->
DbSizeNone = db_disk_size(DbName),
ViewSizeNone = view_disk_size(DbName),
ExternalSizeNone = db_external_size(DbName),
+ ViewExternalSizeNone = view_external_size(DbName),
config:set("couchdb", "file_compression", "snappy", false),
compact_db(DbName),
@@ -132,6 +133,7 @@ compare_compression_methods(DbName) ->
DbSizeSnappy = db_disk_size(DbName),
ViewSizeSnappy = view_disk_size(DbName),
ExternalSizeSnappy = db_external_size(DbName),
+ ViewExternalSizeSnappy = view_external_size(DbName),
?assert(DbSizeNone > DbSizeSnappy),
?assert(ViewSizeNone > ViewSizeSnappy),
@@ -151,12 +153,15 @@ compare_compression_methods(DbName) ->
DbSizeDeflate9 = db_disk_size(DbName),
ViewSizeDeflate9 = view_disk_size(DbName),
ExternalSizeDeflate9 = db_external_size(DbName),
+ ViewExternalSizeDeflate9 = view_external_size(DbName),
?assert(DbSizeDeflate1 > DbSizeDeflate9),
?assert(ViewSizeDeflate1 > ViewSizeDeflate9),
?assert(ExternalSizePreCompact =:= ExternalSizeNone),
?assert(ExternalSizeNone =:= ExternalSizeSnappy),
?assert(ExternalSizeNone =:= ExternalSizeDeflate9).
+ ?assert(ViewExternalSizeNone =:= ViewExternalSizeSnappy),
+ ?assert(ViewExternalSizeNone =:= ViewExternalSizeDeflate9).
populate_db(_Db, NumDocs) when NumDocs =< 0 ->
@@ -214,6 +219,13 @@ view_disk_size(DbName) ->
ok = couch_db:close(Db),
active_size(Info).
+view_external_size(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, DDoc} = couch_db:open_doc(Db, ?DDOC_ID, [ejson_body]),
+ {ok, Info} = couch_mrview:get_info(Db, DDoc),
+ ok = couch_db:close(Db),
+ external_size(Info).
+
active_size(Info) ->
couch_util:get_nested_json_value({Info}, [sizes, active]).
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index a8462a2..6325222 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -338,7 +338,10 @@ temp_view_to_ddoc({Props}) ->
get_row_count(#mrview{btree=Bt}) ->
- {ok, {Count, _Reds}} = couch_btree:full_reduce(Bt),
+ Count = case couch_btree:full_reduce(Bt) of
+ {ok, {Count0, _Reds, _}} -> Count0;
+ {ok, {Count0, _Reds}} -> Count0
+ end,
{ok, Count}.
@@ -786,27 +789,33 @@ changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
end.
+reduced_external_size(Tree) ->
+ case couch_btree:full_reduce(Tree) of
+ {ok, {_, _, Size}} -> Size;
+ % return 0 for versions of the reduce function without Size
+ {ok, {_, _}} -> 0
+ end.
calculate_external_size(Views) ->
SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
- Size0 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+ Size0 = sum_btree_sizes(Acc, reduced_external_size(Bt)),
Size1 = case SBt of
nil -> Size0;
- _ -> sum_btree_sizes(Size0, couch_btree:size(SBt))
+ _ -> sum_btree_sizes(Size0, reduced_external_size(SBt))
end,
case KSBt of
nil -> Size1;
- _ -> sum_btree_sizes(Size1, couch_btree:size(KSBt))
+ _ -> sum_btree_sizes(Size1, reduced_external_size(KSBt))
end
end,
{ok, lists:foldl(SumFun, 0, Views)}.
sum_btree_sizes(nil, _) ->
- null;
+ 0;
sum_btree_sizes(_, nil) ->
- null;
+ 0;
sum_btree_sizes(Size1, Size2) ->
Size1 + Size2.
@@ -1038,22 +1047,32 @@ get_user_reds(Reduction) ->
element(2, Reduction).
+get_external_size_reds(Reduction) when tuple_size(Reduction) == 2 ->
+ 0;
+
+get_external_size_reds(Reduction) when tuple_size(Reduction) == 3 ->
+ element(3, Reduction).
+
+
make_reduce_fun(Lang, ReduceFuns) ->
FunSrcs = [FunSrc || {_, FunSrc} <- ReduceFuns],
fun
(reduce, KVs0) ->
KVs = detuple_kvs(expand_dups(KVs0, []), []),
{ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs),
- {length(KVs), Result};
+ ExternalSize = kv_external_size(KVs, Result),
+ {length(KVs), Result, ExternalSize};
(rereduce, Reds) ->
- ExtractFun = fun(Red, {CountsAcc0, URedsAcc0}) ->
+ ExtractFun = fun(Red, {CountsAcc0, URedsAcc0, ExtAcc0}) ->
CountsAcc = CountsAcc0 + get_count(Red),
URedsAcc = lists:append(URedsAcc0, [get_user_reds(Red)]),
- {CountsAcc, URedsAcc}
+ ExtAcc = ExtAcc0 + get_external_size_reds(Red),
+ {CountsAcc, URedsAcc, ExtAcc}
end,
- {Counts, UReds} = lists:foldl(ExtractFun, {0, []}, Reds),
+ {Counts, UReds, ExternalSize} = lists:foldl(ExtractFun,
+ {0, [], 0}, Reds),
{ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UReds),
- {Counts, Result}
+ {Counts, Result, ExternalSize}
end.
@@ -1130,3 +1149,9 @@ get_view_queries({Props}) ->
_ ->
throw({bad_request, "`queries` member must be a array."})
end.
+
+
+kv_external_size(KVList, Reduction) ->
+ lists:foldl(fun([[Key, _], Value], Acc) ->
+ ?term_size(Key) + ?term_size(Value) + Acc
+ end, ?term_size(Reduction), KVList).
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.