You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ji...@apache.org on 2018/08/09 07:07:48 UTC
[couchdb] branch COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
updated (1499c79 -> 0bbe59f)
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a change to branch COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
omit 1499c79 Create new on_compact trigger
omit 826b17e Enhance PSE tests with setup/teardown functions
omit 3eb11f1 Update to use new couch_pse_tests app
omit 869f9a2 Update PSE test definitions for new util module
omit f3c3804 Rename PSE test modules
omit 0d425fa Move PSE tests to their own app
omit a769b3e Rewrite the PSE test suite to use couch_server
omit b43416b Simplify logic in mem3_rep
omit 7b91b90 Update fabric_doc_open eunit tests
omit c3d7bf8 Fix race on couch_db:reopen/1
omit 39d7803 Fix default security object handling
omit 127bfda Fix bug during purge
omit 6d13153 Fix typos in couch_db_engine.erl
add 5290a32 Update Jenkins build process:
add 6d44e17 Add _approx_count_distinct as a builtin reduce function (#1346)
add 8a46473 Add hyper app to dependencies
add 0392c51 Finalize in couch_mrview, but make it optional
add 5fa3c43 Use finalize operation to simplify _stats
add 398ac18 Ignore trailing characters in a builtin reduce
add 62f71c0 Fix container for package-building step
add 994b370 Jenkinsfile: typo
add 71c33b1 Update skip_deps for 3rd parties eunit (#1386)
add 2bf04a0 Revert "Introduce bin_opt_info erl_opts compilation option"
add 76790d5 Add compile's command line options
add 2fe402f Remove debug_info from compile options
add dfa8780 Make bin_opt_info optional based on env variable
add 41decfa Allow custom compile options with env variable
add c7d35c3 Merge pull request #1387 from cloudant/make-bin_opt_info-optional
add 000766c Fix active size calculations for views
add aebdbc4 Optimize couch_key_tree:stem/2
add 3c98385 Fix couch_key_tree_tests.erl
add f040d75 Add set_mqd_off_heap utility function
add a13efba Call `set_mqd_off_heap` for critical processes
add f3a0f42 refactor process_request to not drop req (#1375)
add fe53e43 Prepare to fabric attachment receiver from a fun closure to a tuple
add 5b5c8a1 Add constant index fields to sort based on the selector (#1376)
add 103a062 Update snappy dep to CouchDB-1.0.1 with 21.0 support
add e6272de Update build dependency docs for FreeBSD
add 1209036 Add regression runs on new dedicated FreeBSD VM
add 3358668 fix(_view changes feed): fix function_clause crash in couch_native_process. Crash was caused by a missing implementation of ddoc function for <<"views">> FunPath, implementation is based on FilterFun but matches return values of the erlang:put() which is called in the native Emit function and also expects ok and false when docs were not emitted.
add 0377e53 tests(couch_changes): add tests for changes feed filtering using an Erlang view
add d36149e Merge pull request #1401 from Spotme/spotme/fix/couch-native-process
add 5bbfbeb Don't set context for compaction files during db delete (#1419)
add 73f1565 remove mention of os daemons and externals in ini files
add 54577f1 allow socket server configuration for TLS httpd
add 142eb6b feat: avoid double builds on PRs
add 1b15d4c feat: remove duplicate authentication check
add 7dfed0c Added tests for checking http status code depending on cluster quorum
add bfc610f Make stem_interactive_updates option work again
add 80e431f Fix for issue #1136 - Error 500 deleting DB without quorum
add 177c22d Deletion responds 200 after a response from every node, and 202 in other case
add 71cf9f4 Adjust deletion tests in different cluster quorum conditions
add 45a0ad9 Add `conflicts: true` option to mango selectors
add 572234f Add tests for mango conflict finding
add e928b88 hook up proxy auth handler to chttpd
add aa73e4e Views now retain update_seq after compaction.
add a5858b9 Replace R16B03 with 17.5.3 for PR#1427
add 0f130cb Drop Erlang R16 support
add 4ea7210 config: improve handling of admin-supplied changes
add 768e9eb build: ease pattern matching to be less pedantic about RC
add a82419b build: release candidate tarball should have -RCx
add 5c93cb7 Use couch_util:trim for greater erlang compatibility
add 3c3546c string:trim() compat for couch_util:trim()
add fa3c812 bump deps
add 1703998 raise timeouts in attachment tests
add 215f49b wait for db deletion before db re-creation. Closes #1197
add dacb658 Clarify bad index creation error messages
add dd3a6b4 validate bind address by @wyc
add 8e5caaf demote notice to debug logs by @wyc
add 45583af Optionally prevent non-admins from accessing /_all_dbs
add 8c2b86a improve ddoc test reliablilty by waiting for ddoc deletion
add 3acf15f Make MD5 hash implementation configurable (#1171)
add 79eb1af Report git sha on welcome request
add 1d69790 Improve detection of git tags/dirty status
add 6ed65e8 add retry to test_request:request()
add 3ef3149 re-raise max_http_request_size to 4GB
add e2f7561 Add missing default case clause, fixes #1450
add 5678245 Switch fabric attachment receiver to use messages
add a239f76 bump hyper dependency, fix Windows build
add c94cc4c Revert "Make loginUser wait for successful authentication"
add 91f84f3 Revert "Replace resource expensive bcrypt test with shorter version (#1231)"
add 7597abf Revert "Add bcrypt hashing option"
add 98eec31 Enable replication client _session auth by default
add b1fbbb1 Bump fauxton dependency
add cec965f [travis] 20.1 -> 20.3
add 10c19dc bump docs dependency
add 56bd13d Improve logging from test_util
add 2446758 bump fauxton
add cc1360d remove obsolete update_notification feature
add aeced65 Fix _stats reducer when map function emits array
add 17aec1f Merge pull request #1484 from cloudant/fix-stats-reducer-for-array
add d29011b Fixed use of find ... -printf "%f" on Mac OS by replacing it with -exec basename
add 0cbaefc added missing dependencies for Debian-based systems
add 99e801e Use new mochiweb recbuf|sndbuf undefined setting
add 54b2eaa Bump fauxton
add 0fefc85 Rename 'module' data provider to 'static_module'
add cf65280 Add 'callback_module' provider
add 7a2a390 Update documentation
add d98e3e1 Merge pull request #1432 from cloudant/support-callback-module-data-provider
add a7f2aa5 Add rexi ping message
add a6bc72e Move mango selector matching to the shard level
add 1e2e8af Fix typos in couch_db_engine.erl
add 00d8379 Fix bug during purge
add c96df45 Fix default security object handling
add 82dabd7 Fix race on couch_db:reopen/1
add 55ff797 Update fabric_doc_open eunit tests
add 966497c Simplify logic in mem3_rep
add fa45f9a Rewrite the PSE test suite to use couch_server
add 56fbe19 Move PSE tests to their own app
add 622bfdc Rename PSE test modules
add 811dca9 Update PSE test definitions for new util module
add 5f4f413 Update to use new couch_pse_tests app
add 163b02e Enhance PSE tests with setup/teardown functions
new 0bbe59f Create new on_compact trigger
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1499c79)
\
N -- N -- N refs/heads/COUCHDB-3326-clustered-purge-pr4-on-compact-plugin (0bbe59f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 2 +-
.travis.yml | 11 +-
INSTALL.Unix.md | 9 +-
Jenkinsfile | 153 ++++++++++----
LICENSE | 135 ++-----------
Makefile | 62 ++++--
Makefile.win | 45 ++++-
NOTICE | 9 +-
build-aux/couchdb-build-release.sh | 6 +-
configure | 9 +
rebar.config.script | 25 ++-
rel/overlay/etc/default.ini | 53 ++---
rel/overlay/etc/local.ini | 16 +-
rel/reltool.config | 4 +-
src/chttpd/src/chttpd.erl | 70 ++++++-
src/chttpd/src/chttpd_auth.erl | 4 +
src/chttpd/src/chttpd_auth_request.erl | 5 +-
src/chttpd/src/chttpd_db.erl | 1 -
src/chttpd/src/chttpd_misc.erl | 11 +-
src/chttpd/test/chttpd_welcome_test.erl | 2 +
src/couch/rebar.config.script | 17 +-
src/couch/src/couch.app.src | 4 +-
src/couch/src/couch.erl | 1 -
src/couch/src/couch_auth_cache.erl | 7 -
src/couch/src/couch_bt_engine.erl | 6 +-
src/couch/src/couch_db.erl | 12 +-
src/couch/src/couch_db_updater.erl | 38 ++--
src/couch/src/couch_file.erl | 8 +-
src/couch/src/{couch_rand.erl => couch_hash.erl} | 50 ++---
src/couch/src/couch_httpd.erl | 2 +-
src/couch/src/couch_httpd_auth.erl | 12 +-
src/couch/src/couch_httpd_misc_handlers.erl | 2 +-
src/couch/src/couch_key_tree.erl | 83 +++++++-
src/couch/src/couch_native_process.erl | 16 +-
src/couch/src/couch_passwords.erl | 19 +-
src/couch/src/couch_query_servers.erl | 80 +++++++-
src/couch/src/couch_server.erl | 7 +-
src/couch/src/couch_stream.erl | 18 +-
src/couch/src/couch_users_db.erl | 10 +-
src/couch/src/couch_util.erl | 50 ++++-
src/couch/src/test_request.erl | 6 +-
src/couch/src/test_util.erl | 3 +
src/couch/test/couch_changes_tests.erl | 38 +++-
src/couch/test/couch_key_tree_tests.erl | 197 +++++++++---------
src/couch/test/couch_passwords_tests.erl | 42 +---
src/couch/test/couchdb_attachments_tests.erl | 12 +-
src/couch_epi/README.md | 35 +++-
src/couch_epi/src/couch_epi_data.erl | 13 +-
src/couch_epi/src/couch_epi_sup.erl | 5 +-
src/couch_epi/src/couch_epi_util.erl | 2 +-
src/couch_epi/test/couch_epi_tests.erl | 67 ++++++-
src/couch_event/src/couch_event_os_sup.erl | 82 --------
src/couch_event/src/couch_event_sup2.erl | 7 -
.../test/couch_index_ddoc_updated_tests.erl | 2 +-
src/couch_log/src/couch_log_server.erl | 1 +
src/couch_mrview/src/couch_mrview.erl | 32 ++-
src/couch_mrview/src/couch_mrview_compactor.erl | 2 +
src/couch_mrview/src/couch_mrview_index.erl | 3 +-
src/couch_mrview/src/couch_mrview_util.erl | 25 ++-
.../test/couch_mrview_index_info_tests.erl | 96 ++++++---
src/couch_pse_tests/src/cpse_util.erl | 33 ++-
src/couch_replicator/src/couch_replicator_auth.erl | 2 +-
src/couch_replicator/src/couch_replicator_ids.erl | 2 +-
.../test/couch_replicator_many_leaves_tests.erl | 2 +-
.../test/couch_replicator_test_helper.erl | 12 +-
src/ddoc_cache/src/ddoc_cache_lru.erl | 1 +
src/fabric/src/fabric.erl | 9 +-
src/fabric/src/fabric_db_delete.erl | 6 +-
...ric_doc_attachments.erl => fabric_doc_atts.erl} | 28 ++-
src/fabric/src/fabric_rpc.erl | 26 ++-
src/fabric/src/fabric_view.erl | 3 +-
src/mango/src/mango_cursor.erl | 18 +-
src/mango/src/mango_cursor_view.erl | 120 +++++++++--
src/mango/src/mango_error.erl | 4 +-
src/mango/src/mango_execution_stats.erl | 7 +-
src/mango/src/mango_idx.erl | 36 +---
src/mango/src/mango_idx_special.erl | 13 +-
src/mango/src/mango_idx_view.erl | 30 ++-
src/mango/src/mango_selector.erl | 113 ++++++++++-
src/mango/test/02-basic-find-test.py | 7 +
src/mango/test/18-json-sort.py | 222 +++++++++++++++++++++
src/mango/test/19-find-conflicts.py | 41 ++++
src/mango/test/20-no-timeout-test.py | 38 ++++
src/mango/test/mango.py | 5 +
src/mem3/src/mem3_rep.erl | 6 +-
src/mem3/src/mem3_shards.erl | 1 +
src/rexi/src/rexi.erl | 9 +
src/rexi/src/rexi_server.erl | 1 +
src/rexi/src/rexi_utils.erl | 2 +
src/setup/src/setup.erl | 43 ++--
test/javascript/run | 8 +-
.../with-quorum/attachments.js} | 30 +--
.../with-quorum/attachments_delete.js} | 28 ++-
.../attachments_delete_overridden_quorum.js | 36 ++++
.../with-quorum/attachments_overridden_quorum.js | 40 ++++
.../with-quorum/{db-creation.js => db_creation.js} | 0
...reation.js => db_creation_overridden_quorum.js} | 11 +-
.../db-creation.js => with-quorum/db_deletion.js} | 16 +-
...reation.js => db_deletion_overridden_quorum.js} | 14 +-
.../with-quorum/{db-creation.js => doc_bulk.js} | 20 +-
...b-creation.js => doc_bulk_overridden_quorum.js} | 20 +-
.../with-quorum/{db-creation.js => doc_copy.js} | 20 +-
.../with-quorum/doc_copy_overridden_quorum.js} | 26 +--
.../with-quorum/{db-creation.js => doc_crud.js} | 24 ++-
...b-creation.js => doc_crud_overridden_quorum.js} | 24 ++-
.../tests-cluster/without-quorum/attachments.js | 39 ++++
.../without-quorum/attachments_delete.js | 37 ++++
.../attachments_delete_overridden_quorum.js | 36 ++++
.../attachments_overridden_quorum.js | 42 ++++
.../{db-creation.js => db_creation.js} | 3 +-
...reation.js => db_creation_overridden_quorum.js} | 12 +-
.../{db-creation.js => db_deletion.js} | 18 +-
...reation.js => db_deletion_overridden_quorum.js} | 15 +-
.../without-quorum/{db-creation.js => doc_bulk.js} | 20 +-
...b-creation.js => doc_bulk_overridden_quorum.js} | 20 +-
.../without-quorum/{db-creation.js => doc_copy.js} | 20 +-
...b-creation.js => doc_copy_overridden_quorum.js} | 23 ++-
.../without-quorum/{db-creation.js => doc_crud.js} | 25 ++-
...b-creation.js => doc_crud_overridden_quorum.js} | 24 ++-
test/javascript/tests/design_docs.js | 8 +-
test/javascript/tests/reduce_builtin.js | 20 ++
test/javascript/tests/users_db.js | 7 +
test/javascript/tests/users_db_security.js | 167 ++--------------
123 files changed, 2254 insertions(+), 1183 deletions(-)
copy src/couch/src/{couch_rand.erl => couch_hash.erl} (50%)
delete mode 100644 src/couch_event/src/couch_event_os_sup.erl
copy src/fabric/src/{fabric_doc_attachments.erl => fabric_doc_atts.erl} (93%)
create mode 100644 src/mango/test/18-json-sort.py
create mode 100644 src/mango/test/19-find-conflicts.py
create mode 100644 src/mango/test/20-no-timeout-test.py
copy test/javascript/{tests/large_docs.js => tests-cluster/with-quorum/attachments.js} (54%)
copy test/javascript/{tests/large_docs.js => tests-cluster/with-quorum/attachments_delete.js} (57%)
create mode 100644 test/javascript/tests-cluster/with-quorum/attachments_delete_overridden_quorum.js
create mode 100644 test/javascript/tests-cluster/with-quorum/attachments_overridden_quorum.js
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => db_creation.js} (100%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => db_creation_overridden_quorum.js} (62%)
copy test/javascript/tests-cluster/{without-quorum/db-creation.js => with-quorum/db_deletion.js} (70%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => db_deletion_overridden_quorum.js} (67%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => doc_bulk.js} (70%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => doc_bulk_overridden_quorum.js} (63%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => doc_copy.js} (70%)
copy test/javascript/{tests/large_docs.js => tests-cluster/with-quorum/doc_copy_overridden_quorum.js} (55%)
copy test/javascript/tests-cluster/with-quorum/{db-creation.js => doc_crud.js} (68%)
rename test/javascript/tests-cluster/with-quorum/{db-creation.js => doc_crud_overridden_quorum.js} (55%)
create mode 100644 test/javascript/tests-cluster/without-quorum/attachments.js
create mode 100644 test/javascript/tests-cluster/without-quorum/attachments_delete.js
create mode 100644 test/javascript/tests-cluster/without-quorum/attachments_delete_overridden_quorum.js
create mode 100644 test/javascript/tests-cluster/without-quorum/attachments_overridden_quorum.js
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => db_creation.js} (89%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => db_creation_overridden_quorum.js} (64%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => db_deletion.js} (72%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => db_deletion_overridden_quorum.js} (65%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_bulk.js} (74%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_bulk_overridden_quorum.js} (67%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_copy.js} (72%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_copy_overridden_quorum.js} (56%)
copy test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_crud.js} (65%)
rename test/javascript/tests-cluster/without-quorum/{db-creation.js => doc_crud_overridden_quorum.js} (58%)
[couchdb] 01/01: Create new on_compact trigger
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangphcn pushed a commit to branch COUCHDB-3326-clustered-purge-pr4-on-compact-plugin
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0bbe59fb7e3e8d5d415e2bd6e117951ff41b296c
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed May 30 14:34:44 2018 -0500
Create new on_compact trigger
This trigger allows any storage engine that makes use of compaction to
notify that compaction is starting. This is preparatory work for
clustered indexes so that existing indexes are allowed to ensure they
have a clustered purge local doc before compaction runs.
COUCHDB-3326
Co-Authored-By: jiangphcn <ji...@cn.ibm.com>
---
src/couch/src/couch_bt_engine_compactor.erl | 2 ++
src/couch/src/couch_db_engine.erl | 31 ++++++++++++++++++++++++++++-
src/couch/src/couch_db_plugin.erl | 5 +++++
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl
index 4e52064..2c5b78e 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -44,6 +44,8 @@ start(#st{} = St, DbName, Options, Parent) ->
} = St,
couch_log:debug("Compaction process spawned for db \"~s\"", [DbName]),
+ couch_db_engine:trigger_on_compact(DbName),
+
{ok, NewSt, DName, DFd, MFd, Retry} =
open_compaction_files(Header, FilePath, Options),
erlang:monitor(process, MFd),
diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl
index 4974201..2fe0b0d 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -627,7 +627,8 @@
count_changes_since/2,
start_compaction/1,
- finish_compaction/2
+ finish_compaction/2,
+ trigger_on_compact/1
]).
@@ -891,3 +892,31 @@ finish_compaction(Db, CompactInfo) ->
end,
ok = gen_server:call(couch_server, {db_updated, NewDb}, infinity),
{ok, NewDb}.
+
+
+trigger_on_compact(DbName) ->
+ {ok, DDocs} = get_ddocs(DbName),
+ couch_db_plugin:on_compact(DbName, DDocs).
+
+
+get_ddocs(<<"shards/", _/binary>> = DbName) ->
+ {_, Ref} = spawn_monitor(fun() ->
+ exit(fabric:design_docs(mem3:dbname(DbName)))
+ end),
+ receive
+ {'DOWN', Ref, _, _, {ok, JsonDDocs}} ->
+ {ok, lists:map(fun(JsonDDoc) ->
+ couch_doc:from_json_obj(JsonDDoc)
+ end, JsonDDocs)};
+ {'DOWN', Ref, _, _, Else} ->
+ Else
+ end;
+get_ddocs(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ FoldFun = fun(FDI, Acc) ->
+ Doc = couch_db:open_doc_int(Db, FDI, []),
+ {ok, [Doc | Acc]}
+ end,
+ {ok, Docs} = couch_db:fold_design_docs(Db, FoldFun, [], []),
+ {ok, lists:reverse(Docs)}
+ end).
diff --git a/src/couch/src/couch_db_plugin.erl b/src/couch/src/couch_db_plugin.erl
index 740b812..8163256 100644
--- a/src/couch/src/couch_db_plugin.erl
+++ b/src/couch/src/couch_db_plugin.erl
@@ -18,6 +18,7 @@
after_doc_read/2,
validate_docid/1,
check_is_admin/1,
+ on_compact/2,
on_delete/2
]).
@@ -56,6 +57,10 @@ check_is_admin(Db) ->
%% callbacks return true only if it specifically allow the given Id
couch_epi:any(Handle, ?SERVICE_ID, check_is_admin, [Db], []).
+on_compact(DbName, DDocs) ->
+ Handle = couch_epi:get_handle(?SERVICE_ID),
+ couch_epi:apply(Handle, ?SERVICE_ID, on_compact, [DbName, DDocs], []).
+
on_delete(DbName, Options) ->
Handle = couch_epi:get_handle(?SERVICE_ID),
couch_epi:apply(Handle, ?SERVICE_ID, on_delete, [DbName, Options], []).