You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2022/09/01 16:05:29 UTC
[couchdb] branch raft_storemodule updated (52eafa818 -> c5569e709)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
from 52eafa818 use global so raft member names don't have to be atoms (we'll run out)
new 2017162ff Add some utility functions to couch_replicator_test_helper
new 2bc960b73 Update couch_replicator_attachments_too_large to use fabric
new 250285e42 Update couch_replicator_compact_tests
new 63c9d89ca Update couch_replicator_connection_tests
new cd04b1454 Update couch_replicator_create_target_with_options_tests
new 410ed0bc3 Update couch_replicator_error_reporting_tests
new fda897a6f Update couch_replicator_filtered_tests
new e7de68e58 Update couch_replicator_httpc_pool_tests
new 7fc302f05 Update couch_replicator_id_too_long_tests
new 62c761613 Update couch_replicator_large_atts_tests
new 24547d5d2 Update couch_replicator_many_leaves_tests
new 690e9b52d Update couch_replicator_missing_stubs_tests
new f61cabf3c Update couch_replicator_proxy_tests
new bc8ede12a Update couch_replicator_rate_limiter_tests
new 9a03f29ce Update couch_replicator_retain_stats_between_job_runs
new a8d7da8b1 Update couch_replicator_selector_tests
new 362e82d9a Update couch_replicator_small_max_request_size_target
new 811bcf073 Update couch_replicator_use_checkpoints_tests
new 87fe58999 update devcontainer
new a3c09e12a update variable name and readme
new 97a747496 moved the name property to the correct spot.
new af42d364f Address race in cpse_incref_decref test
new 387f32ae5 config section for require_valid_user is only [chttpd]
new 37e177fd7 fix missing "=" for admin party in #4153
new e762b10a2 Upgrade hash algorithm for cookie auth (#4140)
new 4d24d8989 Refactor hash algorithms test
new ab5e3daad Fix variable already bound compiler warnings
new eea0293f5 Allow and evaluate nested json claim roles in JWT token
new 88d7f23a8 Implement _bulk_get support for the replicator
new d6fa02d4a include raft name and node in log lines
new c5569e709 HACK: demonstrate the shards running elections in a very hackish way
The 31 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.devcontainer/Dockerfile | 31 +-
.devcontainer/devcontainer.json | 30 +-
README-DEV.rst | 16 ++
README.rst | 19 ++
dev/run | 12 +-
rel/overlay/etc/default.ini | 40 ++-
rel/overlay/etc/local.ini | 6 -
src/chttpd/src/chttpd.erl | 2 +-
.../eunit/chttpd_auth_hash_algorithms_tests.erl | 102 +++++++
src/chttpd/test/eunit/chttpd_csp_tests.erl | 7 +-
.../test/eunit/chttpd_db_attachment_size_tests.erl | 5 +-
src/chttpd/test/eunit/chttpd_delayed_test.erl | 5 +-
src/chttpd/test/eunit/chttpd_session_tests.erl | 10 +-
src/chttpd/test/eunit/chttpd_util_test.erl | 46 +--
src/couch/include/couch_db.hrl | 2 +
src/couch/src/couch_bt_engine.erl | 14 +-
src/couch/src/couch_bt_engine.hrl | 3 +-
src/couch/src/couch_httpd.erl | 2 +-
src/couch/src/couch_httpd_auth.erl | 73 ++++-
src/couch/src/couch_raft.erl | 41 +--
src/couch/src/couch_util.erl | 31 ++
src/couch_pse_tests/src/cpse_test_ref_counting.erl | 11 +-
.../src/couch_replicator_api_wrap.erl | 83 +++++-
.../src/couch_replicator_changes_reader.erl | 2 +-
.../src/couch_replicator_httpc.erl | 4 +
.../src/couch_replicator_scheduler_job.erl | 30 +-
.../src/couch_replicator_stats.erl | 14 +-
.../src/couch_replicator_worker.erl | 294 ++++++++++++++-----
.../couch_replicator_attachments_too_large.erl | 79 ++----
.../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------
.../eunit/couch_replicator_connection_tests.erl | 264 ++++++++---------
...replicator_create_target_with_options_tests.erl | 74 ++---
.../couch_replicator_error_reporting_tests.erl | 300 +++++++++-----------
.../test/eunit/couch_replicator_filtered_tests.erl | 257 +++++++----------
.../eunit/couch_replicator_httpc_pool_tests.erl | 96 +++----
.../eunit/couch_replicator_id_too_long_tests.erl | 82 ++----
.../eunit/couch_replicator_large_atts_tests.erl | 92 ++----
.../eunit/couch_replicator_many_leaves_tests.erl | 190 +++++--------
.../eunit/couch_replicator_missing_stubs_tests.erl | 146 +++-------
.../test/eunit/couch_replicator_proxy_tests.erl | 137 +++++----
.../eunit/couch_replicator_rate_limiter_tests.erl | 77 +++--
...ch_replicator_retain_stats_between_job_runs.erl | 159 ++++-------
.../test/eunit/couch_replicator_selector_tests.erl | 121 +++-----
...ch_replicator_small_max_request_size_target.erl | 158 +++--------
.../test/eunit/couch_replicator_test_helper.erl | 134 ++++++---
.../couch_replicator_use_checkpoints_tests.erl | 182 ++++--------
.../test/eunit/global_changes_hooks_tests.erl | 2 +-
test/elixir/test/jwt_roles_claim_test.exs | 167 +++++++++++
48 files changed, 2031 insertions(+), 1935 deletions(-)
create mode 100644 src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
create mode 100644 test/elixir/test/jwt_roles_claim_test.exs
[couchdb] 17/31: Update couch_replicator_small_max_request_size_target
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 362e82d9a51b1e7cbaf6deb01a23097828231e38
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:41:14 2022 -0400
Update couch_replicator_small_max_request_size_target
Use the clustered version of the source and target endoints and switch to using
common test setup and teardown function functions. Overall it added to quite a
few number of lines saved.
---
...ch_replicator_small_max_request_size_target.erl | 158 ++++++---------------
1 file changed, 43 insertions(+), 115 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
index 3b020927d..4a905850d 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
@@ -2,137 +2,61 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1,
- compare_dbs/3
-]).
+-include("couch_replicator_test.hrl").
-define(TIMEOUT_EUNIT, 360).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("chttpd", "max_http_request_size", "10000", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
reduce_max_request_size_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs when target has a small max_http_request_size",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_all_docs/2}
- || Pair <- Pairs
- ] ++
- [
- {Pair, fun should_replicate_one/2}
- || Pair <- Pairs
- ] ++
- % Disabled. See issue 574. Sometimes PUTs with a doc and
- % attachment which exceed maximum request size are simply
- % closed instead of returning a 413 request. That makes these
- % tests flaky.
- [
- {Pair, fun should_replicate_one_with_attachment/2}
- || Pair <- Pairs
- ]
+ ?TDEF_FE(should_replicate_all_docs, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one_with_attachment, ?TIMEOUT_EUNIT)
+ ]
}
}.
% Test documents which are below max_http_request_size but when batched, batch size
% will be greater than max_http_request_size. Replicator could automatically split
% the batch into smaller batches and POST those separately.
-should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [])
- ]}
- }.
+should_replicate_all_docs({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, []).
% If a document is too large to post as a single request, that document is
% skipped but replication overall will make progress and not crash.
-should_replicate_one({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_one_small(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_one_small(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
% If a document has an attachment > 64 * 1024 bytes, replicator will switch to
% POST-ing individual documents directly and skip bulk_docs. Test that case
% separately
% See note in main test function why this was disabled.
-should_replicate_one_with_attachment({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_attachment(Source),
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one_with_attachment({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_attachment(Source),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
+
+populate_source(Source) ->
+ add_docs(Source, 5, 3000, 0).
+
+populate_source_one_large_one_small(Source) ->
+ one_large_one_small(Source, 12000, 3000).
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(add_docs(Source, 5, 3000, 0))}.
-
-should_populate_source_one_large_one_small({remote, Source}) ->
- should_populate_source_one_large_one_small(Source);
-should_populate_source_one_large_one_small(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_one_small(Source, 12000, 3000))}.
-
-should_populate_source_one_large_attachment({remote, Source}) ->
- should_populate_source_one_large_attachment(Source);
-should_populate_source_one_large_attachment(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_attachment(Source, 70000, 70000))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, {remote, Target}, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, Target, ExceptIds) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target, ExceptIds))}.
+populate_source_one_large_attachment(Source) ->
+ one_large_attachment(Source, 70000, 70000).
binary_chunk(Size) when is_integer(Size), Size > 0 ->
<<<<"x">> || _ <- lists:seq(1, Size)>>.
@@ -155,11 +79,9 @@ one_large_attachment(DbName, Size, AttSize) ->
add_doc(DbName, <<"doc0">>, Size, AttSize).
add_doc(DbName, DocId, Size, AttSize) when is_binary(DocId) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Doc0 = #doc{id = DocId, body = {[{<<"x">>, binary_chunk(Size)}]}},
Doc = Doc0#doc{atts = atts(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
atts(0) ->
[];
@@ -173,12 +95,18 @@ atts(Size) ->
])
].
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
- % This make batch_size predictable
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
+ % This makes batch_size more predictable
{<<"worker_processes">>, "1"}
]}
).
+
+compare(Source, Target, ExceptIds) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target, ExceptIds).
[couchdb] 26/31: Refactor hash algorithms test
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4d24d8989a44a83aab5b978932624ebe72f82f8f
Author: Ronny <ro...@apache.org>
AuthorDate: Thu Aug 25 19:29:22 2022 +0200
Refactor hash algorithms test
The test doesn't check if the hash algorithm is supported by the
erlang vm. The test for supported hash algorithms was only missing
in the test itself and not in CouchDB.
Refactor test and verify hash names during test runs.
---
.../eunit/chttpd_auth_hash_algorithms_tests.erl | 25 +++++++++-------
src/couch/include/couch_db.hrl | 2 ++
src/couch/src/couch_httpd_auth.erl | 35 ++--------------------
src/couch/src/couch_util.erl | 31 +++++++++++++++++++
4 files changed, 49 insertions(+), 44 deletions(-)
diff --git a/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
index 3d872aa46..c78427d24 100644
--- a/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
@@ -18,8 +18,8 @@
-define(ADM_USER, "adm_user").
-define(ADM_PASS, "adm_pass").
--define(ALLOWED_HASHES, "sha256, sha512, sha, blake2s").
--define(DISALLOWED_HASHES, "md4, md5, ripemd160").
+-define(WORKING_HASHES, "sha256, sha512, sha, blake2s").
+-define(FAILING_HASHES, "md4, md5, ripemd160").
hash_algorithms_test_() ->
{
@@ -43,12 +43,13 @@ setup() ->
config:set("admins", ?ADM_USER, ?b2l(Hashed), false),
config:set("chttpd_auth", "secret", NewSecret, false),
config:set("chttpd", "require_valid_user", "true", false),
- config:set("chttpd_auth", "hash_algorithms", ?ALLOWED_HASHES, false),
- AllowedHashes = re:split(config:get("chttpd_auth", "hash_algorithms"), "\\s*,\\s*", [
+ config:set("chttpd_auth", "hash_algorithms", ?WORKING_HASHES, false),
+ HashesShouldWork = re:split(config:get("chttpd_auth", "hash_algorithms"), "\\s*,\\s*", [
trim, {return, binary}
]),
- DisallowedHashes = re:split(?DISALLOWED_HASHES, "\\s*,\\s*", [trim, {return, binary}]),
- {Ctx, {AllowedHashes, DisallowedHashes}}.
+ HashesShouldFail = re:split(?FAILING_HASHES, "\\s*,\\s*", [trim, {return, binary}]),
+ SupportedHashAlgorithms = crypto:supports(hashs),
+ {Ctx, {HashesShouldWork, HashesShouldFail, SupportedHashAlgorithms}}.
teardown({Ctx, _}) ->
config:delete("chttpd_auth", "hash_algorithms", false),
@@ -83,7 +84,7 @@ test_hash_algorithm([], _) ->
test_hash_algorithm([DefaultHashAlgorithm | DecodingHashAlgorithmsList] = _, Status) ->
CurrentTime = couch_httpd_auth:make_cookie_time(),
Cookie = make_auth_session_string(
- erlang:binary_to_existing_atom(DefaultHashAlgorithm),
+ DefaultHashAlgorithm,
?ADM_USER,
get_full_secret(?ADM_USER),
CurrentTime
@@ -92,8 +93,10 @@ test_hash_algorithm([DefaultHashAlgorithm | DecodingHashAlgorithmsList] = _, Sta
?assertEqual(Status, ReqStatus),
test_hash_algorithm(DecodingHashAlgorithmsList, Status).
-test_hash_algorithms_should_work({_, {AllowedHashes, _}} = _) ->
- test_hash_algorithm(AllowedHashes, 200).
+test_hash_algorithms_should_work({_, {WorkingHashes, _, SupportedHashAlgorithms}} = _) ->
+ Hashes = couch_util:verify_hash_names(WorkingHashes, SupportedHashAlgorithms),
+ test_hash_algorithm(Hashes, 200).
-test_hash_algorithms_should_fail({_, {_, DisallowedHashes}} = _) ->
- test_hash_algorithm(DisallowedHashes, 401).
+test_hash_algorithms_should_fail({_, {_, FailingHashes, SupportedHashAlgorithms}} = _) ->
+ Hashes = couch_util:verify_hash_names(FailingHashes, SupportedHashAlgorithms),
+ test_hash_algorithm(Hashes, 401).
diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl
index 233836d16..e70706a7f 100644
--- a/src/couch/include/couch_db.hrl
+++ b/src/couch/include/couch_db.hrl
@@ -15,6 +15,8 @@
-define(DESIGN_DOC_PREFIX, "_design/").
-define(DEFAULT_COMPRESSION, snappy).
+-define(DEFAULT_HASH_ALGORITHM, sha256).
+
-define(MIN_STR, <<"">>).
-define(MAX_STR, <<255>>). % illegal utf string
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index e2cb02f8c..b3c984174 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -16,8 +16,6 @@
-include_lib("couch/include/couch_db.hrl").
--define(DEFAULT_HASH_ALGORITHM, sha256).
-
-export([party_mode_handler/1]).
-export([
@@ -298,7 +296,7 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) ->
end,
% Verify expiry and hash
CurrentTime = make_cookie_time(),
- HashAlgorithms = get_config_hash_algorithms(),
+ HashAlgorithms = couch_util:get_config_hash_algorithms(),
case chttpd_util:get_chttpd_auth_config("secret") of
undefined ->
couch_log:debug("cookie auth secret is not set", []),
@@ -373,7 +371,7 @@ cookie_auth_header(_Req, _Headers) ->
cookie_auth_cookie(Req, User, Secret, TimeStamp) ->
SessionData = User ++ ":" ++ erlang:integer_to_list(TimeStamp, 16),
- [HashAlgorithm | _] = get_config_hash_algorithms(),
+ [HashAlgorithm | _] = couch_util:get_config_hash_algorithms(),
Hash = couch_util:hmac(HashAlgorithm, Secret, SessionData),
mochiweb_cookies:cookie(
"AuthSession",
@@ -702,32 +700,3 @@ authentication_warning(#httpd{mochi_req = Req}, User) ->
"~p: Authentication failed for user ~s from ~s",
[?MODULE, User, Peer]
).
-
-verify_hash_names(HashAlgorithms, SupportedHashFun) ->
- verify_hash_names(HashAlgorithms, SupportedHashFun, []).
-verify_hash_names([], _, HashNames) ->
- lists:reverse(HashNames);
-verify_hash_names([H | T], SupportedHashFun, HashNames) ->
- try
- HashAtom = binary_to_existing_atom(H),
- Result =
- case lists:member(HashAtom, SupportedHashFun) of
- true -> [HashAtom | HashNames];
- false -> HashNames
- end,
- verify_hash_names(T, SupportedHashFun, Result)
- catch
- error:badarg ->
- couch_log:warning("~p: Hash algorithm ~s is not valid.", [?MODULE, H]),
- verify_hash_names(T, SupportedHashFun, HashNames)
- end.
-
--spec get_config_hash_algorithms() -> list(atom()).
-get_config_hash_algorithms() ->
- SupportedHashAlgorithms = crypto:supports(hashs),
- HashAlgorithmsStr = chttpd_util:get_chttpd_auth_config("hash_algorithms", "sha256, sha"),
- HashAlgorithms = re:split(HashAlgorithmsStr, "\\s*,\\s*", [trim, {return, binary}]),
- case verify_hash_names(HashAlgorithms, SupportedHashAlgorithms) of
- [] -> [?DEFAULT_HASH_ALGORITHM];
- VerifiedHashNames -> VerifiedHashNames
- end.
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index 84691d14e..e916bbc69 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -43,6 +43,8 @@
-export([set_process_priority/2]).
-export([hmac/3]).
-export([version_to_binary/1]).
+-export([verify_hash_names/2]).
+-export([get_config_hash_algorithms/0]).
-include_lib("couch/include/couch_db.hrl").
@@ -829,3 +831,32 @@ hex(X) ->
16#6530, 16#6531, 16#6532, 16#6533, 16#6534, 16#6535, 16#6536, 16#6537, 16#6538, 16#6539, 16#6561, 16#6562, 16#6563, 16#6564, 16#6565, 16#6566,
16#6630, 16#6631, 16#6632, 16#6633, 16#6634, 16#6635, 16#6636, 16#6637, 16#6638, 16#6639, 16#6661, 16#6662, 16#6663, 16#6664, 16#6665, 16#6666
}).
+
+verify_hash_names(HashAlgorithms, SupportedHashes) ->
+ verify_hash_names(HashAlgorithms, SupportedHashes, []).
+verify_hash_names([], _, HashNames) ->
+ lists:reverse(HashNames);
+verify_hash_names([H | T], SupportedHashes, HashNames) ->
+ try
+ HashAtom = binary_to_existing_atom(H),
+ Result =
+ case lists:member(HashAtom, SupportedHashes) of
+ true -> [HashAtom | HashNames];
+ false -> HashNames
+ end,
+ verify_hash_names(T, SupportedHashes, Result)
+ catch
+ error:badarg ->
+ couch_log:warning("~p: Hash algorithm ~s is not valid.", [?MODULE, H]),
+ verify_hash_names(T, SupportedHashes, HashNames)
+ end.
+
+-spec get_config_hash_algorithms() -> list(atom()).
+get_config_hash_algorithms() ->
+ SupportedHashes = crypto:supports(hashs),
+ HashAlgorithmsStr = chttpd_util:get_chttpd_auth_config("hash_algorithms", "sha256, sha"),
+ HashAlgorithms = re:split(HashAlgorithmsStr, "\\s*,\\s*", [trim, {return, binary}]),
+ case verify_hash_names(HashAlgorithms, SupportedHashes) of
+ [] -> [?DEFAULT_HASH_ALGORITHM];
+ VerifiedHashNames -> VerifiedHashNames
+ end.
[couchdb] 06/31: Update couch_replicator_error_reporting_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 410ed0bc3a36f01c021cf4efd1d3aa57af32df65
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:17:37 2022 -0400
Update couch_replicator_error_reporting_tests
Use the TDEF_FE macro and clean up ?_test(begin...end) cruft.
---
.../couch_replicator_error_reporting_tests.erl | 272 +++++++++------------
1 file changed, 111 insertions(+), 161 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
index b0863614c..7e198562f 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
@@ -15,149 +15,115 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
+error_reporting_test_() ->
+ {
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_fail_bulk_docs),
+ ?TDEF_FE(t_fail_changes_reader),
+ ?TDEF_FE(t_fail_revs_diff),
+ ?TDEF_FE(t_fail_changes_queue),
+ ?TDEF_FE(t_fail_changes_manager),
+ ?TDEF_FE(t_fail_changes_reader_proc)
+ ]
+ }.
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
+t_fail_bulk_docs({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
-setup() ->
- meck:unload(),
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
-teardown({Source, Target}) ->
- meck:unload(),
- teardown_db(Source),
- teardown_db(Target),
- ok.
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-error_reporting_test_() ->
- {
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_fail_bulk_docs/1,
- fun t_fail_changes_reader/1,
- fun t_fail_revs_diff/1,
- fun t_fail_changes_queue/1,
- fun t_fail_changes_manager/1,
- fun t_fail_changes_reader_proc/1
- ]
- }
- }.
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_revs_diff({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
-t_fail_bulk_docs({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_revs_diff({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_queue({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesQueue = element(20, State),
- ?assert(is_process_alive(ChangesQueue)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesQueue, boom),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_queue_died, boom}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_manager({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesManager = element(21, State),
- ?assert(is_process_alive(ChangesManager)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesManager, bam),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_manager_died, bam}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader_proc({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesReader = element(22, State),
- ?assert(is_process_alive(ChangesReader)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesReader, kapow),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_reader_died, kapow}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
+t_fail_changes_queue({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesQueue = element(20, State),
+ ?assert(is_process_alive(ChangesQueue)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesQueue, boom),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_queue_died, boom}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_manager({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesManager = element(21, State),
+ ?assert(is_process_alive(ChangesManager)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesManager, bam),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_manager_died, bam}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader_proc({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesReader = element(22, State),
+ ?assert(is_process_alive(ChangesReader)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesReader, kapow),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_reader_died, kapow}, Result),
+ couch_replicator_notifier:stop(Listener).
mock_fail_req(Path, Return) ->
meck:expect(
@@ -190,17 +156,7 @@ wait_rep_result(RepId) ->
{error, RepId, Reason} -> {error, Reason}
end.
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
populate_db(DbName, Start, End) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -210,14 +166,10 @@ populate_db(DbName, Start, End) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ {ok, SourceDocCount} = fabric:get_doc_count(Source),
wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
@@ -229,10 +181,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -242,12 +191,10 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
end.
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, url(Source)},
+ {<<"target">>, url(Target)},
{<<"continuous">>, true},
{<<"worker_processes">>, 1},
{<<"retries_per_request">>, 1},
@@ -258,3 +205,6 @@ replicate(Source, Target) ->
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
{ok, Rep#rep.id}.
+
+url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
[couchdb] 14/31: Update couch_replicator_rate_limiter_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bc8ede12a77bed2cb0d81d5e7cf59c40098830d8
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:35:50 2022 -0400
Update couch_replicator_rate_limiter_tests
Use the TDEF_FE macro and cleanup ?_test(begin...end) instances.
---
.../eunit/couch_replicator_rate_limiter_tests.erl | 77 ++++++++++------------
1 file changed, 33 insertions(+), 44 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
index a214d4607..0b7c48b62 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
@@ -1,6 +1,7 @@
-module(couch_replicator_rate_limiter_tests).
-include_lib("couch/include/couch_eunit.hrl").
+-include("couch_replicator_test.hrl").
rate_limiter_test_() ->
{
@@ -8,58 +9,46 @@ rate_limiter_test_() ->
fun setup/0,
fun teardown/1,
[
- t_new_key(),
- t_1_failure(),
- t_2_failures_back_to_back(),
- t_2_failures(),
- t_success_threshold(),
- t_1_failure_2_successes()
+ ?TDEF_FE(t_new_key),
+ ?TDEF_FE(t_1_failure),
+ ?TDEF_FE(t_2_failures_back_to_back),
+ ?TDEF_FE(t_2_failures),
+ ?TDEF_FE(t_success_threshold),
+ ?TDEF_FE(t_1_failure_2_successes)
]
}.
-t_new_key() ->
- ?_test(begin
- ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get}))
- end).
+t_new_key(_) ->
+ ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get})).
-t_1_failure() ->
- ?_test(begin
- ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get}))
- end).
+t_1_failure(_) ->
+ ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get})).
-t_2_failures() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(29, Interval)
- end).
+t_2_failures(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(29, Interval).
-t_2_failures_back_to_back() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(24, Interval)
- end).
+t_2_failures_back_to_back(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(24, Interval).
-t_success_threshold() ->
- ?_test(begin
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval),
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval)
- end).
+t_success_threshold(_) ->
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval),
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval).
-t_1_failure_2_successes() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(20, Succ1),
- low_pass_filter_delay(),
- Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Succ2)
- end).
+t_1_failure_2_successes(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(20, Succ1),
+ low_pass_filter_delay(),
+ Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Succ2).
low_pass_filter_delay() ->
timer:sleep(100).
[couchdb] 23/31: config section for require_valid_user is only [chttpd]
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 387f32ae5c1404a332258db7fc93ff9a05d3244a
Author: Ronny Berndt <ro...@apache.org>
AuthorDate: Mon Aug 22 16:43:07 2022 +0200
config section for require_valid_user is only [chttpd]
---
dev/run | 12 +++++-------
rel/overlay/etc/default.ini | 3 +--
rel/overlay/etc/local.ini | 6 ------
src/chttpd/src/chttpd.erl | 2 +-
src/couch/src/couch_httpd.erl | 2 +-
src/couch/src/couch_httpd_auth.erl | 4 ++--
src/global_changes/test/eunit/global_changes_hooks_tests.erl | 2 +-
7 files changed, 11 insertions(+), 20 deletions(-)
diff --git a/dev/run b/dev/run
index f87786058..32c68d1a2 100755
--- a/dev/run
+++ b/dev/run
@@ -472,15 +472,10 @@ def hack_default_ini(ctx, node, contents):
def hack_local_ini(ctx, contents):
- # make sure all three nodes have the same secret
- secret_line = "secret = %s\n" % COMMON_SALT
- previous_line = "; require_valid_user = false\n"
- contents = contents.replace(previous_line, previous_line + secret_line)
-
if ctx["with_admin_party"]:
os.environ["COUCHDB_TEST_ADMIN_PARTY_OVERRIDE"] = "1"
ctx["admin"] = ("Admin Party!", "You do not need any password.")
- return contents
+ return contents + "\n\n[chttpd_auth]\nsecret %s\n" % COMMON_SALT
# handle admin credentials passed from cli or generate own one
if ctx["admin"] is None:
@@ -488,7 +483,10 @@ def hack_local_ini(ctx, contents):
else:
user, pswd = ctx["admin"]
- return contents + "\n%s = %s" % (user, hashify(pswd))
+ # this relies on [admin] being the last section at the end of the file
+ contents = contents + "\n%s = %s" % (user, hashify(pswd))
+
+ return contents + "\n\n[chttpd_auth]\nsecret = %s\n" % COMMON_SALT
def gen_password():
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 316c7960c..b88dbcbce 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -247,7 +247,6 @@ bind_address = 127.0.0.1
; These options are moved from [couch_httpd_auth]
;authentication_redirect = /_utils/session.html
-;require_valid_user = false
;timeout = 600 ; number of seconds before automatic logout
;auth_cache_size = 50 ; size is number of cache entries
;allow_persistent_cookies = true ; set to false to disallow persistent cookies
@@ -323,7 +322,7 @@ bind_address = 127.0.0.1
authentication_db = _users
; These settings were moved to [chttpd_auth]
-; authentication_redirect, require_valid_user, timeout,
+; authentication_redirect, timeout,
; auth_cache_size, allow_persistent_cookies, iterations, min_iterations,
; max_iterations, password_scheme, password_regexp, proxy_use_secret,
; public_fields, secret, users_db_public, cookie_domain, same_site
diff --git a/rel/overlay/etc/local.ini b/rel/overlay/etc/local.ini
index 4c847617c..17353a368 100644
--- a/rel/overlay/etc/local.ini
+++ b/rel/overlay/etc/local.ini
@@ -43,12 +43,6 @@
; the whitelist.
;config_whitelist = [{httpd,config_whitelist}, {log,level}, {etc,etc}]
-[chttpd_auth]
-; If you set this to true, you should also uncomment the WWW-Authenticate line
-; above. If you don't configure a WWW-Authenticate header, CouchDB will send
-; Basic realm="server" in order to prevent you getting logged out.
-; require_valid_user = false
-
[ssl]
;enable = true
;cert_file = /full/path/to/server_cert.pem
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 93b610719..13e919cb5 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1173,7 +1173,7 @@ error_headers(#httpd{mochi_req = MochiReq} = Req, 401 = Code, ErrorStr, ReasonSt
{Code, []};
AuthRedirect ->
case
- chttpd_util:get_chttpd_auth_config_boolean(
+ chttpd_util:get_chttpd_config_boolean(
"require_valid_user", false
)
of
diff --git a/src/couch/src/couch_httpd.erl b/src/couch/src/couch_httpd.erl
index 39faea418..76f8279f6 100644
--- a/src/couch/src/couch_httpd.erl
+++ b/src/couch/src/couch_httpd.erl
@@ -1080,7 +1080,7 @@ error_headers(#httpd{mochi_req = MochiReq} = Req, Code, ErrorStr, ReasonStr) ->
{Code, []};
AuthRedirect ->
case
- chttpd_util:get_chttpd_auth_config_boolean(
+ chttpd_util:get_chttpd_config_boolean(
"require_valid_user", false
)
of
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index c74ca9bd8..a5a876b18 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -43,7 +43,7 @@
party_mode_handler(Req) ->
case
- chttpd_util:get_chttpd_auth_config_boolean(
+ chttpd_util:get_chttpd_config_boolean(
"require_valid_user", false
)
of
@@ -131,7 +131,7 @@ default_authentication_handler(Req, AuthModule) ->
Req;
false ->
case
- chttpd_util:get_chttpd_auth_config_boolean(
+ chttpd_util:get_chttpd_config_boolean(
"require_valid_user", false
)
of
diff --git a/src/global_changes/test/eunit/global_changes_hooks_tests.erl b/src/global_changes/test/eunit/global_changes_hooks_tests.erl
index 5d6bbd13d..4872da82b 100644
--- a/src/global_changes/test/eunit/global_changes_hooks_tests.erl
+++ b/src/global_changes/test/eunit/global_changes_hooks_tests.erl
@@ -34,7 +34,7 @@ stop({Ctx, DbName}) ->
setup(default) ->
add_admin("admin", <<"pass">>),
config:delete("chttpd_auth", "authentication_redirect", false),
- config:set("chttpd_auth", "require_valid_user", "false", false),
+ config:set("chttpd", "require_valid_user", "false", false),
get_host();
setup(A) ->
Host = setup(default),
[couchdb] 29/31: Implement _bulk_get support for the replicator
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 88d7f23a8aecf0841d77a37c81b595c8fdd394a3
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:00:13 2022 -0400
Implement _bulk_get support for the replicator
By now most of the CouchDB implementations support `_bulk_get`, so
let's update the replicator to take advantage of that.
To be backwards compatible assume some endpoints will not support
`_bulk_get` and may return either a 500 or 400 error. In that case the
replicator will fall back to fetching individual document revisions
like it did previously. For additional backward compatibility, and to
keep things simple, support only the `application/json` `_bulk_get`
response format. (Ideally, we'd send multiple Accept headers with
various `q` preference parameters for `json` and `multipart/related`
content, then do the right thing based on the response, however, none
of the recent Apache CouchDB implementations support that scheme
properly).
Since fetching attachments with application/json response is not
optimal, attachments are fetched individually. This means there are
two main reasons for the replicator to fall back to fetching
individual revisions: 1) when _bulk_get endpoint is not supported and
2) when the document revisions contain attachments.
To avoid wasting resource repeatedly attempting to use `_bulk_get `and
then falling back to individual doc fetches, maintain some historical
stats about the rate of failure, and if it crosses a threshold, skip
calling `_bulk_get` altogether. This is implemented with a moving
exponential average, along with periodic probing to see if `_bulk_get`
usage becomes viable again.
To give the users some indication about how successful `_bulk_get`
usage is, introduce two replication statistics parameters:
* `bulk_get_attempts`: _bulk_get document revisions attempts made.
* `bulk_get_docs` : `_bulk_get` document revisions successfully retrieved.
These are persisted in the replication checkpoints along with the rest
of the job statistics and visible in `_scheduler/jobs` and
`_active_tasks` output.
Since we updated the replication job statistics, perform some minor
cleanups in that area:
- Stop using the process dictionary for the reporting timestamp. Use
a regular record state field instead.
- Use casts instead of a calls when possible. We still rely on
report_seq_done calls as a synchronization point to make sure we
don't overrun the message queues for the replication worker and
scheduler job process.
- Add stats update API functions instead of relying on naked
`gen_server` calls and casts. The functions make it clear which
process is being updated: the replication worker or the main
replication scheduler job process.
For testing, rely on the variety of existing replication tests running
and passing. The recently merged replication test overhaul from [pull
the tests form using the node-local (back-end API) to chttpd (the
cluster API), which actually implements `_bulk_get`. In this way, the
majority of replication tests should test the `_bulk_get` API usage
alongside whatever else they are testing. There there is new test
checking that `_bulk_get` fallback works and testing the
characteristics of the new statistics parameters.
---
.../src/couch_replicator_api_wrap.erl | 83 +++++-
.../src/couch_replicator_changes_reader.erl | 2 +-
.../src/couch_replicator_httpc.erl | 4 +
.../src/couch_replicator_scheduler_job.erl | 30 ++-
.../src/couch_replicator_stats.erl | 14 +-
.../src/couch_replicator_worker.erl | 294 ++++++++++++++++-----
.../couch_replicator_error_reporting_tests.erl | 26 ++
...ch_replicator_retain_stats_between_job_runs.erl | 6 +
8 files changed, 382 insertions(+), 77 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_api_wrap.erl b/src/couch_replicator/src/couch_replicator_api_wrap.erl
index 3c956c8de..a6e39cb02 100644
--- a/src/couch_replicator/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator/src/couch_replicator_api_wrap.erl
@@ -35,6 +35,7 @@
update_docs/4,
ensure_full_commit/1,
get_missing_revs/2,
+ bulk_get/3,
open_doc/3,
open_doc_revs/6,
changes_since/5,
@@ -206,6 +207,74 @@ get_missing_revs(#httpdb{} = Db, IdRevs) ->
end
).
+bulk_get(#httpdb{} = Db, #{} = IdRevs, Options) ->
+ FoldFun = fun({Id, Rev}, PAs, Acc) -> [{Id, Rev, PAs} | Acc] end,
+ ReqDocsList = lists:sort(maps:fold(FoldFun, [], IdRevs)),
+ MapFun = fun({Id, Rev, PAs}) ->
+ #{
+ <<"id">> => Id,
+ <<"rev">> => couch_doc:rev_to_str(Rev),
+ <<"atts_since">> => couch_doc:revs_to_strs(PAs)
+ }
+ end,
+ ReqDocsMaps = lists:map(MapFun, ReqDocsList),
+ % We are also sending request parameters in the doc body with the hopes
+ % that at some point in the future we could make that the default, instead
+ % of having to send query parameters with a POST request as we do today
+ Body = options_to_json_map(Options, #{<<"docs">> => ReqDocsMaps}),
+ Req = [
+ {method, post},
+ {path, "_bulk_get"},
+ {qs, options_to_query_args(Options, [])},
+ {body, ?JSON_ENCODE(Body)},
+ {headers, [
+ {"Content-Type", "application/json"},
+ {"Accept", "application/json"}
+ ]}
+ ],
+ try
+ send_req(Db, Req, fun
+ (200, _, {[{<<"results">>, Res}]}) when is_list(Res) ->
+ Zip = lists:zipwith(fun bulk_get_zip/2, ReqDocsList, Res),
+ {ok, maps:from_list(Zip)};
+ (200, _, _) ->
+ {error, {bulk_get_failed, invalid_results}};
+ (ErrCode, _, _) when is_integer(ErrCode) ->
+ % On older Apache CouchDB instances where _bulk_get is not
+ % implemented we would hit the POST db/doc form uploader
+ % handler. When that fails the request body is not consumed and
+ % we'd end up recycling a worker with an unsent body in the
+ % connection stream. Instead of waiting for it to blow up
+ % eventually and consuming an extra retry attempt, proactively
+ % advise httpc logic to stop this worker and not return back to
+ % the pool.
+ couch_replicator_httpc:stop_http_worker(),
+ {error, {bulk_get_failed, ErrCode}}
+ end)
+ catch
+ exit:{http_request_failed, _, _, {error, {code, ErrCode}}} ->
+ % We are being a bit more tolerant of _bulk_get errors as we can
+ % always fallback to individual fetches
+ {error, {bulk_get_failed, ErrCode}}
+ end.
+
+bulk_get_zip({Id, Rev, _}, {[_ | _] = Props}) ->
+ Docs = couch_util:get_value(<<"docs">>, Props),
+ ResId = couch_util:get_value(<<"id">>, Props),
+ % "docs" is a one item list, either [{"ok": Doc}] or [{"error": Error}]
+ case Docs of
+ [{[{<<"ok">>, {[_ | _]} = Doc}]}] when ResId =:= Id ->
+ {{Id, Rev}, couch_doc:from_json_obj(Doc)};
+ [{[{<<"error">>, {[_ | _] = Err}}]}] when ResId =:= Id ->
+ Tag = couch_util:get_value(<<"error">>, Err),
+ Reason = couch_util:get_value(<<"reason">>, Err),
+ couch_log:debug("~p bulk_get zip error ~p:~p", [?MODULE, Tag, Reason]),
+ {{Id, Rev}, {error, {Tag, Reason}}};
+ Other ->
+ couch_log:debug("~p bulk_get zip other error:~p", [?MODULE, Other]),
+ {{Id, Rev}, {error, {unexpected_bulk_get_response, Other}}}
+ end.
+
-spec open_doc_revs(#httpdb{}, binary(), list(), list(), function(), any()) -> no_return().
open_doc_revs(#httpdb{retries = 0} = HttpDb, Id, Revs, Options, _Fun, _Acc) ->
Path = encode_doc_id(Id),
@@ -647,7 +716,19 @@ options_to_query_args([latest | Rest], Acc) ->
options_to_query_args(Rest, [{"latest", "true"} | Acc]);
options_to_query_args([{open_revs, Revs} | Rest], Acc) ->
JsonRevs = ?b2l(iolist_to_binary(?JSON_ENCODE(couch_doc:revs_to_strs(Revs)))),
- options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]).
+ options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]);
+options_to_query_args([{attachments, Bool} | Rest], Acc) when is_atom(Bool) ->
+ BoolStr = atom_to_list(Bool),
+ options_to_query_args(Rest, [{"attachments", BoolStr} | Acc]).
+
+options_to_json_map([], #{} = Acc) ->
+ Acc;
+options_to_json_map([latest | Rest], #{} = Acc) ->
+ options_to_json_map(Rest, Acc#{<<"latest">> => true});
+options_to_json_map([revs | Rest], #{} = Acc) ->
+ options_to_json_map(Rest, Acc#{<<"revs">> => true});
+options_to_json_map([{attachments, Bool} | Rest], #{} = Acc) when is_atom(Bool) ->
+ options_to_json_map(Rest, Acc#{<<"attachments">> => Bool}).
atts_since_arg(_UrlLen, [], _MaxLen, Acc) ->
lists:reverse(Acc);
diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl b/src/couch_replicator/src/couch_replicator_changes_reader.erl
index 7fa8c26c2..bb6733608 100644
--- a/src/couch_replicator/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl
@@ -119,7 +119,7 @@ process_change(#doc_info{id = Id} = DocInfo, {Parent, Db, ChangesQueue, _}) ->
[Id, SourceDb]
),
Stats = couch_replicator_stats:new([{doc_write_failures, 1}]),
- ok = gen_server:call(Parent, {add_stats, Stats}, infinity);
+ ok = couch_replicator_scheduler_job:sum_stats(Parent, Stats);
false ->
ok = couch_work_queue:queue(ChangesQueue, DocInfo),
put(last_seq, DocInfo#doc_info.high_seq)
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl
index c6f22468d..cd5e4d75d 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -18,6 +18,7 @@
-export([setup/1]).
-export([send_req/3]).
+-export([stop_http_worker/0]).
-export([full_url/2]).
-import(couch_util, [
@@ -102,6 +103,9 @@ send_req(HttpDb, Params1, Callback) ->
Ret
end.
+stop_http_worker() ->
+ put(?STOP_HTTP_WORKER, stop).
+
send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
Method = get_value(method, Params, get),
UserHeaders = get_value(headers, Params, []),
diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
index 2ae8718ad..38de8a45a 100644
--- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl
+++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl
@@ -25,7 +25,9 @@
handle_info/2,
handle_cast/2,
code_change/3,
- format_status/2
+ format_status/2,
+ sum_stats/2,
+ report_seq_done/3
]).
-include_lib("couch/include/couch_db.hrl").
@@ -181,20 +183,13 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx = UserCtx} = Rep) -
workers = Workers
}}.
-handle_call({add_stats, Stats}, From, State) ->
- gen_server:reply(From, ok),
- NewStats = couch_replicator_utils:sum_stats(State#rep_state.stats, Stats),
- {noreply, State#rep_state{stats = NewStats}};
-handle_call(
- {report_seq_done, Seq, StatsInc},
- From,
+handle_call({report_seq_done, Seq, StatsInc}, From, State) ->
#rep_state{
seqs_in_progress = SeqsInProgress,
highest_seq_done = HighestDone,
current_through_seq = ThroughSeq,
stats = Stats
- } = State
-) ->
+ } = State,
gen_server:reply(From, ok),
{NewThroughSeq0, NewSeqsInProgress} =
case SeqsInProgress of
@@ -237,6 +232,9 @@ handle_call(
update_task(NewState),
{noreply, NewState}.
+handle_cast({sum_stats, Stats}, State) ->
+ NewStats = couch_replicator_utils:sum_stats(State#rep_state.stats, Stats),
+ {noreply, State#rep_state{stats = NewStats}};
handle_cast(checkpoint, State) ->
case do_checkpoint(State) of
{ok, NewState} ->
@@ -467,6 +465,12 @@ format_status(_Opt, [_PDict, State]) ->
{highest_seq_done, HighestSeqDone}
].
+sum_stats(Pid, Stats) when is_pid(Pid) ->
+ gen_server:cast(Pid, {sum_stats, Stats}).
+
+report_seq_done(Pid, ReportSeq, Stats) when is_pid(Pid) ->
+ gen_server:call(Pid, {report_seq_done, ReportSeq, Stats}, infinity).
+
startup_jitter() ->
Jitter = config:get_integer(
"replicator",
@@ -792,7 +796,9 @@ do_checkpoint(State) ->
{<<"missing_found">>, couch_replicator_stats:missing_found(Stats)},
{<<"docs_read">>, couch_replicator_stats:docs_read(Stats)},
{<<"docs_written">>, couch_replicator_stats:docs_written(Stats)},
- {<<"doc_write_failures">>, couch_replicator_stats:doc_write_failures(Stats)}
+ {<<"doc_write_failures">>, couch_replicator_stats:doc_write_failures(Stats)},
+ {<<"bulk_get_docs">>, couch_replicator_stats:bulk_get_docs(Stats)},
+ {<<"bulk_get_attempts">>, couch_replicator_stats:bulk_get_attempts(Stats)}
]},
BaseHistory =
[
@@ -1056,6 +1062,8 @@ rep_stats(State) ->
{docs_written, couch_replicator_stats:docs_written(Stats)},
{changes_pending, get_pending_count(State)},
{doc_write_failures, couch_replicator_stats:doc_write_failures(Stats)},
+ {bulk_get_docs, couch_replicator_stats:bulk_get_docs(Stats)},
+ {bulk_get_attempts, couch_replicator_stats:bulk_get_attempts(Stats)},
{checkpointed_source_seq, CommittedSeq}
].
diff --git a/src/couch_replicator/src/couch_replicator_stats.erl b/src/couch_replicator/src/couch_replicator_stats.erl
index e1f23a1bc..ff8f1b62f 100644
--- a/src/couch_replicator/src/couch_replicator_stats.erl
+++ b/src/couch_replicator/src/couch_replicator_stats.erl
@@ -26,7 +26,9 @@
missing_found/1,
docs_read/1,
docs_written/1,
- doc_write_failures/1
+ doc_write_failures/1,
+ bulk_get_docs/1,
+ bulk_get_attempts/1
]).
new() ->
@@ -51,6 +53,12 @@ docs_written(Stats) ->
doc_write_failures(Stats) ->
get(doc_write_failures, Stats).
+bulk_get_docs(Stats) ->
+ get(bulk_get_docs, Stats).
+
+bulk_get_attempts(Stats) ->
+ get(bulk_get_attempts, Stats).
+
get(Field, Stats) ->
case orddict:find(Field, Stats) of
{ok, Value} ->
@@ -84,4 +92,8 @@ fmap({docs_written, _}) -> true;
fmap({<<"docs_written">>, V}) -> {true, {docs_written, V}};
fmap({doc_write_failures, _}) -> true;
fmap({<<"doc_write_failures">>, V}) -> {true, {doc_write_failures, V}};
+fmap({bulk_get_docs, _}) -> true;
+fmap({<<"bulk_get_docs">>, V}) -> {true, {bulk_get_docs, V}};
+fmap({bulk_get_attempts, _}) -> true;
+fmap({<<"bulk_get_attempts">>, V}) -> {true, {bulk_get_attempts, V}};
fmap({_, _}) -> false.
diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl
index 3c6d6d040..94e3e028b 100644
--- a/src/couch_replicator/src/couch_replicator_worker.erl
+++ b/src/couch_replicator/src/couch_replicator_worker.erl
@@ -25,13 +25,12 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
-% TODO: maybe make both buffer max sizes configurable
-
-% for remote targets
-define(DOC_BUFFER_BYTE_SIZE, 512 * 1024).
-% 10 seconds (in microseconds)
--define(STATS_DELAY, 10000000).
+-define(STATS_DELAY_SEC, 10).
-define(MISSING_DOC_RETRY_MSEC, 2000).
+-define(BULK_GET_RATIO_THRESHOLD, 0.5).
+-define(BULK_GET_RATIO_DECAY, 0.25).
+-define(BULK_GET_RETRY_SEC, 37).
-import(couch_util, [
to_binary/1,
@@ -54,9 +53,24 @@
pending_fetch = nil,
flush_waiter = nil,
stats = couch_replicator_stats:new(),
+ last_stats_report_sec = 0,
batch = #batch{}
}).
+-record(bulk_get_stats, {
+ ratio = 0,
+ tsec = 0
+}).
+
+-record(fetch_st, {
+ source,
+ target,
+ parent,
+ cp,
+ changes_manager,
+ bulk_get_stats
+}).
+
start_link(Cp, #httpdb{} = Source, Target, ChangesManager, MaxConns) ->
gen_server:start_link(
?MODULE, {Cp, Source, Target, ChangesManager, MaxConns}, []
@@ -64,17 +78,22 @@ start_link(Cp, #httpdb{} = Source, Target, ChangesManager, MaxConns) ->
init({Cp, Source, Target, ChangesManager, MaxConns}) ->
process_flag(trap_exit, true),
- Parent = self(),
- LoopPid = spawn_link(fun() ->
- queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager)
- end),
- erlang:put(last_stats_report, os:timestamp()),
+ NowSec = erlang:monotonic_time(second),
+ FetchSt = #fetch_st{
+ cp = Cp,
+ source = Source,
+ target = Target,
+ parent = self(),
+ changes_manager = ChangesManager,
+ bulk_get_stats = #bulk_get_stats{ratio = 0, tsec = NowSec}
+ },
State = #state{
cp = Cp,
max_parallel_conns = MaxConns,
- loop = LoopPid,
+ loop = spawn_link(fun() -> queue_fetch_loop(FetchSt) end),
source = Source,
- target = Target
+ target = Target,
+ last_stats_report_sec = NowSec
},
{ok, State}.
@@ -106,11 +125,6 @@ handle_call(
handle_call({batch_doc, Doc}, From, State) ->
gen_server:reply(From, ok),
{noreply, maybe_flush_docs(Doc, State)};
-handle_call({add_stats, IncStats}, From, #state{stats = Stats} = State) ->
- gen_server:reply(From, ok),
- NewStats = couch_replicator_utils:sum_stats(Stats, IncStats),
- NewStats2 = maybe_report_stats(State#state.cp, NewStats),
- {noreply, State#state{stats = NewStats2}};
handle_call(
flush,
{Pid, _} = From,
@@ -131,8 +145,11 @@ handle_call(
end,
{noreply, State2#state{flush_waiter = From}}.
+handle_cast({sum_stats, IncStats}, #state{stats = Stats} = State) ->
+ SumStats = couch_replicator_utils:sum_stats(Stats, IncStats),
+ {noreply, maybe_report_stats(State#state{stats = SumStats})};
handle_cast(Msg, State) ->
- {stop, {unexpected_async_call, Msg}, State}.
+ {stop, {unexpected_cast, Msg}, State}.
handle_info({'EXIT', Pid, normal}, #state{loop = Pid} = State) ->
#state{
@@ -188,6 +205,8 @@ handle_info({'EXIT', _Pid, max_backoff}, State) ->
{stop, {shutdown, max_backoff}, State};
handle_info({'EXIT', _Pid, {bulk_docs_failed, _, _} = Err}, State) ->
{stop, {shutdown, Err}, State};
+handle_info({'EXIT', _Pid, {bulk_get_failed, _, _} = Err}, State) ->
+ {stop, {shutdown, Err}, State};
handle_info({'EXIT', _Pid, {revs_diff_failed, _, _} = Err}, State) ->
{stop, {shutdown, Err}, State};
handle_info({'EXIT', _Pid, {http_request_failed, _, _, _} = Err}, State) ->
@@ -221,40 +240,109 @@ format_status(_Opt, [_PDict, State]) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) ->
+sum_stats(Pid, Stats) when is_pid(Pid) ->
+ ok = gen_server:cast(Pid, {sum_stats, Stats}).
+
+report_seq_done(Cp, Seq) ->
+ ok = report_seq_done(Cp, Seq, couch_replicator_stats:new()).
+
+report_seq_done(Cp, Seq, Stats) ->
+ ok = couch_replicator_scheduler_job:report_seq_done(Cp, Seq, Stats).
+
+queue_fetch_loop(#fetch_st{} = St) ->
+ #fetch_st{
+ cp = Cp,
+ source = Source,
+ target = Target,
+ parent = Parent,
+ changes_manager = ChangesManager,
+ bulk_get_stats = BgSt
+ } = St,
ChangesManager ! {get_changes, self()},
receive
{closed, ChangesManager} ->
ok;
{changes, ChangesManager, [], ReportSeq} ->
- Stats = couch_replicator_stats:new(),
- ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity),
- queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager);
+ ok = report_seq_done(Cp, ReportSeq),
+ queue_fetch_loop(St);
{changes, ChangesManager, Changes, ReportSeq} ->
- {IdRevs, Stats0} = find_missing(Changes, Target),
- ok = gen_server:call(Parent, {add_stats, Stats0}, infinity),
- remote_process_batch(IdRevs, Parent),
+ % Find missing revisions (POST to _revs_diff)
+ IdRevs = find_missing(Changes, Target, Parent),
+ {Docs, BgSt1} = bulk_get(Source, IdRevs, Parent, BgSt),
+ % Documents without attachments can be uploaded right away
+ BatchFun = fun({_, #doc{} = Doc}) ->
+ ok = gen_server:call(Parent, {batch_doc, Doc}, infinity)
+ end,
+ lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))),
+ % Fetch individually if _bulk_get failed or there are attachments
+ FetchFun = fun({Id, Rev}, PAs) ->
+ ok = gen_server:call(Parent, {fetch_doc, {Id, [Rev], PAs}}, infinity)
+ end,
+ maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs)),
{ok, Stats} = gen_server:call(Parent, flush, infinity),
- ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity),
- erlang:put(last_stats_report, os:timestamp()),
+ ok = report_seq_done(Cp, ReportSeq, Stats),
couch_log:debug("Worker reported completion of seq ~p", [ReportSeq]),
- queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager)
+ queue_fetch_loop(St#fetch_st{bulk_get_stats = BgSt1})
end.
-remote_process_batch([], _Parent) ->
- ok;
-remote_process_batch([{Id, Revs, PAs} | Rest], Parent) ->
- % When the source is a remote database, we fetch a single document revision
- % per HTTP request. This is mostly to facilitate retrying of HTTP requests
- % due to network transient failures. It also helps not exceeding the maximum
- % URL length allowed by proxies and Mochiweb.
- lists:foreach(
- fun(Rev) ->
- ok = gen_server:call(Parent, {fetch_doc, {Id, [Rev], PAs}}, infinity)
+% Return revisions without attachments. Maintain an exponential moving failure
+% ratio. When the ratio becomes greater than the threshold, skip calling
+% bulk_get altogether. To avoid getting permanently stuck with a high failure
+% ratio after replicating lots of attachments, periodically attempt to use
+% _bulk_get. After a few successful attempts that should lower the failure rate
+% enough to start allow using _bulk_get again.
+%
+bulk_get(Source, IdRevs, Parent, #bulk_get_stats{} = St) ->
+ NowSec = erlang:monotonic_time(second),
+ case attempt_bulk_get(St, NowSec) of
+ true ->
+ Docs = bulk_get(Source, IdRevs),
+ Attempts = map_size(IdRevs),
+ Successes = map_size(Docs),
+ Stats = couch_replicator_stats:new([
+ {bulk_get_docs, Successes},
+ {bulk_get_attempts, Attempts}
+ ]),
+ ok = sum_stats(Parent, Stats),
+ St1 = update_bulk_get_ratio(St, Successes, Attempts),
+ {Docs, St1#bulk_get_stats{tsec = NowSec}};
+ false ->
+ {#{}, St}
+ end.
+
+bulk_get(#httpdb{} = Source, #{} = IdRevs) ->
+ Opts = [latest, revs, {attachments, false}],
+ case couch_replicator_api_wrap:bulk_get(Source, IdRevs, Opts) of
+ {ok, #{} = Docs} ->
+ FilterFun = fun
+ (_, #doc{atts = []}) -> true;
+ (_, #doc{atts = [_ | _]}) -> false;
+ (_, {error, _}) -> false
+ end,
+ maps:filter(FilterFun, Docs);
+ {error, Error} ->
+ couch_log:debug("_bulk_get failed ~p", [Error]),
+ #{}
+ end.
+
+attempt_bulk_get(#bulk_get_stats{} = St, NowSec) ->
+ #bulk_get_stats{tsec = TSec, ratio = Ratio} = St,
+ TimeThreshold = (NowSec - TSec) > ?BULK_GET_RETRY_SEC,
+ RatioThreshold = Ratio =< ?BULK_GET_RATIO_THRESHOLD,
+ TimeThreshold orelse RatioThreshold.
+
+% Update fail ratio. Use the basic exponential moving average formula to smooth
+% over minor bumps in case we encounter a few % attachments and then get back
+% to replicationg documents without attachments.
+%
+update_bulk_get_ratio(#bulk_get_stats{} = St, Successes, Attempts) ->
+ #bulk_get_stats{ratio = Avg} = St,
+ Ratio =
+ case Attempts > 0 of
+ true -> (Attempts - Successes) / Attempts;
+ false -> 0
end,
- Revs
- ),
- remote_process_batch(Rest, Parent).
+ St#bulk_get_stats{ratio = ?BULK_GET_RATIO_DECAY * (Ratio - Avg) + Avg}.
-spec spawn_doc_reader(#httpdb{}, #httpdb{}, {list(), list(), list()}) -> no_return().
spawn_doc_reader(Source, Target, FetchParams) ->
@@ -331,7 +419,7 @@ doc_handler_flush_doc(#doc{} = Doc, {Parent, Target} = Acc) ->
false ->
{{skip, Acc}, couch_replicator_stats:increment(doc_write_failures, Stats)}
end,
- ok = gen_server:call(Parent, {add_stats, Stats2}, infinity),
+ ok = sum_stats(Parent, Stats2),
Result.
spawn_writer(Target, #batch{docs = DocList, size = Size}) ->
@@ -345,32 +433,30 @@ spawn_writer(Target, #batch{docs = DocList, size = Size}) ->
spawn_link(
fun() ->
Stats = flush_docs(Target, DocList),
- ok = gen_server:call(Parent, {add_stats, Stats}, infinity)
+ ok = sum_stats(Parent, Stats)
end
).
after_full_flush(#state{stats = Stats, flush_waiter = Waiter} = State) ->
gen_server:reply(Waiter, {ok, Stats}),
- erlang:put(last_stats_report, os:timestamp()),
State#state{
stats = couch_replicator_stats:new(),
flush_waiter = nil,
writer = nil,
- batch = #batch{}
+ batch = #batch{},
+ last_stats_report_sec = erlang:monotonic_time(second)
}.
maybe_flush_docs(Doc, State) ->
#state{
target = Target,
batch = Batch,
- stats = Stats,
- cp = Cp
+ stats = Stats
} = State,
{Batch2, WStats} = maybe_flush_docs(Target, Batch, Doc),
Stats2 = couch_replicator_stats:sum_stats(Stats, WStats),
Stats3 = couch_replicator_stats:increment(docs_read, Stats2),
- Stats4 = maybe_report_stats(Cp, Stats3),
- State#state{stats = Stats4, batch = Batch2}.
+ maybe_report_stats(State#state{stats = Stats3, batch = Batch2}).
maybe_flush_docs(#httpdb{} = Target, Batch, Doc) ->
#batch{docs = DocAcc, size = SizeAcc} = Batch,
@@ -481,7 +567,7 @@ flush_doc(Target, #doc{id = Id, revs = {Pos, [RevId | _]}} = Doc) ->
{error, Err}
end.
-find_missing(DocInfos, Target) ->
+find_missing(DocInfos, Target, Parent) ->
{IdRevs, AllRevsCount} = lists:foldr(
fun
(#doc_info{revs = []}, {IdRevAcc, CountAcc}) ->
@@ -504,21 +590,37 @@ find_missing(DocInfos, Target) ->
0,
Missing
),
- Stats = couch_replicator_stats:new([
- {missing_checked, AllRevsCount},
- {missing_found, MissingRevsCount}
- ]),
- {Missing, Stats}.
-
-maybe_report_stats(Cp, Stats) ->
- Now = os:timestamp(),
- case timer:now_diff(erlang:get(last_stats_report), Now) >= ?STATS_DELAY of
+ ok = sum_stats(
+ Parent,
+ couch_replicator_stats:new([
+ {missing_checked, AllRevsCount},
+ {missing_found, MissingRevsCount}
+ ])
+ ),
+ % Turn {Id, [Rev1, Rev2, ...], PAs} into a map:
+ % #{{Id, Rev1} => PAs, {Id, Rev2} => PAs, ...}
+ id_rev_map(Missing).
+
+id_rev_map(IdRevs) ->
+ id_rev_map(IdRevs, #{}).
+
+id_rev_map([], #{} = Acc) ->
+ Acc;
+id_rev_map([{_, [], _} | Docs], #{} = Acc) ->
+ id_rev_map(Docs, Acc);
+id_rev_map([{Id, [Rev | Revs], PAs} | Docs], #{} = Acc) ->
+ id_rev_map([{Id, Revs, PAs} | Docs], Acc#{{Id, Rev} => PAs}).
+
+maybe_report_stats(#state{} = State) ->
+ #state{cp = Cp, stats = Stats, last_stats_report_sec = LastReport} = State,
+ Now = erlang:monotonic_time(second),
+ case Now - LastReport >= ?STATS_DELAY_SEC of
true ->
- ok = gen_server:call(Cp, {add_stats, Stats}, infinity),
- erlang:put(last_stats_report, Now),
- couch_replicator_stats:new();
+ ok = couch_replicator_scheduler_job:sum_stats(Cp, Stats),
+ NewStats = couch_replicator_stats:new(),
+ State#state{stats = NewStats, last_stats_report_sec = Now};
false ->
- Stats
+ State
end.
-ifdef(TEST).
@@ -544,4 +646,70 @@ replication_worker_format_status_test() ->
?assertEqual(nil, proplists:get_value(pending_fetch, Format)),
?assertEqual(5, proplists:get_value(batch_size, Format)).
+bulk_get_attempt_test() ->
+ Now = erlang:monotonic_time(second),
+ St = #bulk_get_stats{ratio = 0, tsec = Now},
+ ?assert(attempt_bulk_get(St#bulk_get_stats{ratio = 0.1}, Now)),
+ ?assertNot(attempt_bulk_get(St#bulk_get_stats{ratio = 0.9}, Now)),
+ RetryTime = Now + ?BULK_GET_RETRY_SEC + 1,
+ ?assert(attempt_bulk_get(St#bulk_get_stats{ratio = 0.9}, RetryTime)).
+
+update_bulk_get_ratio_test() ->
+ Init = #bulk_get_stats{ratio = 0, tsec = 0},
+
+ % Almost all failures
+ Fail = lists:foldl(
+ fun(_, Acc) ->
+ update_bulk_get_ratio(Acc, 1, 1000)
+ end,
+ Init,
+ lists:seq(1, 100)
+ ),
+ ?assert(Fail#bulk_get_stats.ratio > 0.9),
+
+ % Almost all successes
+ Success = lists:foldl(
+ fun(_, Acc) ->
+ update_bulk_get_ratio(Acc, 900, 1000)
+ end,
+ Init,
+ lists:seq(1, 100)
+ ),
+ ?assert(Success#bulk_get_stats.ratio < 0.1),
+
+ % Half and half
+ Half = lists:foldl(
+ fun(_, Acc) ->
+ update_bulk_get_ratio(Acc, 500, 1000)
+ end,
+ Init,
+ lists:seq(1, 100)
+ ),
+ ?assert(Half#bulk_get_stats.ratio > 0.49),
+ ?assert(Half#bulk_get_stats.ratio < 0.51),
+
+ % Successes after failures
+ FailThenSuccess = lists:foldl(
+ fun(_, Acc) ->
+ update_bulk_get_ratio(Acc, 1000, 1000)
+ end,
+ Fail,
+ lists:seq(1, 100)
+ ),
+ ?assert(FailThenSuccess#bulk_get_stats.ratio < 0.1),
+
+ % Failures after success
+ SuccessThenFailure = lists:foldl(
+ fun(_, Acc) ->
+ update_bulk_get_ratio(Acc, 0, 1000)
+ end,
+ Success,
+ lists:seq(1, 100)
+ ),
+ ?assert(SuccessThenFailure#bulk_get_stats.ratio > 0.9),
+
+ % 0 attempts doesn't crash with a division by 0
+ ZeroAttempts = update_bulk_get_ratio(Init, 0, 0),
+ ?assertEqual(0.0, ZeroAttempts#bulk_get_stats.ratio).
+
-endif.
diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
index 7e198562f..3f454b002 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
@@ -26,6 +26,7 @@ error_reporting_test_() ->
?TDEF_FE(t_fail_bulk_docs),
?TDEF_FE(t_fail_changes_reader),
?TDEF_FE(t_fail_revs_diff),
+ ?TDEF_FE(t_fail_bulk_get, 15),
?TDEF_FE(t_fail_changes_queue),
?TDEF_FE(t_fail_changes_manager),
?TDEF_FE(t_fail_changes_reader_proc)
@@ -74,6 +75,31 @@ t_fail_revs_diff({_Ctx, {Source, Target}}) ->
couch_replicator_notifier:stop(Listener).
+t_fail_bulk_get({_Ctx, {Source, Target}}) ->
+ % For _bulk_get the expectation is that the replication job will fallback
+ % to a plain GET so the shape of the test is a bit different than the other
+ % tests here.
+ meck:new(couch_replicator_api_wrap, [passthrough]),
+ populate_db(Source, 1, 5),
+ {ok, _} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ % Tolerate a 500 error
+ mock_fail_req("/_bulk_get", {ok, "501", [], [<<"not_implemented">>]}),
+ meck:reset(couch_replicator_api_wrap),
+ populate_db(Source, 6, 6),
+ wait_target_in_sync(Source, Target),
+ % Check that there was a fallback to a plain GET
+ ?assertEqual(1, meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6)),
+
+ % Tolerate a 400 error
+ mock_fail_req("/_bulk_get", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
+ meck:reset(couch_replicator_api_wrap),
+ populate_db(Source, 7, 7),
+ wait_target_in_sync(Source, Target),
+ % Check that there was a falback to a plain GET
+ ?assertEqual(1, meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6)).
+
t_fail_changes_queue({_Ctx, {Source, Target}}) ->
populate_db(Source, 1, 5),
{ok, RepId} = replicate(Source, Target),
diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
index 1da4dfa02..d1116e8b0 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
@@ -98,6 +98,8 @@ check_active_tasks(DocsRead, DocsWritten, DocsFailed) ->
RepTask = wait_for_task_status(DocsWritten),
?assertNotEqual(timeout, RepTask),
?assertEqual(DocsRead, couch_util:get_value(docs_read, RepTask)),
+ ?assertEqual(DocsRead, couch_util:get_value(bulk_get_docs, RepTask)),
+ ?assertEqual(DocsRead, couch_util:get_value(bulk_get_attempts, RepTask)),
?assertEqual(DocsWritten, couch_util:get_value(docs_written, RepTask)),
?assertEqual(
DocsFailed,
@@ -112,12 +114,16 @@ check_scheduler_jobs(DocsRead, DocsWritten, DocFailed) ->
?assert(maps:is_key(<<"changes_pending">>, Info)),
?assert(maps:is_key(<<"doc_write_failures">>, Info)),
?assert(maps:is_key(<<"docs_read">>, Info)),
+ ?assert(maps:is_key(<<"bulk_get_docs">>, Info)),
+ ?assert(maps:is_key(<<"bulk_get_attempts">>, Info)),
?assert(maps:is_key(<<"docs_written">>, Info)),
?assert(maps:is_key(<<"missing_revisions_found">>, Info)),
?assert(maps:is_key(<<"checkpointed_source_seq">>, Info)),
?assert(maps:is_key(<<"source_seq">>, Info)),
?assert(maps:is_key(<<"revisions_checked">>, Info)),
?assertMatch(#{<<"docs_read">> := DocsRead}, Info),
+ ?assertMatch(#{<<"bulk_get_docs">> := DocsRead}, Info),
+ ?assertMatch(#{<<"bulk_get_attempts">> := DocsRead}, Info),
?assertMatch(#{<<"docs_written">> := DocsWritten}, Info),
?assertMatch(#{<<"doc_write_failures">> := DocFailed}, Info).
[couchdb] 25/31: Upgrade hash algorithm for cookie auth (#4140)
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e762b10a20c932d987688d5e73a18aa1d51d9947
Author: Ronny <ro...@apache.org>
AuthorDate: Wed Aug 24 18:45:32 2022 +0200
Upgrade hash algorithm for cookie auth (#4140)
Introduce a new config setting "hash_algorithms".
The values of the new config parameter is a list of comma-separated values of Erlang hash algorithms.
An example:
hash_algorithms = sha256, sha, md5
This line will use and generate new cookies with the sha256 hash algorithm and accept/verify cookies with the given hash algorithms sha256, sha and md5.
---
rel/overlay/etc/default.ini | 8 ++
.../eunit/chttpd_auth_hash_algorithms_tests.erl | 99 ++++++++++++++++++++++
src/couch/src/couch_httpd_auth.erl | 42 ++++++++-
3 files changed, 146 insertions(+), 3 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index b88dbcbce..15cd0d4bd 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -267,6 +267,14 @@ bind_address = 127.0.0.1
; Set the SameSite cookie property for the auth cookie. If empty, the SameSite property is not set.
;same_site =
+; Set the HMAC algorithm used by cookie authentication
+; Possible values: sha,sha224,sha256,sha384,sha512,sha3_224,sha3_256,sha3_384,sha3_512,
+; blake2b,blake2s,md4,md5,ripemd160
+; New cookie sessions are generated with the first hash algorithm.
+; All values can be used to decode the session.
+; Default: sha256, sha
+hash_algorithms = sha256, sha
+
; [chttpd_auth_cache]
; max_lifetime = 600000
; max_objects =
diff --git a/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
new file mode 100644
index 000000000..3d872aa46
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_auth_hash_algorithms_tests.erl
@@ -0,0 +1,99 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(chttpd_auth_hash_algorithms_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("chttpd/test/eunit/chttpd_test.hrl").
+
+-define(ADM_USER, "adm_user").
+-define(ADM_PASS, "adm_pass").
+-define(ALLOWED_HASHES, "sha256, sha512, sha, blake2s").
+-define(DISALLOWED_HASHES, "md4, md5, ripemd160").
+
+hash_algorithms_test_() ->
+ {
+ "Testing hash algorithms for cookie auth",
+ {
+ setup,
+ fun setup/0,
+ fun teardown/1,
+ with([
+ ?TDEF(test_hash_algorithms_should_work),
+ ?TDEF(test_hash_algorithms_should_fail)
+ ])
+ }
+ }.
+
+% Test utility functions
+setup() ->
+ Ctx = test_util:start_couch([chttpd]),
+ Hashed = couch_passwords:hash_admin_password(?ADM_PASS),
+ NewSecret = ?b2l(couch_uuids:random()),
+ config:set("admins", ?ADM_USER, ?b2l(Hashed), false),
+ config:set("chttpd_auth", "secret", NewSecret, false),
+ config:set("chttpd", "require_valid_user", "true", false),
+ config:set("chttpd_auth", "hash_algorithms", ?ALLOWED_HASHES, false),
+ AllowedHashes = re:split(config:get("chttpd_auth", "hash_algorithms"), "\\s*,\\s*", [
+ trim, {return, binary}
+ ]),
+ DisallowedHashes = re:split(?DISALLOWED_HASHES, "\\s*,\\s*", [trim, {return, binary}]),
+ {Ctx, {AllowedHashes, DisallowedHashes}}.
+
+teardown({Ctx, _}) ->
+ config:delete("chttpd_auth", "hash_algorithms", false),
+ config:delete("chttpd", "require_valid_user", false),
+ config:delete("chttpd_auth", "secret", false),
+ config:delete("admins", ?ADM_USER, false),
+ test_util:stop_couch(Ctx).
+
+% Helper functions
+base_url() ->
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)),
+ "http://" ++ Addr ++ ":" ++ Port.
+
+make_auth_session_string(HashAlgorithm, User, Secret, TimeStamp) ->
+ SessionData = User ++ ":" ++ erlang:integer_to_list(TimeStamp, 16),
+ Hash = couch_util:hmac(HashAlgorithm, Secret, SessionData),
+ "AuthSession=" ++ couch_util:encodeBase64Url(SessionData ++ ":" ++ ?b2l(Hash)).
+
+get_user_props(User) ->
+ couch_auth_cache:get_user_creds(User).
+
+get_full_secret(User) ->
+ {ok, UserProps, _AuthCtx} = get_user_props(User),
+ UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<"">>),
+ Secret = ?l2b(chttpd_util:get_chttpd_auth_config("secret")),
+ <<Secret/binary, UserSalt/binary>>.
+
+% Test functions
+test_hash_algorithm([], _) ->
+ ok;
+test_hash_algorithm([DefaultHashAlgorithm | DecodingHashAlgorithmsList] = _, Status) ->
+ CurrentTime = couch_httpd_auth:make_cookie_time(),
+ Cookie = make_auth_session_string(
+ erlang:binary_to_existing_atom(DefaultHashAlgorithm),
+ ?ADM_USER,
+ get_full_secret(?ADM_USER),
+ CurrentTime
+ ),
+ {ok, ReqStatus, _, _} = test_request:request(get, base_url(), [{cookie, Cookie}]),
+ ?assertEqual(Status, ReqStatus),
+ test_hash_algorithm(DecodingHashAlgorithmsList, Status).
+
+test_hash_algorithms_should_work({_, {AllowedHashes, _}} = _) ->
+ test_hash_algorithm(AllowedHashes, 200).
+
+test_hash_algorithms_should_fail({_, {_, DisallowedHashes}} = _) ->
+ test_hash_algorithm(DisallowedHashes, 401).
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index a5a876b18..e2cb02f8c 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -16,6 +16,8 @@
-include_lib("couch/include/couch_db.hrl").
+-define(DEFAULT_HASH_ALGORITHM, sha256).
+
-export([party_mode_handler/1]).
-export([
@@ -296,6 +298,7 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) ->
end,
% Verify expiry and hash
CurrentTime = make_cookie_time(),
+ HashAlgorithms = get_config_hash_algorithms(),
case chttpd_util:get_chttpd_auth_config("secret") of
undefined ->
couch_log:debug("cookie auth secret is not set", []),
@@ -308,15 +311,18 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) ->
{ok, UserProps, _AuthCtx} ->
UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<"">>),
FullSecret = <<Secret/binary, UserSalt/binary>>,
- ExpectedHash = couch_util:hmac(sha, FullSecret, User ++ ":" ++ TimeStr),
Hash = ?l2b(HashStr),
+ VerifyHash = fun(HashAlg) ->
+ Hmac = couch_util:hmac(HashAlg, FullSecret, User ++ ":" ++ TimeStr),
+ couch_passwords:verify(Hmac, Hash)
+ end,
Timeout = chttpd_util:get_chttpd_auth_config_integer(
"timeout", 600
),
couch_log:debug("timeout ~p", [Timeout]),
case (catch erlang:list_to_integer(TimeStr, 16)) of
TimeStamp when CurrentTime < TimeStamp + Timeout ->
- case couch_passwords:verify(ExpectedHash, Hash) of
+ case lists:any(VerifyHash, HashAlgorithms) of
true ->
TimeLeft = TimeStamp + Timeout - CurrentTime,
couch_log:debug(
@@ -367,7 +373,8 @@ cookie_auth_header(_Req, _Headers) ->
cookie_auth_cookie(Req, User, Secret, TimeStamp) ->
SessionData = User ++ ":" ++ erlang:integer_to_list(TimeStamp, 16),
- Hash = couch_util:hmac(sha, Secret, SessionData),
+ [HashAlgorithm | _] = get_config_hash_algorithms(),
+ Hash = couch_util:hmac(HashAlgorithm, Secret, SessionData),
mochiweb_cookies:cookie(
"AuthSession",
couch_util:encodeBase64Url(SessionData ++ ":" ++ ?b2l(Hash)),
@@ -695,3 +702,32 @@ authentication_warning(#httpd{mochi_req = Req}, User) ->
"~p: Authentication failed for user ~s from ~s",
[?MODULE, User, Peer]
).
+
+verify_hash_names(HashAlgorithms, SupportedHashFun) ->
+ verify_hash_names(HashAlgorithms, SupportedHashFun, []).
+verify_hash_names([], _, HashNames) ->
+ lists:reverse(HashNames);
+verify_hash_names([H | T], SupportedHashFun, HashNames) ->
+ try
+ HashAtom = binary_to_existing_atom(H),
+ Result =
+ case lists:member(HashAtom, SupportedHashFun) of
+ true -> [HashAtom | HashNames];
+ false -> HashNames
+ end,
+ verify_hash_names(T, SupportedHashFun, Result)
+ catch
+ error:badarg ->
+ couch_log:warning("~p: Hash algorithm ~s is not valid.", [?MODULE, H]),
+ verify_hash_names(T, SupportedHashFun, HashNames)
+ end.
+
+-spec get_config_hash_algorithms() -> list(atom()).
+get_config_hash_algorithms() ->
+ SupportedHashAlgorithms = crypto:supports(hashs),
+ HashAlgorithmsStr = chttpd_util:get_chttpd_auth_config("hash_algorithms", "sha256, sha"),
+ HashAlgorithms = re:split(HashAlgorithmsStr, "\\s*,\\s*", [trim, {return, binary}]),
+ case verify_hash_names(HashAlgorithms, SupportedHashAlgorithms) of
+ [] -> [?DEFAULT_HASH_ALGORITHM];
+ VerifiedHashNames -> VerifiedHashNames
+ end.
[couchdb] 27/31: Fix variable already bound compiler warnings
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ab5e3daad12c4ed1556e7ee818f51a502249b94e
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Aug 23 22:36:35 2022 -0700
Fix variable already bound compiler warnings
OTP 25 generates warnings like the following:
src/chttpd/test/eunit/chttpd_util_test.erl:33:48: Warning: variable '_Persist' is already bound. If you mean to ignore this value, use '_' or a different underscore-prefixed name
Create an explicit `Persist` variable set `false` to suppress those warnings.
---
src/chttpd/test/eunit/chttpd_csp_tests.erl | 7 ++--
.../test/eunit/chttpd_db_attachment_size_tests.erl | 5 ++-
src/chttpd/test/eunit/chttpd_delayed_test.erl | 5 ++-
src/chttpd/test/eunit/chttpd_session_tests.erl | 10 +++--
src/chttpd/test/eunit/chttpd_util_test.erl | 46 +++++++++++-----------
5 files changed, 40 insertions(+), 33 deletions(-)
diff --git a/src/chttpd/test/eunit/chttpd_csp_tests.erl b/src/chttpd/test/eunit/chttpd_csp_tests.erl
index 4c77c5ab0..c4a9e930e 100644
--- a/src/chttpd/test/eunit/chttpd_csp_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_csp_tests.erl
@@ -223,9 +223,10 @@ setup() ->
DbName.
cleanup(DbName) ->
- config:delete("csp", "utils_enable", _Persist = false),
- config:delete("csp", "attachments_enable", _Persist = false),
- config:delete("csp", "showlist_enable", _Persist = false),
+ Persist = false,
+ config:delete("csp", "utils_enable", Persist),
+ config:delete("csp", "attachments_enable", Persist),
+ config:delete("csp", "showlist_enable", Persist),
DbUrl = base_url() ++ "/" ++ DbName,
{200, _} = req(delete, ?ADM, DbUrl),
UsersDb = config:get("chttpd_auth", "authentication_db"),
diff --git a/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl b/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl
index e3975bb6e..6e886935a 100644
--- a/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_db_attachment_size_tests.erl
@@ -23,8 +23,9 @@
setup() ->
Hashed = couch_passwords:hash_admin_password(?PASS),
- ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false),
- ok = config:set("couchdb", "max_attachment_size", "50", _Persist = false),
+ Persist = false,
+ ok = config:set("admins", ?USER, ?b2l(Hashed), Persist),
+ ok = config:set("couchdb", "max_attachment_size", "50", Persist),
TmpDb = ?tempdb(),
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
Port = integer_to_list(mochiweb_socket_server:get(chttpd, port)),
diff --git a/src/chttpd/test/eunit/chttpd_delayed_test.erl b/src/chttpd/test/eunit/chttpd_delayed_test.erl
index 4b0fbd55b..a6ee6b3f1 100644
--- a/src/chttpd/test/eunit/chttpd_delayed_test.erl
+++ b/src/chttpd/test/eunit/chttpd_delayed_test.erl
@@ -19,8 +19,9 @@
setup() ->
Hashed = couch_passwords:hash_admin_password(?PASS),
- ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false),
- ok = config:set("chttpd", "buffer_response", "true", _Persist = false),
+ Persist = false,
+ ok = config:set("admins", ?USER, ?b2l(Hashed), Persist),
+ ok = config:set("chttpd", "buffer_response", "true", Persist),
TmpDb = ?tempdb(),
Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
Port = mochiweb_socket_server:get(chttpd, port),
diff --git a/src/chttpd/test/eunit/chttpd_session_tests.erl b/src/chttpd/test/eunit/chttpd_session_tests.erl
index 3d99e3b10..e00406fe1 100644
--- a/src/chttpd/test/eunit/chttpd_session_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_session_tests.erl
@@ -19,14 +19,16 @@
-define(PASS, "pass").
setup() ->
- ok = config:delete("chttpd_auth", "authentication_db", _Persist = false),
+ Persist = false,
+ ok = config:delete("chttpd_auth", "authentication_db", Persist),
Hashed = couch_passwords:hash_admin_password(?PASS),
- ok = config:set("admins", ?USER, binary_to_list(Hashed), _Persist = false),
+ ok = config:set("admins", ?USER, binary_to_list(Hashed), Persist),
root_url() ++ "/_session".
cleanup(_) ->
- ok = config:delete("chttpd_auth", "authentication_db", _Persist = false),
- ok = config:delete("admins", ?USER, _Persist = false).
+ Persist = false,
+ ok = config:delete("chttpd_auth", "authentication_db", Persist),
+ ok = config:delete("admins", ?USER, Persist).
session_test_() ->
{
diff --git a/src/chttpd/test/eunit/chttpd_util_test.erl b/src/chttpd/test/eunit/chttpd_util_test.erl
index 4ad2b8b83..69fb60156 100644
--- a/src/chttpd/test/eunit/chttpd_util_test.erl
+++ b/src/chttpd/test/eunit/chttpd_util_test.erl
@@ -23,36 +23,38 @@ setup() ->
["httpd", "chttpd", "couch_httpd_auth", "chttpd_auth"]
),
+ Persist = false,
ok = config:set(
"httpd",
"authentication_handlers",
"{couch_httpd_auth, cookie_authentication_handler}, "
"{couch_httpd_auth, default_authentication_handler}",
- _Persist = false
+ Persist
),
- ok = config:set("httpd", "backlog", "512", _Persist = false),
- ok = config:set("chttpd", "require_valid_user", "false", _Persist = false),
- ok = config:set("httpd", "both_exist", "get_in_httpd", _Persist = false),
- ok = config:set("chttpd", "both_exist", "get_in_chttpd", _Persist = false),
- ok = config:set("httpd", "httpd_only", "true", _Persist = false),
- ok = config:set("chttpd", "chttpd_only", "1", _Persist = false),
- ok = config:set("couch_httpd_auth", "both_exist", "cha", _Persist = false),
- ok = config:set("chttpd_auth", "both_exist", "ca", _Persist = false),
- ok = config:set("couch_httpd_auth", "cha_only", "true", _Persist = false),
- ok = config:set("chttpd_auth", "ca_only", "1", _Persist = false).
+ ok = config:set("httpd", "backlog", "512", Persist),
+ ok = config:set("chttpd", "require_valid_user", "false", Persist),
+ ok = config:set("httpd", "both_exist", "get_in_httpd", Persist),
+ ok = config:set("chttpd", "both_exist", "get_in_chttpd", Persist),
+ ok = config:set("httpd", "httpd_only", "true", Persist),
+ ok = config:set("chttpd", "chttpd_only", "1", Persist),
+ ok = config:set("couch_httpd_auth", "both_exist", "cha", Persist),
+ ok = config:set("chttpd_auth", "both_exist", "ca", Persist),
+ ok = config:set("couch_httpd_auth", "cha_only", "true", Persist),
+ ok = config:set("chttpd_auth", "ca_only", "1", Persist).
teardown(_) ->
- ok = config:delete("httpd", "authentication_handlers", _Persist = false),
- ok = config:delete("httpd", "backlog", _Persist = false),
- ok = config:delete("chttpd", "require_valid_user", _Persist = false),
- ok = config:delete("httpd", "both_exist", _Persist = false),
- ok = config:delete("chttpd", "both_exist", _Persist = false),
- ok = config:delete("httpd", "httpd_only", _Persist = false),
- ok = config:delete("chttpd", "chttpd_only", _Persist = false),
- ok = config:delete("couch_httpd_auth", "both_exist", _Persist = false),
- ok = config:delete("chttpd_auth", "both_exist", _Persist = false),
- ok = config:delete("couch_httpd_auth", "cha_only", _Persist = false),
- ok = config:delete("chttpd_auth", "ca_only", _Persist = false).
+ Persist = false,
+ ok = config:delete("httpd", "authentication_handlers", Persist),
+ ok = config:delete("httpd", "backlog", Persist),
+ ok = config:delete("chttpd", "require_valid_user", Persist),
+ ok = config:delete("httpd", "both_exist", Persist),
+ ok = config:delete("chttpd", "both_exist", Persist),
+ ok = config:delete("httpd", "httpd_only", Persist),
+ ok = config:delete("chttpd", "chttpd_only", Persist),
+ ok = config:delete("couch_httpd_auth", "both_exist", Persist),
+ ok = config:delete("chttpd_auth", "both_exist", Persist),
+ ok = config:delete("couch_httpd_auth", "cha_only", Persist),
+ ok = config:delete("chttpd_auth", "ca_only", Persist).
config_delete_all_keys(Section) ->
lists:foreach(
[couchdb] 01/31: Add some utility functions to couch_replicator_test_helper
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2017162ff21ab349a72cbbb9f8711ee922205c6f
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:03:22 2022 -0400
Add some utility functions to couch_replicator_test_helper
In preparation to start using chttpd (fabric) endpoints add some common utility
functions to the replication test helper module.
Since `couch_db:fold_docs/4` doesn't exit for fabric, use the changes feed to
get all the revision leafs. That is used when comparing database endpoints.
It turns our the majority replication tests can use the exact same setup,
teardown and db_url functions so make sure those are also available in the
helper module.
---
.../test/eunit/couch_replicator_test_helper.erl | 134 +++++++++++++++------
1 file changed, 100 insertions(+), 34 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
index 4044e7c72..16423e85c 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
@@ -4,39 +4,75 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-define(USERNAME, "rep_test_user").
+-define(PASSWORD, "rep_test_pass").
+
-export([
- compare_dbs/2,
- compare_dbs/3,
- db_url/1,
+ cluster_compare_dbs/2,
+ cluster_compare_dbs/3,
+ cluster_doc_revs/1,
+ cluster_open_rev/3,
+ cluster_url/0,
+ cluster_db_url/1,
replicate/1,
get_pid/1,
- replicate/2
+ replicate/2,
+ test_setup/0,
+ test_teardown/1,
+ setup_db/0,
+ teardown_db/1
]).
-compare_dbs(Source, Target) ->
- compare_dbs(Source, Target, []).
-
-compare_dbs(Source, Target, ExceptIds) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
-
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
- Id = DocSource#doc.id,
- case lists:member(Id, ExceptIds) of
- true ->
- ?assertEqual(not_found, couch_db:get_doc_info(TargetDb, Id));
- false ->
- {ok, TDoc} = couch_db:open_doc(TargetDb, Id),
- compare_docs(DocSource, TDoc)
+cluster_compare_dbs(Source, Target) ->
+ cluster_compare_dbs(Source, Target, []).
+
+cluster_compare_dbs(Source, Target, ExceptIds) ->
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Source)),
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Target)),
+ lists:foreach(
+ fun({Id, Rev}) ->
+ SrcDoc = cluster_open_rev(Source, Id, Rev),
+ TgtDoc = cluster_open_rev(Target, Id, Rev),
+ case lists:member(Id, ExceptIds) of
+ true ->
+ ?assertEqual(not_found, TgtDoc);
+ false ->
+ compare_docs(SrcDoc, TgtDoc)
+ end
end,
- {ok, Acc}
- end,
+ cluster_doc_revs(Source)
+ ).
+
+cluster_open_rev(DbName, Id, Rev) ->
+ {ok, [Result]} = fabric:open_revs(DbName, Id, [Rev], []),
+ case Result of
+ {ok, #doc{} = Doc} ->
+ Doc;
+ {{not_found, missing}, _} ->
+ not_found
+ end.
+
+cluster_doc_revs(DbName) ->
+ Opts = [{style, all_docs}],
+ {ok, Acc} = fabric_util:isolate(fun() ->
+ fabric:changes(DbName, fun changes_callback/2, [], Opts)
+ end),
+ Acc.
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+changes_callback(start, Acc) ->
+ {ok, Acc};
+changes_callback({change, {Change}}, Acc) ->
+ Id = proplists:get_value(id, Change),
+ Revs = proplists:get_value(changes, Change),
+ IdRevs = [{Id, couch_doc:parse_rev(R)} || {[{<<"rev">>, R}]} <- Revs],
+ {ok, IdRevs ++ Acc};
+changes_callback(timeout, Acc) ->
+ {ok, Acc};
+changes_callback({stop, _EndSeq, _Pending}, Acc) ->
+ {ok, Acc}.
+compare_docs(#doc{} = Doc1, not_found) ->
+ error({not_found, Doc1#doc.id});
compare_docs(Doc1, Doc2) ->
?assertEqual(Doc1#doc.body, Doc2#doc.body),
#doc{atts = Atts1} = Doc1,
@@ -111,15 +147,17 @@ att_decoded_md5(Att) ->
),
couch_hash:md5_hash_final(Md50).
-db_url(DbName) ->
- iolist_to_binary([
- "http://",
- config:get("httpd", "bind_address", "127.0.0.1"),
- ":",
- integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/",
- DbName
- ]).
+cluster_url() ->
+ Fmt = "http://~s:~s@~s:~b",
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Args = [?USERNAME, ?PASSWORD, Addr, Port],
+ ?l2b(io_lib:format(Fmt, Args)).
+
+cluster_db_url(<<"/", _/binary>> = Path) ->
+ <<(cluster_url())/binary, Path/binary>>;
+cluster_db_url(Path) ->
+ <<(cluster_url())/binary, "/", Path/binary>>.
get_pid(RepId) ->
Pid = global:whereis_name({couch_replicator_scheduler_job, RepId}),
@@ -145,3 +183,31 @@ replicate({[_ | _]} = RepObject) ->
ok
end,
ok = couch_replicator_scheduler:remove_job(Rep#rep.id).
+
+setup_db() ->
+ DbName = ?tempdb(),
+ ok = fabric:create_db(DbName, [{q, 1}, {n, 1}, ?ADMIN_CTX]),
+ DbName.
+
+teardown_db(DbName) ->
+ try
+ ok = fabric:delete_db(DbName, [?ADMIN_CTX])
+ catch
+ error:database_does_not_exist ->
+ ok
+ end.
+
+test_setup() ->
+ Ctx = test_util:start_couch([fabric, mem3, chttpd, couch_replicator]),
+ Hashed = couch_passwords:hash_admin_password(?PASSWORD),
+ ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ meck:unload(),
+ teardown_db(Source),
+ teardown_db(Target),
+ config:delete("admins", ?USERNAME, _Persist = false),
+ ok = test_util:stop_couch(Ctx).
[couchdb] 12/31: Update couch_replicator_missing_stubs_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 690e9b52d59a488a87f6519f8fdee365e69bae88
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:31:46 2022 -0400
Update couch_replicator_missing_stubs_tests
Use common setup and teardown helpers, TDEF_FE macros and remove all the
foreachx nonsense.
---
.../eunit/couch_replicator_missing_stubs_tests.erl | 146 ++++++---------------
1 file changed, 39 insertions(+), 107 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
index ff3b5ee98..b0e205729 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
@@ -14,106 +14,34 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(REVS_LIMIT, 3).
-define(TIMEOUT_EUNIT, 30).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
missing_stubs_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with missing stubs (COUCHDB-1365)",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_docs_with_missed_att_stubs/2}
- || Pair <- Pairs
+ ?TDEF_FE(replicate_docs_with_missing_att_stubs, ?TIMEOUT_EUNIT)
]
}
}.
-should_replicate_docs_with_missed_att_stubs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_set_target_revs_limit(Target, ?REVS_LIMIT),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target),
- should_update_source_docs(Source, ?REVS_LIMIT * 2),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_set_target_revs_limit({remote, Target}, RevsLimit) ->
- should_set_target_revs_limit(Target, RevsLimit);
-should_set_target_revs_limit(Target, RevsLimit) ->
- ?_test(begin
- {ok, Db} = couch_db:open_int(Target, [?ADMIN_CTX]),
- ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
- ok = couch_db:close(Db)
- end).
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-should_update_source_docs({remote, Source}, Times) ->
- should_update_source_docs(Source, Times);
-should_update_source_docs(Source, Times) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+replicate_docs_with_missing_att_stubs({_Ctx, {Source, Target}}) ->
+ populate_db(Source),
+ fabric:set_revs_limit(Target, ?REVS_LIMIT, [?ADMIN_CTX]),
+ replicate(Source, Target),
+ compare(Source, Target),
+ update_docs(Source, ?REVS_LIMIT * 2),
+ replicate(Source, Target),
+ compare(Source, Target).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
AttData = crypto:strong_rand_bytes(6000),
Doc = #doc{
id = <<"doc1">>,
@@ -126,34 +54,38 @@ populate_db(DbName) ->
])
]
},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
-update_db_docs(DbName, Times) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _} = couch_db:fold_docs(
- Db,
- fun(FDI, Acc) -> db_fold_fun(FDI, Acc) end,
- {DbName, Times},
- []
- ),
- ok = couch_db:close(Db).
+update_docs(DbName, Times) ->
+ lists:foreach(
+ fun({Id, _Rev}) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ update_doc(DbName, Doc, Times)
+ end,
+ couch_replicator_test_helper:cluster_doc_revs(DbName)
+ ).
-db_fold_fun(FullDocInfo, {DbName, Times}) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+update_doc(DbName, Doc, Times) ->
+ {Pos0, [Rev0 | _]} = Doc#doc.revs,
lists:foldl(
fun(_, {Pos, RevId}) ->
- {ok, Db2} = couch_db:reopen(Db),
- NewDocVersion = Doc#doc{
+ Val = base64:encode(crypto:strong_rand_bytes(100)),
+ NewDoc = Doc#doc{
revs = {Pos, [RevId]},
- body = {[{<<"value">>, base64:encode(crypto:strong_rand_bytes(100))}]}
+ body = {[{<<"value">>, Val}]}
},
- {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+ {ok, NewRev} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]),
NewRev
end,
- {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+ {Pos0, Rev0},
lists:seq(1, Times)
- ),
- ok = couch_db:close(Db),
- {ok, {DbName, Times}}.
+ ).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
+
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 10/31: Update couch_replicator_large_atts_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 62c76161316a33b09e877f661f02eddfc30c0ab3
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:25:37 2022 -0400
Update couch_replicator_large_atts_tests
Use commong setup functions and TDEF_FE macro.
Removing the foreachx and the remote vs local junk really trimmed down the
size. The test content was tiny compared to the clunky EUnit setup logic.
---
.../eunit/couch_replicator_large_atts_tests.erl | 92 ++++++----------------
1 file changed, 23 insertions(+), 69 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
index 2f0e2a1f0..8190c7205 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
@@ -14,12 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(ATT_SIZE_1, 2 * 1024 * 1024).
-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
@@ -27,83 +22,37 @@
-define(TIMEOUT_EUNIT, 120).
setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("attachments", "compressible_types", "text/*", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ Ctx = couch_replicator_test_helper:test_setup(),
+ config:set("attachments", "compressible_types", "text/*", _Persist = false),
+ Ctx.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
-
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+teardown(Ctx) ->
+ config:delete("attachments", "compressible_types", _Persist = false),
+ couch_replicator_test_helper:test_teardown(Ctx).
large_atts_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with large attachments",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup/0,
+ fun teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(should_replicate_atts, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+should_replicate_atts({_Ctx, {Source, Target}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ ?assertEqual(ok, replicate(Source, Target)),
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Doc = #doc{
- id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ id = integer_to_binary(DocIdCounter),
body = {[]},
atts = [
att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
@@ -115,8 +64,7 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
att(Name, Size, Type) ->
couch_att:new([
@@ -125,3 +73,9 @@ att(Name, Size, Type) ->
{att_len, Size},
{data, fun(Count) -> crypto:strong_rand_bytes(Count) end}
]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 24/31: fix missing "=" for admin party in #4153
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 37e177fd7598564ed522a4ed5e141e7541b35a27
Author: Ronny Berndt <ro...@apache.org>
AuthorDate: Tue Aug 23 23:23:48 2022 +0200
fix missing "=" for admin party in #4153
---
dev/run | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dev/run b/dev/run
index 32c68d1a2..52db65255 100755
--- a/dev/run
+++ b/dev/run
@@ -475,7 +475,7 @@ def hack_local_ini(ctx, contents):
if ctx["with_admin_party"]:
os.environ["COUCHDB_TEST_ADMIN_PARTY_OVERRIDE"] = "1"
ctx["admin"] = ("Admin Party!", "You do not need any password.")
- return contents + "\n\n[chttpd_auth]\nsecret %s\n" % COMMON_SALT
+ return contents + "\n\n[chttpd_auth]\nsecret = %s\n" % COMMON_SALT
# handle admin credentials passed from cli or generate own one
if ctx["admin"] is None:
[couchdb] 09/31: Update couch_replicator_id_too_long_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7fc302f059c99c8782e4aa5c0fb66d82e7a1a8c0
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:23:47 2022 -0400
Update couch_replicator_id_too_long_tests
Use common setup and teardown helpers along with some local replicate/2 and db_url/2 functions.
Remove foreachx goop and use TDEF_FE for consistency with other tests.
---
.../eunit/couch_replicator_id_too_long_tests.erl | 82 +++++++---------------
1 file changed, 26 insertions(+), 56 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
index 9ed415a29..08454bd71 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
@@ -14,73 +14,43 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:set("replicator", "max_document_id_length", "infinity"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
id_too_long_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Doc id too long tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "5"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
-
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "4"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "5", _Persist = false),
+ replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "4", _Persist = false),
+ replicate(Source, Target),
+ ?assertError({not_found, <<"12345">>}, compare(Source, Target)).
create_doc(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = couch_doc:from_json_obj({[{<<"_id">>, <<"12345">>}]}),
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 07/31: Update couch_replicator_filtered_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fda897a6f2e67255c240eac65e813e645ad2a6b4
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:20:15 2022 -0400
Update couch_replicator_filtered_tests
Take advantage of the helper setup and teardown functions.
Switching to a simpler TDEF_FE macro instead of foreachx and inorder setup
cruft also saves some lines of code.
---
.../test/eunit/couch_replicator_filtered_tests.erl | 257 ++++++++-------------
1 file changed, 100 insertions(+), 157 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
index b77b83daa..267c4fab6 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
@@ -14,7 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DDOC,
{[
@@ -57,206 +57,149 @@
]}
).
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
filtered_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_succeed_with_query),
+ ?TDEF_FE(should_succeed_with_view)
+ ]
}
}.
-query_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with query replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_query/2} || Pair <- Pairs]
- }
- }.
-
-view_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with a view replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_view/2} || Pair <- Pairs]
- }
- }.
-
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/testfilter">>}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ replicate(RepObject),
%% FilteredFun is an Erlang version of following JS function
%% function(doc, req){if (doc.class == 'mammal') return true;}
- FilterFun = fun(_DocId, {Props}) ->
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs,
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_query({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/queryfilter">>},
{<<"query_params">>,
{[
{<<"starts">>, <<"a">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
case couch_util:get_value(<<"class">>, Props) of
<<"a", _/binary>> -> true;
_ -> false
end
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(2, TargetDocCount),
+ % All the docs filtered as expected,
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_view({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_view({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"_view">>},
{<<"query_params">>,
{[
{<<"view">>, <<"filter_ddoc/mammals">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
- DDoc = couch_doc:from_json_obj(?DDOC),
- Doc1 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc1">>},
- {<<"class">>, <<"mammal">>},
- {<<"value">>, 1}
- ]}
- ),
- Doc2 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc2">>},
- {<<"class">>, <<"amphibians">>},
- {<<"value">>, 2}
- ]}
- ),
- Doc3 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc3">>},
- {<<"class">>, <<"reptiles">>},
- {<<"value">>, 3}
- ]}
- ),
- Doc4 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc4">>},
- {<<"class">>, <<"arthropods">>},
- {<<"value">>, 2}
- ]}
- ),
- {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3, Doc4]),
- couch_db:close(Db).
+ Docs = [
+ couch_doc:from_json_obj(?DDOC),
+ #doc{
+ id = <<"doc1">>,
+ body =
+ {[
+ {<<"class">>, <<"mammal">>},
+ {<<"value">>, 1}
+ ]}
+ },
+ #doc{
+ id = <<"doc2">>,
+ body =
+ {[
+ {<<"class">>, <<"amphibians">>},
+ {<<"value">>, 2}
+ ]}
+ },
+ #doc{
+ id = <<"doc3">>,
+ body =
+ {[
+ {<<"class">>, <<"reptiles">>},
+ {<<"value">>, 3}
+ ]}
+ },
+ #doc{
+ id = <<"doc4">>,
+ body =
+ {[
+ {<<"class">>, <<"arthropods">>},
+ {<<"value">>, 2}
+ ]}
+ }
+ ],
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 30/31: include raft name and node in log lines
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d6fa02d4a763db43287676e4659d91c79d1a5836
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Sep 1 16:55:38 2022 +0100
include raft name and node in log lines
---
src/couch/src/couch_raft.erl | 41 ++++++++++++++++++++++-------------------
1 file changed, 22 insertions(+), 19 deletions(-)
diff --git a/src/couch/src/couch_raft.erl b/src/couch/src/couch_raft.erl
index f20207a21..b847a1b50 100644
--- a/src/couch/src/couch_raft.erl
+++ b/src/couch/src/couch_raft.erl
@@ -67,24 +67,24 @@ callback_mode() ->
%% erlfmt-ignore
handle_event(cast, #{term := FutureTerm} = Msg, _State, #{term := CurrentTerm} = Data) when FutureTerm > CurrentTerm ->
- couch_log:notice("~p received message from future term ~B, moving to that term, becoming follower and clearing votedFor", [node(), FutureTerm]),
+ couch_log:notice("~p received message from future term ~B, moving to that term, becoming follower and clearing votedFor", [id(Data), FutureTerm]),
persist({next_state, follower, Data#{term => FutureTerm, votedFor => undefined, votesGranted => undefined}, {next_event, cast, Msg}});
handle_event(enter, _OldState, follower, Data) ->
#{term := Term, froms := Froms} = Data,
- couch_log:notice("~p became follower in term ~B", [node(), Term]),
+ couch_log:notice("~p became follower in term ~B", [id(Data), Term]),
Replies = [{reply, From, {error, deposed}} || From <- maps:values(Froms)],
persist({keep_state, maps:without([nextIndex, matchIndex], Data#{votedFor => undefined, votesGranted => undefined, froms => #{}}),
[state_timeout(follower) | Replies]});
handle_event(enter, _OldState, candidate, Data) ->
#{term := Term} = Data,
- couch_log:notice("~p became candidate in term ~B", [node(), Term]),
+ couch_log:notice("~p became candidate in term ~B", [id(Data), Term]),
persist({keep_state, start_election(Data), state_timeout(candidate)});
handle_event(enter, _OldState, leader, Data) ->
#{store_module := StoreModule, cohort := Cohort, term := Term} = Data,
- couch_log:notice("~p became leader in term ~B", [node(), Term]),
+ couch_log:notice("~p became leader in term ~B", [id(Data), Term]),
Peers = peers(Cohort),
{LastIndex, _} = StoreModule:last(Data),
{keep_state, Data#{
@@ -106,7 +106,7 @@ handle_event(cast, #{type := 'RequestVoteRequest', term := Term} = Msg, State, #
{LastIndex, LastTerm} = StoreModule:last(Data),
LogOk = MLastLogTerm > LastTerm orelse (MLastLogTerm == LastTerm andalso MLastLogIndex >= LastIndex),
Grant = Term == CurrentTerm andalso LogOk andalso (VotedFor == undefined orelse VotedFor == MSource),
- couch_log:notice("~p received RequestVoteRequest from ~p in term ~B when in term ~B (Grant:~p, LogOk:~p, VotedFor:~p)", [node(), MSource, Term, CurrentTerm, Grant, LogOk, VotedFor]),
+ couch_log:notice("~p received RequestVoteRequest from ~p in term ~B when in term ~B (Grant:~p, LogOk:~p, VotedFor:~p)", [id(Data), MSource, Term, CurrentTerm, Grant, LogOk, VotedFor]),
Reply = #{
type => 'RequestVoteResponse',
term => CurrentTerm,
@@ -121,18 +121,18 @@ handle_event(cast, #{type := 'RequestVoteRequest', term := Term} = Msg, State, #
{keep_state_and_data, state_timeout(State)}
end;
-handle_event(cast, #{type := 'RequestVoteResponse', term := PastTerm}, _State, #{term := CurrentTerm}) when PastTerm < CurrentTerm ->
- couch_log:notice("~p ignored RequestVoteResponse from past term ~B", [node(), PastTerm]),
+handle_event(cast, #{type := 'RequestVoteResponse', term := PastTerm}, _State, #{term := CurrentTerm} = Data) when PastTerm < CurrentTerm ->
+ couch_log:notice("~p ignored RequestVoteResponse from past term ~B", [id(Data), PastTerm]),
keep_state_and_data;
handle_event(cast, #{type := 'RequestVoteResponse', term := Term} = Msg, _State, #{term := Term} = Data) ->
#{source := MSource, voteGranted := MVoteGranted} = Msg,
#{cohort := Cohort, votesGranted := VotesGranted0} = Data,
VotesGranted1 = if MVoteGranted -> lists:usort([MSource | VotesGranted0]); true -> VotesGranted0 end,
- couch_log:notice("~p received RequestVoteResponse from ~p in current term ~B (VotesGranted:~p)", [node(), MSource, Term, VotesGranted1]),
+ couch_log:notice("~p received RequestVoteResponse from ~p in current term ~B (VotesGranted:~p)", [id(Data), MSource, Term, VotesGranted1]),
if
length(VotesGranted1) >= length(Cohort) div 2 + 1 ->
- couch_log:notice("~p has enough votes to be leader in term ~B", [node(), Term]),
+ couch_log:notice("~p has enough votes to be leader in term ~B", [id(Data), Term]),
{next_state, leader, Data#{votesGranted => VotesGranted1}};
true ->
{keep_state, Data#{votesGranted => VotesGranted1}}
@@ -181,7 +181,7 @@ handle_event(cast, #{type := 'AppendEntriesRequest', term := Term} = Msg, State,
matchIndex => MPrevLogIndex,
source => node()
},
- couch_log:debug("~p received heartbeat and everything matches, sending matchIndex:~p", [node(), MPrevLogIndex]),
+ couch_log:debug("~p received heartbeat and everything matches, sending matchIndex:~p", [id(Data), MPrevLogIndex]),
cast(MSource, Reply, Data),
{keep_state, update_state_machine(Data#{commitIndex => MCommitIndex}), state_timeout(State)};
true ->
@@ -199,11 +199,11 @@ handle_event(cast, #{type := 'AppendEntriesRequest', term := Term} = Msg, State,
matchIndex => MPrevLogIndex + length(MEntries),
source => node()
},
- couch_log:notice("~p received entry:~p that's already applied, sending matchIndex:~p", [node(), MEntries, MPrevLogIndex + length(MEntries)]),
+ couch_log:notice("~p received entry:~p that's already applied, sending matchIndex:~p", [id(Data), MEntries, MPrevLogIndex + length(MEntries)]),
cast(MSource, Reply, Data),
{keep_state, update_state_machine(Data#{commitIndex => MCommitIndex}), state_timeout(State)};
NthLogTerm /= FirstEntryTerm ->
- couch_log:notice("~p received conflicting entry:~p, deleting it", [node(), MEntries]),
+ couch_log:notice("~p received conflicting entry:~p, deleting it", [id(Data), MEntries]),
case StoreModule:truncate(LastIndex - 1, Data) of
{ok, NewData} ->
{keep_state, NewData, [{next_event, cast, Msg}, state_timeout(State)]};
@@ -212,7 +212,7 @@ handle_event(cast, #{type := 'AppendEntriesRequest', term := Term} = Msg, State,
end
end;
LastIndex == MPrevLogIndex ->
- couch_log:notice("~p received new entries:~p, appending it to log", [node(), MEntries]),
+ couch_log:notice("~p received new entries:~p, appending it to log", [id(Data), MEntries]),
case StoreModule:append(MEntries, Data) of
{ok, _EntryIndex, NewData} ->
{keep_state, NewData, [{next_event, cast, Msg}, state_timeout(State)]};
@@ -223,14 +223,14 @@ handle_event(cast, #{type := 'AppendEntriesRequest', term := Term} = Msg, State,
end
end;
-handle_event(cast, #{type := 'AppendEntriesResponse', term := PastTerm}, _State, #{term := CurrentTerm}) when PastTerm < CurrentTerm ->
- couch_log:notice("~p ignored AppendEntriesResponse from past term ~B", [node(), PastTerm]),
+handle_event(cast, #{type := 'AppendEntriesResponse', term := PastTerm}, _State, #{term := CurrentTerm} = Data) when PastTerm < CurrentTerm ->
+ couch_log:notice("~p ignored AppendEntriesResponse from past term ~B", [id(Data), PastTerm]),
keep_state_and_data;
handle_event(cast, #{type := 'AppendEntriesResponse', term := Term} = Msg, leader, #{term := Term} = Data) ->
#{success := MSuccess, matchIndex := MMatchIndex, source := MSource} = Msg,
#{nextIndex := NextIndex, matchIndex := MatchIndex} = Data,
- couch_log:debug("~p received AppendEntriesResponse from ~p in current term ~B (Success:~p)", [node(), MSource, Term, MSuccess]),
+ couch_log:debug("~p received AppendEntriesResponse from ~p in current term ~B (Success:~p)", [id(Data), MSource, Term, MSuccess]),
SourceNextIndex = maps:get(MSource, NextIndex),
if
MSuccess ->
@@ -263,12 +263,12 @@ handle_event({call, From}, #{type := 'ClientRequest'}, _State, _Data) ->
handle_event(state_timeout, new_election, State, Data) when State == follower; State == candidate ->
#{term := Term} = Data,
- couch_log:notice("~p election timeout in state ~p, term ~B", [node(), State, Term]),
+ couch_log:notice("~p election timeout in state ~p, term ~B", [id(Data), State, Term]),
persist({next_state, candidate, start_election(Data), state_timeout(State)});
handle_event(state_timeout, heartbeat, leader, Data) ->
#{term := Term} = Data,
- couch_log:debug("~p leader sending a heartbeat in term ~B", [node(), Term]),
+ couch_log:debug("~p leader sending a heartbeat in term ~B", [id(Data), Term]),
ok = send_append_entries(Data),
{keep_state, advance_commit_index(Data), state_timeout(leader)};
@@ -341,7 +341,7 @@ update_state_machine(#{lastApplied := LastApplied, commitIndex := CommitIndex} =
start_election(Data) ->
#{term := Term, cohort := Cohort, store_module := StoreModule} = Data,
ElectionTerm = Term + 1,
- couch_log:notice("~p starting election in term ~B", [node(), ElectionTerm]),
+ couch_log:notice("~p starting election in term ~B", [id(Data), ElectionTerm]),
{LastLogIndex, LastLogTerm} = StoreModule:last(Data),
RequestVote = #{
type => 'RequestVoteRequest',
@@ -393,3 +393,6 @@ persist(Data, HandleEventResult) ->
{error, Reason} ->
{stop, Reason}
end.
+
+id(#{name := Name}) ->
+ [Name, node()].
[couchdb] 13/31: Update couch_replicator_proxy_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f61cabf3ce9b976fe600aa71fa6160a3af3075dc
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:34:26 2022 -0400
Update couch_replicator_proxy_tests
Use the TDEF_FE macro and remove the ugly ?_test(begin...end) construct.
---
.../test/eunit/couch_replicator_proxy_tests.erl | 137 ++++++++++-----------
1 file changed, 64 insertions(+), 73 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
index ca1816b33..184d81aaf 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
@@ -15,6 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include("couch_replicator_test.hrl").
setup() ->
ok.
@@ -34,90 +35,80 @@ replicator_proxy_test_() ->
fun setup/0,
fun teardown/1,
[
- fun parse_rep_doc_without_proxy/1,
- fun parse_rep_doc_with_proxy/1,
- fun parse_rep_source_target_proxy/1,
- fun mutually_exclusive_proxy_and_source_proxy/1,
- fun mutually_exclusive_proxy_and_target_proxy/1
+ ?TDEF_FE(parse_rep_doc_without_proxy),
+ ?TDEF_FE(parse_rep_doc_with_proxy),
+ ?TDEF_FE(parse_rep_source_target_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_source_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_target_proxy)
]
}
}
}.
parse_rep_doc_without_proxy(_) ->
- ?_test(begin
- NoProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined)
- end).
+ NoProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined).
parse_rep_doc_with_proxy(_) ->
- ?_test(begin
- ProxyURL = <<"http://myproxy.com">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, ProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL))
- end).
+ ProxyURL = <<"http://myproxy.com">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, ProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL)).
parse_rep_source_target_proxy(_) ->
- ?_test(begin
- SrcProxyURL = <<"http://mysrcproxy.com">>,
- TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"source_proxy">>, SrcProxyURL},
- {<<"target_proxy">>, TgtProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual(
- (Rep#rep.source)#httpdb.proxy_url,
- binary_to_list(SrcProxyURL)
- ),
- ?assertEqual(
- (Rep#rep.target)#httpdb.proxy_url,
- binary_to_list(TgtProxyURL)
- )
- end).
+ SrcProxyURL = <<"http://mysrcproxy.com">>,
+ TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"source_proxy">>, SrcProxyURL},
+ {<<"target_proxy">>, TgtProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual(
+ (Rep#rep.source)#httpdb.proxy_url,
+ binary_to_list(SrcProxyURL)
+ ),
+ ?assertEqual(
+ (Rep#rep.target)#httpdb.proxy_url,
+ binary_to_list(TgtProxyURL)
+ ).
mutually_exclusive_proxy_and_source_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"source_proxy">>, <<"sourceproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"source_proxy">>, <<"sourceproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
mutually_exclusive_proxy_and_target_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"target_proxy">>, <<"targetproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"target_proxy">>, <<"targetproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
[couchdb] 28/31: Allow and evaluate nested json claim roles in JWT token
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit eea0293f57634c96f29b90a2cab1bdc62236121e
Author: Ronny Berndt <ro...@apache.org>
AuthorDate: Fri May 27 11:29:54 2022 +0200
Allow and evaluate nested json claim roles in JWT token
---
rel/overlay/etc/default.ini | 29 +++++-
src/couch/src/couch_httpd_auth.erl | 58 +++++++++--
test/elixir/test/jwt_roles_claim_test.exs | 167 ++++++++++++++++++++++++++++++
3 files changed, 241 insertions(+), 13 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 15cd0d4bd..929c08351 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -181,10 +181,31 @@ bind_address = 127.0.0.1
; List of claims to validate
; can be the name of a claim like "exp" or a tuple if the claim requires
; a parameter
-; required_claims = exp, {iss, "IssuerNameHere"}
-; roles_claim_name = https://example.com/roles
-;
-; [jwt_keys]
+;required_claims = exp, {iss, "IssuerNameHere"}
+; roles_claim_name is marked as deprecated. Please use roles_claim_path instead!
+; Values for ``roles_claim_name`` can only be top-level attributes in the JWT
+; token. If ``roles_claim_path`` is set, then ``roles_claim_name`` is ignored!
+;roles_claim_name = my-couchdb-roles
+; roles_claim_path was introduced to overcome disadvantages of ``roles_claim_name``,
+; because it is not possible with ``roles_claim_name`` to map nested role
+; attributes in the JWT token. There are only two characters with a special meaning.
+; These are
+; - ``.`` for nesting json attributes and
+; - ``\.`` to skip nesting
+; Example JWT data-payload:
+; {
+; "my": {
+; "nested": {
+; "_couchdb.roles": [
+; ...
+; ]
+; }
+; }
+; }
+; would result in the following parameter config:
+;roles_claim_path = my.nested._couchdb\.roles
+
+;[jwt_keys]
; Configure at least one key here if using the JWT auth handler.
; If your JWT tokens do not include a "kid" attribute, use "_default"
; as the config key, otherwise use the kid as the config key.
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index b3c984174..cdb790f57 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -227,6 +227,7 @@ jwt_authentication_handler(Req) ->
RequiredClaims = get_configured_claims(),
case jwtf:decode(?l2b(Jwt), [alg | RequiredClaims], fun jwtf_keystore:get/2) of
{ok, {Claims}} ->
+ Roles = get_roles_claim(Claims),
case lists:keyfind(<<"sub">>, 1, Claims) of
false ->
throw({unauthorized, <<"Token missing sub claim.">>});
@@ -234,15 +235,7 @@ jwt_authentication_handler(Req) ->
Req#httpd{
user_ctx = #user_ctx{
name = User,
- roles = couch_util:get_value(
- ?l2b(
- config:get(
- "jwt_auth", "roles_claim_name", "_couchdb.roles"
- )
- ),
- Claims,
- []
- )
+ roles = Roles
}
}
end;
@@ -253,6 +246,53 @@ jwt_authentication_handler(Req) ->
Req
end.
+tokenize_json_path(Path, SliceStart, [], Result) ->
+ Result1 = Result ++ [?l2b(string:slice(Path, SliceStart))],
+ [?l2b(string:replace(X, "\\.", ".", all)) || X <- Result1];
+tokenize_json_path(Path, SliceStart, [[{Pos, _}] | T], Result) ->
+ Slice = string:slice(Path, SliceStart, Pos - SliceStart),
+ NewResult = Result ++ [?l2b(Slice)],
+ tokenize_json_path(Path, Pos + 1, T, NewResult).
+
+tokenize_json_path(Path, SplitPositions) ->
+ tokenize_json_path(Path, 0, SplitPositions, []).
+
+get_roles_claim(Claims) ->
+ RolesClaimPath = config:get(
+ "jwt_auth", "roles_claim_path"
+ ),
+ Result =
+ case RolesClaimPath of
+ undefined ->
+ couch_util:get_value(
+ ?l2b(
+ config:get(
+ "jwt_auth", "roles_claim_name", "_couchdb.roles"
+ )
+ ),
+ Claims,
+ []
+ );
+ Defined when is_list(Defined) ->
+ % find all "." but no "\."
+ PathRegex = "(?<!\\\\)\\.",
+ MatchPositions =
+ case re:run(RolesClaimPath, PathRegex, [global]) of
+ nomatch -> [];
+ {match, Pos} -> Pos
+ end,
+ TokenizedJsonPath = tokenize_json_path(RolesClaimPath, MatchPositions),
+ couch_util:get_nested_json_value({Claims}, TokenizedJsonPath)
+ end,
+ case lists:all(fun erlang:is_binary/1, Result) of
+ true ->
+ Result;
+ false ->
+ throw(
+ {bad_request, <<"Malformed token">>}
+ )
+ end.
+
get_configured_claims() ->
Claims = config:get("jwt_auth", "required_claims", ""),
Re = "((?<key1>[a-z]+)|{(?<key2>[a-z]+)\s*,\s*\"(?<val>[^\"]+)\"})",
diff --git a/test/elixir/test/jwt_roles_claim_test.exs b/test/elixir/test/jwt_roles_claim_test.exs
new file mode 100644
index 000000000..cd23a3c25
--- /dev/null
+++ b/test/elixir/test/jwt_roles_claim_test.exs
@@ -0,0 +1,167 @@
+defmodule JwtRolesClaimTest do
+ use CouchTestCase
+
+ @global_server_config [
+ %{
+ :section => "chttpd",
+ :key => "authentication_handlers",
+ :value => [
+ "{chttpd_auth, jwt_authentication_handler}, ",
+ "{chttpd_auth, cookie_authentication_handler}, ",
+ "{chttpd_auth, default_authentication_handler})"
+ ] |> Enum.join
+ },
+ %{
+ :section => "jwt_keys",
+ :key => "hmac:myjwttestkey",
+ :value => ~w(
+ NTNv7j0TuYARvmNMmWXo6fKvM4o6nv/aUi9ryX38ZH+L1bkrnD1ObOQ8JAUmHCBq7
+ Iy7otZcyAagBLHVKvvYaIpmMuxmARQ97jUVG16Jkpkp1wXOPsrF9zwew6TpczyH
+ kHgX5EuLg2MeBuiT/qJACs1J0apruOOJCg/gOtkjB4c=) |> Enum.join()
+ }
+ ]
+
+ test "case: roles_claim_name (undefined) / roles_claim_path (undefined)" do
+ server_config = @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles(["_couchdb.roles_1", "_couchdb.roles_2"])
+ end)
+ end
+
+ test "case: roles_claim_name (defined) / roles_claim_path (undefined)" do
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_name",
+ :value => "my._couchdb.roles"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles(["my._couchdb.roles_1", "my._couchdb.roles_2"])
+ end)
+ end
+
+ test "case: roles_claim_name (undefined) / roles_claim_path (defined)" do
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_path",
+ :value => "foo.bar\\.zonk.baz\\.buu.baa.baa\\.bee.roles"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles(["my_nested_role_1", "my_nested_role_2"])
+ end)
+ end
+
+ test "case: roles_claim_name (defined) / roles_claim_path (defined)" do
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_name",
+ :value => "my._couchdb.roles"
+ },
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_path",
+ :value => "foo.bar\\.zonk.baz\\.buu.baa.baa\\.bee.roles"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles(["my_nested_role_1", "my_nested_role_2"])
+ end)
+ end
+
+ test "case: roles_claim_path with bad input" do
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_path",
+ :value => "<<foo.bar\\.zonk.baz\\.buu.baa.baa\\.bee.roles"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles_with_bad_input()
+ end)
+
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_path",
+ :value => "foo.bar\\.zonk.baz\\.buu.baa.baa\\.bee.roles>>"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles_with_bad_input()
+ end)
+
+ server_config =
+ [
+ %{
+ :section => "jwt_auth",
+ :key => "roles_claim_path",
+ :value => "123456"
+ }
+ ] ++ @global_server_config
+
+ run_on_modified_server(server_config, fn ->
+ test_roles_with_bad_input()
+ end)
+ end
+
+ def test_roles(roles) do
+ token = ~w(
+ eyJ0eXAiOiJKV1QiLCJraWQiOiJteWp3dHRlc3RrZXkiLCJhbGciOiJIUzI1NiJ9.
+ eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRyd
+ WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxNzU1Mjk5NDEwLCJteSI6eyJuZXN0ZW
+ QiOnsiX2NvdWNoZGIucm9sZXMiOlsibXlfbmVzdGVkX2NvdWNoZGIucm9sZXNfMSI
+ sIm15X25lc3RlZF9jb3VjaGRiLnJvbGVzXzEiXX19LCJfY291Y2hkYi5yb2xlcyI6
+ WyJfY291Y2hkYi5yb2xlc18xIiwiX2NvdWNoZGIucm9sZXNfMiJdLCJteS5fY291Y
+ 2hkYi5yb2xlcyI6WyJteS5fY291Y2hkYi5yb2xlc18xIiwibXkuX2NvdWNoZGIucm
+ 9sZXNfMiJdLCJmb28iOnsiYmFyLnpvbmsiOnsiYmF6LmJ1dSI6eyJiYWEiOnsiYmF
+ hLmJlZSI6eyJyb2xlcyI6WyJteV9uZXN0ZWRfcm9sZV8xIiwibXlfbmVzdGVkX3Jv
+ bGVfMiJdfX19fX19.F6kQK-FK0z1kP01bTyw-moXfy2klWfubgF7x7Xitd-0) |> Enum.join()
+
+ resp =
+ Couch.get("/_session",
+ headers: [authorization: "Bearer #{token}"]
+ )
+
+ assert resp.body["userCtx"]["name"] == "1234567890"
+ assert resp.body["userCtx"]["roles"] == roles
+ assert resp.body["info"]["authenticated"] == "jwt"
+ end
+
+ def test_roles_with_bad_input() do
+ token = ~w(
+ eyJ0eXAiOiJKV1QiLCJraWQiOiJteWp3dHRlc3RrZXkiLCJhbGciOiJIUzI1NiJ9.
+ eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRyd
+ WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxNzU1Mjk5NDEwLCJteSI6eyJuZXN0ZW
+ QiOnsiX2NvdWNoZGIucm9sZXMiOlsibXlfbmVzdGVkX2NvdWNoZGIucm9sZXNfMSI
+ sIm15X25lc3RlZF9jb3VjaGRiLnJvbGVzXzEiXX19LCJfY291Y2hkYi5yb2xlcyI6
+ WyJfY291Y2hkYi5yb2xlc18xIiwiX2NvdWNoZGIucm9sZXNfMiJdLCJteS5fY291Y
+ 2hkYi5yb2xlcyI6WyJteS5fY291Y2hkYi5yb2xlc18xIiwibXkuX2NvdWNoZGIucm
+ 9sZXNfMiJdLCJmb28iOnsiYmFyLnpvbmsiOnsiYmF6LmJ1dSI6eyJiYWEiOnsiYmF
+ hLmJlZSI6eyJyb2xlcyI6WyJteV9uZXN0ZWRfcm9sZV8xIiwibXlfbmVzdGVkX3Jv
+ bGVfMiJdfX19fX19.F6kQK-FK0z1kP01bTyw-moXfy2klWfubgF7x7Xitd-0) |> Enum.join()
+
+ resp =
+ Couch.get("/_session",
+ headers: [authorization: "Bearer #{token}"]
+ )
+
+ assert resp.status_code == 404
+ end
+
+end
[couchdb] 16/31: Update couch_replicator_selector_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a8d7da8b1156a2a5bb56a5533cb87fc0091b5dba
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:39:17 2022 -0400
Update couch_replicator_selector_tests
Use the clustered versions of endpoints for the test.
Also, use the common setup and teardown helpers and remove the foreachx
silliness.
---
.../test/eunit/couch_replicator_selector_tests.erl | 121 ++++++++-------------
1 file changed, 44 insertions(+), 77 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
index 8f61a638c..31c5dc93c 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
@@ -14,92 +14,62 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
selector_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Selector filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
+ replicate(
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"selector">>, {[{<<"_id">>, <<"doc2">>}]}}
- ]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ ]}
+ ),
%% FilteredFun is an Erlang version of following mango selector
- FilterFun = fun(_DocId, {Props}) ->
- couch_util:get_value(<<"_id">>, Props) == <<"doc2">>
+ FilterFun = fun(DocId, #doc{}) ->
+ DocId == <<"doc2">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"All the docs selected as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs selected as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc1 = couch_doc:from_json_obj(
{[
{<<"_id">>, <<"doc1">>}
@@ -110,13 +80,10 @@ create_docs(DbName) ->
{<<"_id">>, <<"doc2">>}
]}
),
- {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2]),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, [Doc1, Doc2], [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 08/31: Update couch_replicator_httpc_pool_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e7de68e58a3c9aa9ad872873b7bb3b21e3708013
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:22:30 2022 -0400
Update couch_replicator_httpc_pool_tests
Use the TDEF_FE macro and remove the ?_test(begin...end) construct.
---
.../eunit/couch_replicator_httpc_pool_tests.erl | 96 +++++++++++-----------
1 file changed, 46 insertions(+), 50 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
index 31f1da48e..5fce5e886 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
@@ -13,7 +13,7 @@
-module(couch_replicator_httpc_pool_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
+-include("couch_replicator_test.hrl").
-define(TIMEOUT, 1000).
@@ -35,71 +35,67 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun should_block_new_clients_when_full/1,
- fun should_replace_worker_on_death/1
+ ?TDEF_FE(should_block_new_clients_when_full),
+ ?TDEF_FE(should_replace_worker_on_death)
]
}
}
}.
should_block_new_clients_when_full(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
+ Client1 = spawn_client(Pool),
+ Client2 = spawn_client(Pool),
+ Client3 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- ?assertEqual(ok, ping_client(Client2)),
- ?assertEqual(ok, ping_client(Client3)),
+ ?assertEqual(ok, ping_client(Client1)),
+ ?assertEqual(ok, ping_client(Client2)),
+ ?assertEqual(ok, ping_client(Client3)),
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
+ Worker1 = get_client_worker(Client1, "1"),
+ Worker2 = get_client_worker(Client2, "2"),
+ Worker3 = get_client_worker(Client3, "3"),
- ?assert(is_process_alive(Worker1)),
- ?assert(is_process_alive(Worker2)),
- ?assert(is_process_alive(Worker3)),
+ ?assert(is_process_alive(Worker1)),
+ ?assert(is_process_alive(Worker2)),
+ ?assert(is_process_alive(Worker3)),
- ?assertNotEqual(Worker1, Worker2),
- ?assertNotEqual(Worker2, Worker3),
- ?assertNotEqual(Worker3, Worker1),
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertNotEqual(Worker2, Worker3),
+ ?assertNotEqual(Worker3, Worker1),
- Client4 = spawn_client(Pool),
- ?assertEqual(timeout, ping_client(Client4)),
+ Client4 = spawn_client(Pool),
+ ?assertEqual(timeout, ping_client(Client4)),
- ?assertEqual(ok, stop_client(Client1)),
- ?assertEqual(ok, ping_client(Client4)),
+ ?assertEqual(ok, stop_client(Client1)),
+ ?assertEqual(ok, ping_client(Client4)),
- Worker4 = get_client_worker(Client4, "4"),
- ?assertEqual(Worker1, Worker4),
+ Worker4 = get_client_worker(Client4, "4"),
+ ?assertEqual(Worker1, Worker4),
- lists:foreach(
- fun(C) ->
- ?assertEqual(ok, stop_client(C))
- end,
- [Client2, Client3, Client4]
- )
- end).
+ lists:foreach(
+ fun(C) ->
+ ?assertEqual(ok, stop_client(C))
+ end,
+ [Client2, Client3, Client4]
+ ).
should_replace_worker_on_death(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- Worker1 = get_client_worker(Client1, "1"),
- ?assert(is_process_alive(Worker1)),
-
- ?assertEqual(ok, kill_client_worker(Client1)),
- ?assertNot(is_process_alive(Worker1)),
- ?assertEqual(ok, stop_client(Client1)),
-
- Client2 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client2)),
- Worker2 = get_client_worker(Client2, "2"),
- ?assert(is_process_alive(Worker2)),
-
- ?assertNotEqual(Worker1, Worker2),
- ?assertEqual(ok, stop_client(Client2))
- end).
+ Client1 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client1)),
+ Worker1 = get_client_worker(Client1, "1"),
+ ?assert(is_process_alive(Worker1)),
+
+ ?assertEqual(ok, kill_client_worker(Client1)),
+ ?assertNot(is_process_alive(Worker1)),
+ ?assertEqual(ok, stop_client(Client1)),
+
+ Client2 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client2)),
+ Worker2 = get_client_worker(Client2, "2"),
+ ?assert(is_process_alive(Worker2)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertEqual(ok, stop_client(Client2)).
spawn_client(Pool) ->
Parent = self(),
[couchdb] 21/31: moved the name property to the correct spot.
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 97a747496129e923ec1417c98587776236bfe641
Author: Zach Lankton <za...@gmail.com>
AuthorDate: Thu Aug 18 08:26:49 2022 -0400
moved the name property to the correct spot.
---
.devcontainer/devcontainer.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 0c581315a..a14b7b1e6 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,9 +1,9 @@
{
+ "name": "couchdb-dev",
"build": {
"dockerfile": "Dockerfile",
"context": "..",
"args": {
- "name": "couchdb-dev",
// Useful choices include:
// apache/couchdbci-debian:bullseye-erlang-25.0.2
// apache/couchdbci-debian:bullseye-erlang-24.3.4.2
[couchdb] 02/31: Update couch_replicator_attachments_too_large to use fabric
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2bc960b736e094eadee982506b74db5a9d60fff4
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:08:17 2022 -0400
Update couch_replicator_attachments_too_large to use fabric
Switch test module to use the clustered enpdpoints and TDEF_FE test
macros.
---
.../couch_replicator_attachments_too_large.erl | 79 +++++++---------------
1 file changed, 24 insertions(+), 55 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
index 2d58f847e..4c4ff9c14 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
@@ -14,69 +14,37 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc_with_attachment(Source, <<"doc">>, 1000),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:delete("couchdb", "max_attachment_size"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
attachment_too_large_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Attachment size too large replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "1000", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
+ ok = replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "999", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
-
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ ok = replicate(Source, Target),
+ ?assertError({not_found, <<"doc">>}, compare(Source, Target)).
create_doc_with_attachment(DbName, DocId, AttSize) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = #doc{id = DocId, atts = att(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db),
- ok.
+ {ok, {1, _Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
att(Size) when is_integer(Size), Size >= 1 ->
[
@@ -90,10 +58,11 @@ att(Size) when is_integer(Size), Size >= 1 ->
])
].
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 19/31: update devcontainer
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 87fe58999fb8d4ddeefd2e0cbb880f89f447f105
Author: Zach Lankton <za...@gmail.com>
AuthorDate: Mon Aug 15 17:17:25 2022 +0000
update devcontainer
---
.devcontainer/Dockerfile | 31 ++++++++++---------------------
.devcontainer/devcontainer.json | 30 ++++++++++++++++++++----------
README-DEV.rst | 15 +++++++++++++++
README.rst | 18 ++++++++++++++++++
4 files changed, 63 insertions(+), 31 deletions(-)
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 04a117cb2..46a09d0cf 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,24 +1,13 @@
-ARG ELIXIR_VERSION
-FROM elixir:${ELIXIR_VERSION}
+ARG IMG
+FROM ${IMG}
-# Install SpiderMonkey 60 and tell CouchDB to use it in configure
-ENV SM_VSN=60
+# Install SpiderMonkey 78 and tell CouchDB to use it in configure
+ENV SM_VSN=78
-# Use NodeSource binaries for Node.js (Fauxton dependency)
-RUN set -ex; \
- curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -; \
- echo "deb https://deb.nodesource.com/node_10.x buster main" | tee /etc/apt/sources.list.d/nodesource.list; \
- echo "deb-src https://deb.nodesource.com/node_10.x buster main" | tee -a /etc/apt/sources.list.d/nodesource.list
+USER root
-RUN set -ex; \
- apt-get update; \
- apt-get install -y --no-install-recommends \
- libmozjs-${SM_VSN}-dev \
- libicu-dev \
- python3-venv \
- python3-pip \
- python3-sphinx \
- nodejs
-
-# Documentation theme
-RUN pip3 install sphinx_rtd_theme
+# These lines are necessary if the user has cloned the repo to their local machine
+# and clicked the "Reopen in container button"
+RUN mkdir -p /workspaces/couchdb
+WORKDIR /workspaces/couchdb
+COPY . .
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 666f9fa16..5dbf45dbd 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,19 +1,29 @@
{
"build": {
"dockerfile": "Dockerfile",
+ "context": "..",
"args": {
+ "name": "couchdb-dev",
// Useful choices include:
- // 1.11 -> Erlang 23, Debian Buster
- // 1.10 -> Erlang 22, Debian Buster
- // 1.9 -> Erlang 22, Debian Buster
+ // apache/couchdbci-debian:bullseye-erlang-25.0.2
+ // apache/couchdbci-debian:bullseye-erlang-24.3.4.2
+ // apache/couchdbci-debian:bullseye-erlang-23.3.4.15
//
- // Older versions based on Debian Stretch will not include
- // SpiderMonkey 60, which the Dockerfile expects to be able
- // to install via apt-get.
- "ELIXIR_VERSION": "1.10"
+ "IMG": "apache/couchdbci-debian:bullseye-erlang-25.0.2"
}
},
- "extensions": [
- "erlang-ls.erlang-ls"
- ]
+
+ // We are using a volume mount to improve performance
+ // https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-named-volume-for-your-entire-source-tree
+ //
+ // and eliminate test flake.
+ // https://github.com/apache/couchdb/discussions/4145
+ //
+ // Your code will not be bound to the host OS folder you started this project from.
+ // Your code will live inside the volume created for the container under /workspace.
+ "workspaceMount": "target=/workspaces/couchdb,type=volume",
+ "workspaceFolder": "/workspaces/couchdb",
+ "postCreateCommand": "./configure && make",
+
+ "extensions": ["erlang-ls.erlang-ls"]
}
diff --git a/README-DEV.rst b/README-DEV.rst
index 863218de9..84e16def7 100644
--- a/README-DEV.rst
+++ b/README-DEV.rst
@@ -64,6 +64,21 @@ associated ``devcontainer.json`` file to quickly provision a
development environment using `GitHub Codespaces <https://github.com/features/codespaces>`_
or `Visual Studio Code <https://code.visualstudio.com/docs/remote/containers>`_.
+
+.. image:: https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode
+ :target: https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/apache/couchdb
+
+If you already have VS Code and Docker installed, you can click the badge above or
+`here <https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/apache/couchdb>`_
+to get started. Clicking these links will cause VS Code to automatically install the
+Remote - Containers extension if needed, clone the source code into a container volume,
+and spin up a dev container for use.
+
+This ``devcontainer`` will automatically run ``./configure && make`` the first time it is created.
+While this may take some extra time to spin up, this tradeoff means you will be able to
+run things like ``./dev/run`` and ``make check`` straight away. Subsequent startups should be quick.
+
+
Debian-based (inc. Ubuntu) Systems
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/README.rst b/README.rst
index e6d294ceb..6bb69cd09 100644
--- a/README.rst
+++ b/README.rst
@@ -60,6 +60,24 @@ Run a basic test suite for CouchDB by browsing here:
Getting started with developing
-------------------------------
+**Quickstart:**
+
+
+.. image:: https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode
+ :target: https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/apache/couchdb
+
+If you already have VS Code and Docker installed, you can click the badge above or
+`here <https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/apache/couchdb>`_
+to get started. Clicking these links will cause VS Code to automatically install the
+Remote - Containers extension if needed, clone the source code into a container volume,
+and spin up a dev container for use.
+
+This ``devcontainer`` will automatically run ``./configure && make`` the first time it is created.
+While this may take some extra time to spin up, this tradeoff means you will be able to
+run things like ``./dev/run`` and ``make check`` straight away. Subsequent startups should be quick.
+
+**Manual Dev Setup:**
+
For more detail, read the README-DEV.rst file in this directory.
Basically you just have to install the needed dependencies which are
[couchdb] 18/31: Update couch_replicator_use_checkpoints_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 811bcf073a79406b868bc83a28bb36b728b563ee
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:45:29 2022 -0400
Update couch_replicator_use_checkpoints_tests
Use the clustered version of source and target endpoints.
Also, use common setup and teardown functions. The test was previously quite
clever in trying to save an extra few lines by parameterizing the "use
checkpoing" vs "don't use checkpoints" scenarios with a foreachx and a custom
notifier function. Instead, opt for more clarity and use the usual TDEF_FE
macro and just setup two separate test one which uses checkpoints and one which
doesn't.
Another major win is is using the utility db comparison function instead of a
duplicate copy of it.
---
.../couch_replicator_use_checkpoints_tests.erl | 182 ++++++---------------
1 file changed, 49 insertions(+), 133 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
index a23f415c0..193855a3d 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
@@ -14,133 +14,78 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_COUNT, 100).
-define(TIMEOUT_EUNIT, 30).
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-start(false) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
- (_) ->
- ok
- end;
-start(true) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertNotEqual(
- false,
- lists:keyfind(
- <<"session_id">>,
- 1,
- CheckpointHistory
- )
- );
- (_) ->
- ok
- end.
-
-stop(_, _) ->
- ok.
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+setup_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_checkpoint_fun/1,
+ {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+ {Ctx, {Source, Target, Listener}}.
-setup(remote) ->
- {remote, setup()};
-setup({_, Fun, {A, B}}) ->
- Ctx = test_util:start_couch([couch_replicator]),
+setup_no_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_no_checkpoint_fun/1,
{ok, Listener} = couch_replicator_notifier:start_link(Fun),
- Source = setup(A),
- Target = setup(B),
{Ctx, {Source, Target, Listener}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+teardown({Ctx, {Source, Target, Listener}}) ->
+ couch_replicator_notifier:stop(Listener),
+ couch_replicator_test_helper:test_teardown({Ctx, {Source, Target}}).
-teardown(_, {Ctx, {Source, Target, Listener}}) ->
- teardown(Source),
- teardown(Target),
+notifier_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ SId = lists:keyfind(<<"session_id">>, 1, CheckpointHistory),
+ SId =/= false orelse ?debugFmt("~nsession_id not found when using checkpoints", []),
+ ?assertNotEqual(false, SId);
+notifier_checkpoint_fun(_) ->
+ ok.
- couch_replicator_notifier:stop(Listener),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+notifier_no_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
+notifier_no_checkpoint_fun(_) ->
+ ok.
use_checkpoints_test_() ->
{
- "Replication use_checkpoints feature tests",
+ "Replication test using checkpoints",
{
- foreachx,
- fun start/1,
- fun stop/2,
+ foreach,
+ fun setup_checkpoints/0,
+ fun teardown/1,
[
- {UseCheckpoints, fun use_checkpoints_tests/2}
- || UseCheckpoints <- [false, true]
+ ?TDEF_FE(use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-use_checkpoints_tests(UseCheckpoints, Fun) ->
- Pairs = [{remote, remote}],
+dont_use_checkpoints_test_() ->
{
- "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+ "Replication test without using checkpoints",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup_no_checkpoints/0,
+ fun teardown/1,
[
- {{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
- || Pair <- Pairs
+ ?TDEF_FE(dont_use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-should_test_checkpoints({UseCheckpoints, _, {From, To}}, {_Ctx, {Source, Target, _}}) ->
- should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
-should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source, ?DOCS_COUNT),
- should_replicate(Source, Target, UseCheckpoints),
- should_compare_databases(Source, Target)
- ]}
- }.
+use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, true),
+ compare_dbs(Source, Target).
-should_populate_source({remote, Source}, DocCount) ->
- should_populate_source(Source, DocCount);
-should_populate_source(Source, DocCount) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
-
-should_replicate({remote, Source}, Target, UseCheckpoints) ->
- should_replicate(db_url(Source), Target, UseCheckpoints);
-should_replicate(Source, {remote, Target}, UseCheckpoints) ->
- should_replicate(Source, db_url(Target), UseCheckpoints);
-should_replicate(Source, Target, UseCheckpoints) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+dont_use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, false),
+ compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
@@ -154,48 +99,19 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
- end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
replicate(Source, Target, UseCheckpoints) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"use_checkpoints">>, UseCheckpoints}
]}
).
[couchdb] 20/31: update variable name and readme
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a3c09e12a3859a1e1e61ccd2f23396afc2242360
Author: Zach Lankton <za...@gmail.com>
AuthorDate: Wed Aug 17 15:33:13 2022 +0000
update variable name and readme
---
.devcontainer/Dockerfile | 4 ++--
.devcontainer/devcontainer.json | 2 +-
README-DEV.rst | 3 ++-
README.rst | 3 ++-
4 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index 46a09d0cf..2670d8795 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,5 +1,5 @@
-ARG IMG
-FROM ${IMG}
+ARG COUCHDB_IMAGE
+FROM ${COUCHDB_IMAGE}
# Install SpiderMonkey 78 and tell CouchDB to use it in configure
ENV SM_VSN=78
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 5dbf45dbd..0c581315a 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -9,7 +9,7 @@
// apache/couchdbci-debian:bullseye-erlang-24.3.4.2
// apache/couchdbci-debian:bullseye-erlang-23.3.4.15
//
- "IMG": "apache/couchdbci-debian:bullseye-erlang-25.0.2"
+ "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-25.0.2"
}
},
diff --git a/README-DEV.rst b/README-DEV.rst
index 84e16def7..e29d632f5 100644
--- a/README-DEV.rst
+++ b/README-DEV.rst
@@ -76,7 +76,8 @@ and spin up a dev container for use.
This ``devcontainer`` will automatically run ``./configure && make`` the first time it is created.
While this may take some extra time to spin up, this tradeoff means you will be able to
-run things like ``./dev/run`` and ``make check`` straight away. Subsequent startups should be quick.
+run things like ``./dev/run``, ``./dev/run --admin=admin:admin``, ``./dev/run --with-admin-party-please``,
+and ``make check`` straight away. Subsequent startups should be quick.
Debian-based (inc. Ubuntu) Systems
diff --git a/README.rst b/README.rst
index 6bb69cd09..d06904fe8 100644
--- a/README.rst
+++ b/README.rst
@@ -74,7 +74,8 @@ and spin up a dev container for use.
This ``devcontainer`` will automatically run ``./configure && make`` the first time it is created.
While this may take some extra time to spin up, this tradeoff means you will be able to
-run things like ``./dev/run`` and ``make check`` straight away. Subsequent startups should be quick.
+run things like ``./dev/run``, ``./dev/run --admin=admin:admin``, ``./dev/run --with-admin-party-please``,
+and ``make check`` straight away. Subsequent startups should be quick.
**Manual Dev Setup:**
[couchdb] 31/31: HACK: demonstrate the shards running elections in a very hackish way
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c5569e7095adb283189b6c0717c8ed542989da26
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Sep 1 16:56:10 2022 +0100
HACK: demonstrate the shards running elections in a very hackish way
---
src/couch/src/couch_bt_engine.erl | 14 +++++++++++++-
src/couch/src/couch_bt_engine.hrl | 3 ++-
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl
index d93071c1e..b41f4d331 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -963,6 +963,17 @@ init_state(FilePath, Fd, Header0, Options) ->
{reduce, fun ?MODULE:raft_tree_reduce/2}
]),
+ %% ugly hack just to see the elections
+ RaftPid = case re:run(FilePath, "shards/[0-9a-f]+-[0-9a-f]+/[^.]+", [{capture, all, list}]) of
+ {match, [ShardName]} ->
+ Cohort = mem3:nodes(), %% hack
+ {ok, InitialRaftState} = couch_raft_store_sha256:init(Cohort),
+ {ok, P} = couch_raft:start(ShardName, couch_raft_store_sha256, InitialRaftState),
+ P;
+ _ ->
+ undefined
+ end,
+
ok = couch_file:set_db_pid(Fd, self()),
St = #st{
@@ -977,7 +988,8 @@ init_state(FilePath, Fd, Header0, Options) ->
compression = Compression,
purge_tree = PurgeTree,
purge_seq_tree = PurgeSeqTree,
- raft_tree = RaftTree
+ raft_tree = RaftTree,
+ raft_pid = RaftPid
},
% If this is a new database we've just created a
diff --git a/src/couch/src/couch_bt_engine.hrl b/src/couch/src/couch_bt_engine.hrl
index 0d347e99b..8d1dbca2a 100644
--- a/src/couch/src/couch_bt_engine.hrl
+++ b/src/couch/src/couch_bt_engine.hrl
@@ -24,5 +24,6 @@
compression,
purge_tree,
purge_seq_tree,
- raft_tree
+ raft_tree,
+ raft_pid
}).
[couchdb] 03/31: Update couch_replicator_compact_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 250285e421390e8903a96858a7a70e234341ec2e
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:10:15 2022 -0400
Update couch_replicator_compact_tests
Compactor tests are the only tests which continue using the local ports since
they deals with triggering and managing low level compaction processes.
However, it was still possible to improve the tests somewhat by using the
TDEF_FE macros and removing some left-over foreachx cruft.
---
.../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------
1 file changed, 131 insertions(+), 183 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
index 1c093d58c..1f241c753 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
@@ -15,11 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- get_pid/1
-]).
+-include("couch_replicator_test.hrl").
-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
-define(DELAY, 500).
@@ -28,92 +24,60 @@
-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 70).
-define(WRITE_BATCH_SIZE, 25).
-setup() ->
+setup_db() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
ok = couch_db:close(Db),
DbName.
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
+teardown_db(DbName) ->
ok = couch_server:delete(DbName, [?ADMIN_CTX]),
ok.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
+test_setup() ->
+ Ctx = test_util:start_couch([couch_replicator]),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ teardown_db(Source),
+ teardown_db(Target),
ok = test_util:stop_couch(Ctx).
compact_test_() ->
- Pairs = [{remote, remote}],
{
"Compaction during replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun test_setup/0,
+ fun test_teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
+populate_replicate_compact({_Ctx, {Source, Target}}) ->
{ok, RepPid, RepId} = replicate(Source, Target),
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_run_replication(RepPid, RepId, Source, Target),
- should_all_processes_be_alive(RepPid, Source, Target),
- should_populate_and_compact(RepPid, Source, Target, 50, 3),
- should_wait_target_in_sync(Source, Target),
- should_ensure_replication_still_running(RepPid, RepId, Source, Target),
- should_cancel_replication(RepId, RepPid),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_all_processes_be_alive(RepPid, Source, Target) ->
- ?_test(begin
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb)))
- end).
-
-should_run_replication(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
-
-should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+ check_active_tasks(RepPid, RepId, Source, Target),
+ all_processes_are_alive(RepPid, Source, Target),
+ populate_and_compact(RepPid, Source, Target, 50, 3),
+ wait_target_in_sync(Source, Target),
+ check_active_tasks(RepPid, RepId, Source, Target),
+ cancel_replication(RepId, RepPid),
+ compare_databases(Source, Target).
+
+all_processes_are_alive(RepPid, Source, Target) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))).
check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
- Source =
- case Src of
- {remote, NameSrc} ->
- <<(db_url(NameSrc))/binary, $/>>;
- _ ->
- Src
- end,
- Target =
- case Tgt of
- {remote, NameTgt} ->
- <<(db_url(NameTgt))/binary, $/>>;
- _ ->
- Tgt
- end,
+ Source = <<(db_url(Src))/binary, $/>>,
+ Target = <<(db_url(Tgt))/binary, $/>>,
FullRepId = ?l2b(BaseId ++ Ext),
Pid = ?l2b(pid_to_list(RepPid)),
RepTasks = wait_for_task_status(),
@@ -152,71 +116,59 @@ wait_for_task_status() ->
end
end).
-should_cancel_replication(RepId, RepPid) ->
- ?_assertNot(begin
- ok = couch_replicator_scheduler:remove_job(RepId),
- is_process_alive(RepPid)
- end).
+cancel_replication(RepId, RepPid) ->
+ ok = couch_replicator_scheduler:remove_job(RepId),
+ ?assertNot(is_process_alive(RepPid)).
+
+populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
+ {ok, SourceDb0} = reopen_db(Source),
+ Writer = spawn_writer(SourceDb0),
+ lists:foreach(
+ fun(N) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ pause_writer(Writer),
+
+ compact_db("source", SourceDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ wait_for_compaction("source", SourceDb),
+
+ compact_db("target", TargetDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
+ wait_for_compaction("target", TargetDb),
+
+ {ok, SourceDb2} = reopen_db(SourceDb),
+ {ok, TargetDb2} = reopen_db(TargetDb),
+
+ resume_writer(Writer),
+ wait_writer(Writer, BatchSize * N),
+
+ compact_db("source", SourceDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("source", SourceDb2),
+ resume_writer(Writer),
+
+ compact_db("target", TargetDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("target", TargetDb2),
+ resume_writer(Writer)
+ end,
+ lists:seq(1, Rounds)
+ ),
+ stop_writer(Writer).
-should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb0} = reopen_db(Source),
- Writer = spawn_writer(SourceDb0),
- lists:foreach(
- fun(N) ->
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- pause_writer(Writer),
-
- compact_db("source", SourceDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- wait_for_compaction("source", SourceDb),
-
- compact_db("target", TargetDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
- wait_for_compaction("target", TargetDb),
-
- {ok, SourceDb2} = reopen_db(SourceDb),
- {ok, TargetDb2} = reopen_db(TargetDb),
-
- resume_writer(Writer),
- wait_writer(Writer, BatchSize * N),
-
- compact_db("source", SourceDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
- pause_writer(Writer),
- wait_for_compaction("source", SourceDb2),
- resume_writer(Writer),
-
- compact_db("target", TargetDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
- pause_writer(Writer),
- wait_for_compaction("target", TargetDb2),
- resume_writer(Writer)
- end,
- lists:seq(1, Rounds)
- ),
- stop_writer(Writer)
- end)}.
-
-should_wait_target_in_sync({remote, Source}, Target) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, {remote, Target}) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_assert(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
- wait_target_in_sync_loop(SourceDocCount, Target, 300)
- end)}.
+wait_target_in_sync(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
+ ok = couch_db:close(SourceDb),
+ SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
erlang:error(
@@ -226,8 +178,6 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
{reason, "Could not get source and target databases in sync"}
]}
);
-wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
{ok, Target} = couch_db:open_int(TargetName, []),
{ok, TargetInfo} = couch_db:get_db_info(Target),
@@ -241,49 +191,40 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, 35,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
+compare_databases(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget =
+ case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed, [
+ {module, ?MODULE},
+ {line, ?LINE},
+ {reason,
+ lists:concat([
+ "Error opening document '",
+ ?b2l(DocId),
+ "' from target: ",
+ couch_util:to_list(Error)
+ ])}
+ ]}
+ )
end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
-reopen_db({remote, Db}) ->
- reopen_db(Db);
reopen_db(DbName) when is_binary(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
@@ -357,21 +298,17 @@ wait_for_compaction(Type, Db) ->
)
end.
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
replicate(Source, Target) ->
RepObject =
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
- Pid = get_pid(Rep#rep.id),
+ Pid = couch_replicator_test_helper:get_pid(Rep#rep.id),
{ok, Pid, Rep#rep.id}.
wait_writer(Pid, NumDocs) ->
@@ -521,3 +458,14 @@ maybe_pause(Parent, Counter) ->
after 0 ->
ok
end.
+
+db_url(DbName) ->
+ % Note we're returning the backend (local) URL here
+ iolist_to_binary([
+ "http://",
+ config:get("httpd", "bind_address", "127.0.0.1"),
+ ":",
+ integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/",
+ DbName
+ ]).
[couchdb] 11/31: Update couch_replicator_many_leaves_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 24547d5d2c03d31d2d3e49ef13afd623739a00f9
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:28:13 2022 -0400
Update couch_replicator_many_leaves_tests
Use comon setup and teardown function and the TDEF_FE macros.
Also, remove quite a bit of foreachx and remote boilerplate which is not needed
any longer.
Most of the changes however consisted in update all the db operations to use
fabric instead of couch. Luckily, most of those have fabric equivalents, and
fabric calls are even shorter as they don't need open, re-open and close
operations.
---
.../eunit/couch_replicator_many_leaves_tests.erl | 190 ++++++++-------------
1 file changed, 67 insertions(+), 123 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
index cdc90e2ea..aa5752650 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
@@ -14,11 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_CONFLICTS, [
{<<"doc1">>, 10},
@@ -32,35 +28,15 @@
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
-setup() ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup_db(),
- Target = setup_db(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {Source, Target}}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok = test_util:stop_couch(Ctx).
-
docs_with_many_leaves_test_() ->
{
"Replicate documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_populate_replicate_compact/1
+ ?TDEF_FE(should_populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
@@ -70,72 +46,40 @@ docs_with_many_leaves_test_winning_revs_only_test_() ->
"Replicate winning revs only for documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_replicate_winning_revs_only/1
+ ?TDEF_FE(should_replicate_winning_revs_only, ?TIMEOUT_EUNIT)
]
}
}.
should_populate_replicate_compact({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs)
- ]}.
+ populate_db(Source),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs).
should_replicate_winning_revs_only({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs)
- ]}.
-
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate(Source, Target) ->
- should_replicate(Source, Target, []).
-
-should_replicate(Source, Target, Options) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- RepObj = {
- [
- {<<"source">>, db_url(Source)},
- {<<"target">>, db_url(Target)}
- ] ++ Options
- },
- replicate(RepObj)
- end)}.
-
-should_verify_target(Source, Target, Mode) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS, Mode),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
-
-should_add_attachments_to_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]),
- add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
- ok = couch_db:close(SourceDb)
- end)}.
+ populate_db(Source),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs).
+
+replicate(Source, Target, Options) ->
+ RepObj = {
+ [
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)}
+ ] ++ Options
+ },
+ couch_replicator_test_helper:replicate(RepObj).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
lists:foreach(
fun({DocId, NumConflicts}) ->
Value = <<"0">>,
@@ -143,22 +87,20 @@ populate_db(DbName) ->
id = DocId,
body = {[{<<"value">>, Value}]}
},
- {ok, {Pos, Rev}} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ {ok, {Pos, Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]),
% Update first initial doc rev twice to ensure it's always a winner
- {ok, Db2} = couch_db:reopen(Db),
Doc1 = Doc#doc{revs = {Pos, [Rev]}},
- {ok, _} = couch_db:update_doc(Db2, Doc1, [?ADMIN_CTX]),
- {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ {ok, _} = fabric:update_doc(DbName, Doc1, [?ADMIN_CTX]),
+ {ok, _} = add_doc_siblings(DbName, DocId, NumConflicts)
end,
?DOCS_CONFLICTS
- ),
- couch_db:close(Db).
+ ).
add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
add_doc_siblings(Db, DocId, NumLeaves, [], []).
add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], ?REPLICATED_CHANGES),
+ {ok, []} = fabric:update_docs(Db, AccDocs, [?REPLICATED_CHANGES, ?ADMIN_CTX]),
{ok, AccRevs};
add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
Value = ?l2b(?i2l(NumLeaves)),
@@ -179,18 +121,8 @@ add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
verify_target(_SourceDb, _TargetDb, [], _Mode) ->
ok;
verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
- {ok, TargetLookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
+ SourceLookups = open_revs_conflicts(SourceDb, DocId),
+ TargetLookups = open_revs_conflicts(TargetDb, DocId),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
Total = NumConflicts + 1,
@@ -205,14 +137,13 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
),
verify_target(SourceDb, TargetDb, Rest, all_revs);
verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs) ->
- {ok, SourceWinner} = couch_db:open_doc(SourceDb, DocId),
- {ok, TargetWinner} = couch_db:open_doc(TargetDb, DocId),
+ SourceWinner = open_doc(SourceDb, DocId),
+ TargetWinner = open_doc(TargetDb, DocId),
SourceWinnerJson = couch_doc:to_json_obj(SourceWinner, [attachments]),
TargetWinnerJson = couch_doc:to_json_obj(TargetWinner, [attachments]),
% Source winner is the same as the target winner
?assertEqual(SourceWinnerJson, TargetWinnerJson),
- Opts = [conflicts, deleted_conflicts],
- {ok, TargetAll} = couch_db:open_doc_revs(TargetDb, DocId, all, Opts),
+ TargetAll = open_revs_conflicts(TargetDb, DocId),
% There is only one version on the target
?assert(length(TargetAll) == 1),
verify_target(SourceDb, TargetDb, Rest, winning_revs).
@@ -220,7 +151,7 @@ verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs)
add_attachments(_SourceDb, _NumAtts, []) ->
ok;
add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceLookups = open_revs(SourceDb, DocId, []),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
Total = NumConflicts + 1,
?assertEqual(Total, length(SourceDocs)),
@@ -228,20 +159,7 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
NewAtts = lists:foldl(
fun(I, AttAcc) ->
- AttData = crypto:strong_rand_bytes(100),
- NewAtt = couch_att:new([
- {name,
- ?io2b([
- "att_",
- ?i2l(I),
- "_",
- couch_doc:rev_to_str({Pos, Rev})
- ])},
- {type, <<"application/foobar">>},
- {att_len, byte_size(AttData)},
- {data, AttData}
- ]),
- [NewAtt | AttAcc]
+ [make_att(I, Pos, Rev, 100) | AttAcc]
end,
[],
lists:seq(1, NumAtts)
@@ -251,7 +169,33 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
[],
SourceDocs
),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ {ok, UpdateResults} = fabric:update_docs(SourceDb, NewDocs, [?ADMIN_CTX]),
NewRevs = [R || {ok, R} <- UpdateResults],
?assertEqual(length(NewDocs), length(NewRevs)),
add_attachments(SourceDb, NumAtts, Rest).
+
+make_att(Id, Pos, Rev, Size) ->
+ AttData = crypto:strong_rand_bytes(Size),
+ RevStr = couch_doc:rev_to_str({Pos, Rev}),
+ couch_att:new([
+ {name, ?io2b(["att_", ?i2l(Id), "_", RevStr])},
+ {type, <<"application/foobar">>},
+ {att_len, byte_size(AttData)},
+ {data, AttData}
+ ]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+open_revs_conflicts(DbName, Id) ->
+ Opts = [conflicts, deleted_conflicts],
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_revs(DbName, Id, Opts) ->
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_doc(DbName, Id) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ Doc.
[couchdb] 04/31: Update couch_replicator_connection_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 63c9d89ca3c77b511277f3053e5388a8a15f86eb
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:13:44 2022 -0400
Update couch_replicator_connection_tests
The main changes are just using the TDEF_FE macros and removng the
_test(begin...end) silliness.
---
.../eunit/couch_replicator_connection_tests.erl | 264 ++++++++++-----------
1 file changed, 123 insertions(+), 141 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
index 7adbb6852..e8c90d02a 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
@@ -13,9 +13,7 @@
-module(couch_replicator_connection_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
-
--define(TIMEOUT, 1000).
+-include("couch_replicator_test.hrl").
setup() ->
Host = config:get("httpd", "bind_address", "127.0.0.1"),
@@ -37,164 +35,148 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun connections_shared_after_release/1,
- fun connections_not_shared_after_owner_death/1,
- fun idle_connections_closed/1,
- fun test_owner_monitors/1,
- fun worker_discards_creds_on_create/1,
- fun worker_discards_url_creds_after_request/1,
- fun worker_discards_creds_in_headers_after_request/1,
- fun worker_discards_proxy_creds_after_request/1
+ ?TDEF_FE(connections_shared_after_release),
+ ?TDEF_FE(connections_not_shared_after_owner_death),
+ ?TDEF_FE(idle_connections_closed),
+ ?TDEF_FE(test_owner_monitors),
+ ?TDEF_FE(worker_discards_creds_on_create),
+ ?TDEF_FE(worker_discards_url_creds_after_request),
+ ?TDEF_FE(worker_discards_creds_in_headers_after_request),
+ ?TDEF_FE(worker_discards_proxy_creds_after_request)
]
}
}
}.
connections_shared_after_release({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection:release(Pid),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL)
- end),
- receive
- {ok, Pid2} ->
- ?assertEqual(Pid, Pid2)
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection:release(Pid),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL)
+ end),
+ receive
+ {ok, Pid2} ->
+ ?assertEqual(Pid, Pid2)
+ end.
connections_not_shared_after_owner_death({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL),
- error("simulate division by zero without compiler warning")
- end),
- receive
- {ok, Pid} ->
- {ok, Pid2} = couch_replicator_connection:acquire(URL),
- ?assertNotEqual(Pid, Pid2),
- MRef = monitor(process, Pid),
- receive
- {'DOWN', MRef, process, Pid, _Reason} ->
- ?assert(not is_process_alive(Pid));
- Other ->
- throw(Other)
- end
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL),
+ error("simulate division by zero without compiler warning")
+ end),
+ receive
+ {ok, Pid} ->
+ {ok, Pid2} = couch_replicator_connection:acquire(URL),
+ ?assertNotEqual(Pid, Pid2),
+ MRef = monitor(process, Pid),
+ receive
+ {'DOWN', MRef, process, Pid, _Reason} ->
+ ?assert(not is_process_alive(Pid));
+ Other ->
+ throw(Other)
+ end
+ end.
idle_connections_closed({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection ! close_idle_connections,
- ?assert(ets:member(couch_replicator_connection, Pid)),
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- couch_replicator_connection:release(Pid),
- couch_replicator_connection ! close_idle_connections,
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- ?assert(not ets:member(couch_replicator_connection, Pid))
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection ! close_idle_connections,
+ ?assert(ets:member(couch_replicator_connection, Pid)),
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ couch_replicator_connection:release(Pid),
+ couch_replicator_connection ! close_idle_connections,
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ ?assert(not ets:member(couch_replicator_connection, Pid)).
test_owner_monitors({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Worker0} = couch_replicator_connection:acquire(URL),
- assert_monitors_equal([{process, self()}]),
- couch_replicator_connection:release(Worker0),
- assert_monitors_equal([]),
- {Workers, Monitors} = lists:foldl(
- fun(_, {WAcc, MAcc}) ->
- {ok, Worker1} = couch_replicator_connection:acquire(URL),
- MAcc1 = [{process, self()} | MAcc],
- assert_monitors_equal(MAcc1),
- {[Worker1 | WAcc], MAcc1}
- end,
- {[], []},
- lists:seq(1, 5)
- ),
- lists:foldl(
- fun(Worker2, Acc) ->
- [_ | NewAcc] = Acc,
- couch_replicator_connection:release(Worker2),
- assert_monitors_equal(NewAcc),
- NewAcc
- end,
- Monitors,
- Workers
- )
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Worker0} = couch_replicator_connection:acquire(URL),
+ assert_monitors_equal([{process, self()}]),
+ couch_replicator_connection:release(Worker0),
+ assert_monitors_equal([]),
+ {Workers, Monitors} = lists:foldl(
+ fun(_, {WAcc, MAcc}) ->
+ {ok, Worker1} = couch_replicator_connection:acquire(URL),
+ MAcc1 = [{process, self()} | MAcc],
+ assert_monitors_equal(MAcc1),
+ {[Worker1 | WAcc], MAcc1}
+ end,
+ {[], []},
+ lists:seq(1, 5)
+ ),
+ lists:foldl(
+ fun(Worker2, Acc) ->
+ [_ | NewAcc] = Acc,
+ couch_replicator_connection:release(Worker2),
+ assert_monitors_equal(NewAcc),
+ NewAcc
+ end,
+ Monitors,
+ Workers
+ ).
worker_discards_creds_on_create({Host, Port}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0).
worker_discards_url_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_creds_in_headers_after_request({Host, _}) ->
- ?_test(begin
- {_User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Headers = [{"Authorization", "Basic " ++ B64Auth}],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {_User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Headers = [{"Authorization", "Basic " ++ B64Auth}],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_proxy_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Opts = [
- {proxy_host, Host},
- {proxy_port, Port},
- {proxy_user, User},
- {proxy_pass, Pass}
- ],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Opts = [
+ {proxy_host, Host},
+ {proxy_port, Port},
+ {proxy_user, User},
+ {proxy_pass, Pass}
+ ],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
send_req(WPid, URL, Headers, Opts) ->
ibrowse:send_req_direct(WPid, URL, Headers, get, [], Opts).
[couchdb] 15/31: Update couch_replicator_retain_stats_between_job_runs
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9a03f29ce426a6cde284399b4763ea2f9a6faf6f
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:37:32 2022 -0400
Update couch_replicator_retain_stats_between_job_runs
Switch the test to use the clustered endpoints.
Use the common test setup and teardown functions as well as the TDEF_FE macros.
---
...ch_replicator_retain_stats_between_job_runs.erl | 153 ++++++++-------------
1 file changed, 54 insertions(+), 99 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
index 9ffcc9e2c..1da4dfa02 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
@@ -15,104 +15,70 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DELAY, 500).
-define(TIMEOUT, 60000).
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
-
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
-
-setup() ->
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
-
-teardown({Source, Target}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok.
-
stats_retained_test_() ->
{
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_stats_retained_by_scheduler/1,
- fun t_stats_retained_on_job_removal/1
- ]
- }
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_stats_retained_by_scheduler),
+ ?TDEF_FE(t_stats_retained_on_job_removal)
+ ]
}.
-t_stats_retained_by_scheduler({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, RepPid, RepId} = replicate(Source, Target),
- wait_target_in_sync(6, Target),
-
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+t_stats_retained_by_scheduler({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, RepPid, RepId} = replicate(Source, Target),
+ wait_target_in_sync(6, Target),
- stop_job(RepPid),
- check_scheduler_jobs(10, 5, 5),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- start_job(),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
- couch_replicator_scheduler:remove_job(RepId)
- end).
-
-t_stats_retained_on_job_removal({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, _, RepId} = replicate(Source, Target),
- % 5 + 1 vdu
- wait_target_in_sync(6, Target),
+ stop_job(RepPid),
+ check_scheduler_jobs(10, 5, 5),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+ start_job(),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
+ couch_replicator_scheduler:remove_job(RepId).
- couch_replicator_scheduler:remove_job(RepId),
+t_stats_retained_on_job_removal({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 5 + 1 vdu
+ wait_target_in_sync(6, Target),
- populate_db_reject_even_docs(Source, 11, 20),
- {ok, _, RepId} = replicate(Source, Target),
- % 6 + 5
- wait_target_in_sync(11, Target),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- check_scheduler_jobs(20, 10, 10),
- check_active_tasks(20, 10, 10),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId),
+ populate_db_reject_even_docs(Source, 11, 20),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 6 + 5
+ wait_target_in_sync(11, Target),
- populate_db_reject_even_docs(Source, 21, 30),
- {ok, _, RepId} = replicate(Source, Target),
- % 11 + 5
- wait_target_in_sync(16, Target),
+ check_scheduler_jobs(20, 10, 10),
+ check_active_tasks(20, 10, 10),
- check_scheduler_jobs(30, 15, 15),
- check_active_tasks(30, 15, 15),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId)
- end).
+ populate_db_reject_even_docs(Source, 21, 30),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 11 + 5
+ wait_target_in_sync(16, Target),
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ check_scheduler_jobs(30, 15, 15),
+ check_active_tasks(30, 15, 15),
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ couch_replicator_scheduler:remove_job(RepId).
stop_job(RepPid) ->
Ref = erlang:monitor(process, RepPid),
@@ -201,7 +167,6 @@ populate_db_reject_even_docs(DbName, Start, End) ->
populate_db(DbName, Start, End, BodyFun).
populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -211,8 +176,7 @@ populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(DocCount, Target) when is_integer(DocCount) ->
wait_target_in_sync_loop(DocCount, Target, 300).
@@ -226,10 +190,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -238,13 +199,14 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
@@ -254,10 +216,8 @@ replicate(Source, Target) ->
{ok, Pid, Rep#rep.id}.
scheduler_jobs() ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- Url = lists:flatten(io_lib:format("http://~s:~b/_scheduler/jobs", [Addr, Port])),
- {ok, 200, _, Body} = test_request:get(Url, []),
+ JobsUrl = couch_replicator_test_helper:cluster_db_url(<<"_scheduler/jobs">>),
+ {ok, 200, _, Body} = test_request:get(?b2l(JobsUrl), []),
Json = jiffy:decode(Body, [return_maps]),
maps:get(<<"jobs">>, Json).
@@ -279,9 +239,4 @@ add_vdu(DbName) ->
{<<"validate_doc_update">>, vdu()}
],
Doc = couch_doc:from_json_obj({DocProps}, []),
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
- try
- {ok, _Rev} = couch_db:update_doc(Db, Doc, [])
- after
- couch_db:close(Db)
- end.
+ {ok, _Rev} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
[couchdb] 22/31: Address race in cpse_incref_decref test
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit af42d364f602c4d06dddd1299157315292dd3b46
Author: Jay Doane <ja...@apache.org>
AuthorDate: Mon Jul 25 15:05:45 2022 -0700
Address race in cpse_incref_decref test
Occasionally, this test fails with the following stack trace:
cpse_gather: make_test_fun (cpse_incref_decref)...*failed*
in function cpse_test_ref_counting:'-cpse_incref_decref/1-fun-0-'/2 (src/cpse_test_ref_counting.erl, line 44)
in call from cpse_test_ref_counting:cpse_incref_decref/1 (src/cpse_test_ref_counting.erl, line 44)
in call from eunit_test:run_testfun/1 (eunit_test.erl, line 71)
in call from eunit_proc:run_test/1 (eunit_proc.erl, line 522)
in call from eunit_proc:with_timeout/3 (eunit_proc.erl, line 347)
in call from eunit_proc:handle_test/2 (eunit_proc.erl, line 505)
in call from eunit_proc:tests_inorder/3 (eunit_proc.erl, line 447)
in call from eunit_proc:with_timeout/3 (eunit_proc.erl, line 337)
**error:{assert,[{module,cpse_test_ref_counting},
{line,44},
{expression,"lists : member ( Pid , Pids1 )"},
{expected,true},
{value,false}]}
output:<<"">>
Wrap the former assertion in a `test_util:wait` call to account for
the apparent race between client readiness and
`couch_db_engine:monitored_by/1`.
---
src/couch_pse_tests/src/cpse_test_ref_counting.erl | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/src/couch_pse_tests/src/cpse_test_ref_counting.erl b/src/couch_pse_tests/src/cpse_test_ref_counting.erl
index a0123d1ca..e56321080 100644
--- a/src/couch_pse_tests/src/cpse_test_ref_counting.erl
+++ b/src/couch_pse_tests/src/cpse_test_ref_counting.erl
@@ -40,8 +40,15 @@ cpse_incref_decref({Db, _}) ->
{Pid, _} = Client = start_client(Db),
wait_client(Client),
- Pids1 = couch_db_engine:monitored_by(Db),
- ?assert(lists:member(Pid, Pids1)),
+ test_util:wait(
+ fun() ->
+ MonitoredPids1 = couch_db_engine:monitored_by(Db),
+ case lists:member(Pid, MonitoredPids1) of
+ true -> ok;
+ false -> wait
+ end
+ end
+ ),
close_client(Client),
[couchdb] 05/31: Update couch_replicator_create_target_with_options_tests
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch raft_storemodule
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cd04b14540b57b4c5e1b662f12820e0742b17c77
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:15:02 2022 -0400
Update couch_replicator_create_target_with_options_tests
Start using the common setup and tear down functions from the test helper.
Also using the test definitions to use the TDEF_FE macro.
Since the setup function already creates a target endpoint database and the
test is also in charge of creating test database, we just remove the target db
before the replication jobs start.
---
...replicator_create_target_with_options_tests.erl | 74 +++++-----------------
1 file changed, 17 insertions(+), 57 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
index 8adcd25bd..fabb6480e 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
@@ -14,35 +14,20 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
--define(USERNAME, "rep_admin").
--define(PASSWORD, "secret").
-
-setup() ->
- Ctx = test_util:start_couch([fabric, mem3, couch_replicator, chttpd]),
- Hashed = couch_passwords:hash_admin_password(?PASSWORD),
- ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
- Source = ?tempdb(),
- Target = ?tempdb(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {_Source, _Target}}) ->
- config:delete("admins", ?USERNAME),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
create_target_with_options_replication_test_() ->
{
"Create target with range partitions tests",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_create_target_with_q_4/1,
- fun should_create_target_with_q_2_n_1/1,
- fun should_create_target_with_default/1,
- fun should_not_create_target_with_q_any/1
+ ?TDEF_FE(should_create_target_with_q_4),
+ ?TDEF_FE(should_create_target_with_q_2_n_1),
+ ?TDEF_FE(should_create_target_with_default),
+ ?TDEF_FE(should_not_create_target_with_q_any)
]
}
}.
@@ -55,15 +40,12 @@ should_create_target_with_q_4({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"4">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(4, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(4, couch_util:get_value(q, ClusterInfo)).
should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
RepObject =
@@ -73,18 +55,13 @@ should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"2">>}, {<<"n">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- [
- ?_assertEqual(2, couch_util:get_value(q, ClusterInfo)),
- ?_assertEqual(1, couch_util:get_value(n, ClusterInfo))
- ].
+ ?assertEqual(2, couch_util:get_value(q, ClusterInfo)),
+ ?assertEqual(1, couch_util:get_value(n, ClusterInfo)).
should_create_target_with_default({_Ctx, {Source, Target}}) ->
RepObject =
@@ -93,16 +70,13 @@ should_create_target_with_default({_Ctx, {Source, Target}}) ->
{<<"target">>, db_url(Target)},
{<<"create_target">>, true}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
Q = config:get_integer("cluster", "q", 2),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
RepObject =
@@ -112,33 +86,19 @@ should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
{<<"create_target">>, false},
{<<"create_target_params">>, {[{<<"q">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{error, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
DbExist = is_list(catch mem3:shards(Target)),
- delete_db(Source),
- ?_assertEqual(false, DbExist).
+ ?assertEqual(false, DbExist).
create_doc(DbName) ->
Body = {[{<<"foo">>, <<"bar">>}]},
NewDoc = #doc{body = Body},
{ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]).
-create_db(DbName) ->
- ok = fabric:create_db(DbName, [?ADMIN_CTX]).
-
delete_db(DbName) ->
ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
db_url(DbName) ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- ?l2b(
- io_lib:format("http://~s:~s@~s:~b/~s", [
- ?USERNAME,
- ?PASSWORD,
- Addr,
- Port,
- DbName
- ])
- ).
+ couch_replicator_test_helper:cluster_db_url(DbName).