You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2026/10/04 17:00:47 UTC
(couchdb) branch 3.5.x-a-few-more-fixes created (now 8274740df)
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a change to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at 8274740df Update whatsnew
This branch includes the following new commits:
new ad461537d Fix purging non-leaf revisions in map-reduce views
new a34e1d4c1 Fix db delete 500 badarg
new 6d3613a8f Fix out of order update error in nouveau
new 1d6fcc14a Fixup nouveau out-of-order purge PR
new a9010b449 Improve redirects in replicator
new efdeafa4e Fix _search_cleanup and _nouveau_cleanup handlers
new a4cfbb22b Fix suffix check for replicator and users dbs
new 78675601d Fix proxy config default
new 2a539fc05 Use incremental gunzip
new ec7ef3a53 Use short names for authentication_handlers
new 99316730e Improve weatherreport checks
new 8274740df Update whatsnew
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb) 02/12: Fix db delete 500 badarg
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a34e1d4c18da87df10ed608539a587b963dddc95
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Oct 1 13:47:34 2026 -0400
Fix db delete 500 badarg
When nodes return `not_found` for db doc deletion (the doc where the shardmap
lives) `fabric_db_delete` tried to log a warning with `~d` but passed a
non-integer value which blew up with a 500 badarg instead of a nice 404.
The value we tried to log was also wonky, the shard record from
delete_shard_db_doc never actually set `dbname` so the warnign would have
printed `undefined` anyway. Fix it to print dbname instead.
In the issue #6142 this was reproduced with a racing DELETE. Presumably it's
when a read raced the deletion and shoved the shard map back into mem3 cache.
It later gets evicted but a second DELETE in the meantimes got a 500.
Fix: #6142
(cherry picked from commit 8e30f230db96d267178f1b0c7d5317e0979c356b)
---
src/chttpd/test/eunit/chttpd_db_test.erl | 24 ++++++++++++++++++++++++
src/fabric/src/fabric_db_delete.erl | 12 ++++++------
2 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/src/chttpd/test/eunit/chttpd_db_test.erl b/src/chttpd/test/eunit/chttpd_db_test.erl
index 6560cc949..85bec025c 100644
--- a/src/chttpd/test/eunit/chttpd_db_test.erl
+++ b/src/chttpd/test/eunit/chttpd_db_test.erl
@@ -57,6 +57,7 @@ all_test_() ->
?TDEF_FE(t_return_400_new_edits_false_no_revs_on_doc_update, ?TIMEOUT),
?TDEF_FE(t_return_ok_true_on_ensure_full_commit, ?TIMEOUT),
?TDEF_FE(t_return_404_for_ensure_full_commit_on_no_db, ?TIMEOUT),
+ ?TDEF_FE(t_return_404_for_delete_db_with_stale_shard_cache, ?TIMEOUT),
?TDEF_FE(t_accept_live_as_an_alias_for_continuous, ?TIMEOUT),
?TDEF_FE(t_return_headers_after_starting_continuous, ?TIMEOUT),
?TDEF_FE(t_return_404_for_delete_att_on_notadoc, ?TIMEOUT),
@@ -141,6 +142,29 @@ t_return_404_for_ensure_full_commit_on_no_db(Db) ->
?assertEqual(404, Status),
?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
+% If we delete and then read the db, the deletion can put shard map back into
+% the cache until dbs changes feed kicks it out again. A second delete request
+% then can reach nodes after the dbs doc is gone and so it should answer with a
+% 404.
+t_return_404_for_delete_db_with_stale_shard_cache(_Db) ->
+ Db = ?tempdb(),
+ create_db(Db),
+ Shards = lists:sort(mem3:shards(Db)),
+ delete_db(Db),
+ test_util:wait(fun() ->
+ try mem3:shards(Db) of
+ _ -> wait
+ catch
+ error:database_does_not_exist -> ok
+ end
+ end),
+ % simulate the re-insert
+ true = ets:insert(mem3_shards, Shards),
+ ?assertEqual(Shards, lists:sort(mem3:shards(Db))),
+ {Status, Response} = req(delete, url(Db)),
+ ?assertEqual(404, Status),
+ ?assertMatch(#{<<"error">> := <<"not_found">>}, Response).
+
t_accept_live_as_an_alias_for_continuous(Db) ->
GetLastSeq =
fun(Chunks) ->
diff --git a/src/fabric/src/fabric_db_delete.erl b/src/fabric/src/fabric_db_delete.erl
index b57342622..10cd86465 100644
--- a/src/fabric/src/fabric_db_delete.erl
+++ b/src/fabric/src/fabric_db_delete.erl
@@ -32,14 +32,14 @@ go(DbName, _Options) ->
Error ->
Error
after
- % delete the shard files
+ % delete the shard files on nodes which hold the shards
fabric_util:submit_jobs(Shards, delete_db, [])
end.
-delete_shard_db_doc(Doc) ->
- Shards = [#shard{node = N} || N <- mem3:nodes()],
+delete_shard_db_doc(DbName) ->
+ Shards = [#shard{node = N, dbname = DbName} || N <- mem3:nodes()],
RexiMon = fabric_util:create_monitors(Shards),
- Workers = fabric_util:submit_jobs(Shards, delete_shard_db_doc, [Doc]),
+ Workers = fabric_util:submit_jobs(Shards, delete_shard_db_doc, [DbName]),
Acc0 = {length(Shards), fabric_dict:init(Workers, nil)},
try fabric_util:recv(Workers, #shard.ref, fun handle_db_update/3, Acc0) of
{timeout, {_, WorkersDict}} ->
@@ -74,8 +74,8 @@ maybe_stop(W, Counters) ->
{Ok, NotFound} = fabric_dict:fold(fun count_replies/3, {0, 0}, Counters),
case {Ok + NotFound, Ok, NotFound} of
{W, 0, W} ->
- {#shard{dbname = Name}, _} = hd(Counters),
- couch_log:warning("~p not_found ~d", [?MODULE, Name]),
+ {#shard{dbname = DbName}, _} = hd(Counters),
+ couch_log:warning("~p not_found ~s", [?MODULE, DbName]),
{stop, not_found};
{W, _, _} ->
{stop, ok};
(couchdb) 06/12: Fix _search_cleanup and _nouveau_cleanup handlers
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit efdeafa4e914427a9a10f7f2bfbf89c1cb3502b8
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sat Oct 3 01:56:51 2026 -0400
Fix _search_cleanup and _nouveau_cleanup handlers
Make them match _view_cleanup.
Co-Authored-By: Jan Lehnardt <ja...@apache.org>
(cherry picked from commit 6917dd8d266ffeb9998649b3ed801506adee0c54)
---
src/chttpd/src/chttpd_auth_request.erl | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/chttpd/src/chttpd_auth_request.erl b/src/chttpd/src/chttpd_auth_request.erl
index b30612037..b62958721 100644
--- a/src/chttpd/src/chttpd_auth_request.erl
+++ b/src/chttpd/src/chttpd_auth_request.erl
@@ -69,6 +69,10 @@ authorize_request_int(#httpd{path_parts = [_DbName, <<"_compact">> | _]} = Req)
require_db_admin(Req);
authorize_request_int(#httpd{path_parts = [_DbName, <<"_view_cleanup">>]} = Req) ->
require_db_admin(Req);
+authorize_request_int(#httpd{path_parts = [_DbName, <<"_search_cleanup">>]} = Req) ->
+ require_db_admin(Req);
+authorize_request_int(#httpd{path_parts = [_DbName, <<"_nouveau_cleanup">>]} = Req) ->
+ require_db_admin(Req);
authorize_request_int(#httpd{path_parts = [_DbName, <<"_sync_shards">>]} = Req) ->
require_admin(Req);
authorize_request_int(#httpd{path_parts = [_DbName, <<"_purge">>]} = Req) ->
(couchdb) 10/12: Use short names for authentication_handlers
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ec7ef3a53a13e08ee7bd07cdc6b9e05d97d98083
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sat Oct 3 15:52:40 2026 -0400
Use short names for authentication_handlers
Move away from erlang terms and use simpler names them. For compatibility we
still accept existing tuple forms. Entries that are not one of these are
ignored.
Co-Authored-By: Jan Lehnardt <ja...@apache.org>
Co-Authored-By: Robert Newson <rn...@apache.org>
(cherry picked from commit af673227cf77a093bf89ce01279525827ca897dd)
---
rel/overlay/etc/default.ini | 8 ++--
src/chttpd/src/chttpd.erl | 46 ++++++++++---------
src/chttpd/test/eunit/chttpd_auth_tests.erl | 2 +-
src/chttpd/test/eunit/chttpd_util_test.erl | 3 +-
src/couch/src/couch_httpd.erl | 64 ++++++++++++++++++---------
src/couch/src/couch_httpd_auth.erl | 4 --
src/couch/src/couch_util.erl | 5 ++-
src/docs/src/api/server/authn.rst | 8 ++--
src/docs/src/api/server/configuration.rst | 4 +-
src/docs/src/config/http.rst | 14 +++---
test/elixir/test/auth_cache_test.exs | 2 +-
test/elixir/test/config/test-config.ini | 2 +-
test/elixir/test/jwt_roles_claim_test.exs | 6 +--
test/elixir/test/security_validation_test.exs | 2 +-
14 files changed, 95 insertions(+), 75 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 781e55654..11dab43c0 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -234,11 +234,11 @@ bind_address = 127.0.0.1
;buffer_response = false
; authentication handlers
-;authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
+;authentication_handlers = cookie, default
; uncomment the next line to enable proxy authentication
-;authentication_handlers = {chttpd_auth, proxy_authentication_handler}, {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
+;authentication_handlers = proxy, cookie, default
; uncomment the next line to enable JWT authentication
-;authentication_handlers = {chttpd_auth, jwt_authentication_handler}, {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
+;authentication_handlers = jwt, cookie, default
; prevent non-admins from accessing /_all_dbs and /_dbs_info
;admin_only_all_dbs = true
@@ -366,7 +366,7 @@ bind_address = 127.0.0.1
[httpd]
port = {{backend_port}}
bind_address = 127.0.0.1
-;authentication_handlers = {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}
+;authentication_handlers = cookie, default
; Options for the MochiWeb HTTP server.
;server_options = [{backlog, 128}, {acceptor_pool_size, 16}]
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 13fad2ffe..b2df8a1e4 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -648,31 +648,35 @@ extract_cookie(#httpd{mochi_req = MochiReq}) ->
%% erlfmt-ignore
set_auth_handlers() ->
- AuthenticationDefault = "{chttpd_auth, cookie_authentication_handler},
- {chttpd_auth, default_authentication_handler}",
- AuthenticationSrcs = couch_httpd:make_fun_spec_strs(
- config:get("chttpd", "authentication_handlers", AuthenticationDefault)),
- AuthHandlers = lists:map(
- fun(A) -> {auth_handler_name(A), couch_httpd:make_arity_1_fun(A)} end, AuthenticationSrcs),
- AuthenticationFuns = AuthHandlers ++ [
+ AuthenticationHandlers0 = config:get("chttpd", "authentication_handlers", "cookie, default"),
+ AuthenticationHandlers1 = couch_httpd:authentication_handler_match(AuthenticationHandlers0),
+ AuthenticationHandlers2 = lists:filtermap(fun authentication_handler_map/1, AuthenticationHandlers1),
+ AuthenticationFuns = AuthenticationHandlers2 ++ [
fun chttpd_auth:party_mode_handler/1 %% must be last
],
ok = application:set_env(chttpd, auth_handlers, AuthenticationFuns).
-% SpecStr is a string like "{my_module, my_fun}"
-% Takes the first token of the function name in front '_' as auth handler name
-% e.g.
-% chttpd_auth:default_authentication_handler: default
-% chttpd_auth_cookie_authentication_handler: cookie
-% couch_http_auth:proxy_authentication_handler: proxy
-%
-% couch_http:auth_handler_name can't be used here, since it assumes the name
-% of the auth handler to be the 6th token split by [\\W_]
-% - this only works for modules with exactly two underscores in their name
-% - is not very robust (a space after the ',' is assumed)
-auth_handler_name(SpecStr) ->
- {ok, {_, Fun}} = couch_util:parse_term(SpecStr),
- hd(binary:split(atom_to_binary(Fun, latin1), <<"_">>)).
+% aliases
+authentication_handler_map("cookie") ->
+ {true, {~"cookie", fun chttpd_auth:cookie_authentication_handler/1}};
+authentication_handler_map("default") ->
+ {true, {~"default", fun chttpd_auth:default_authentication_handler/1}};
+authentication_handler_map("jwt") ->
+ {true, {~"jwt", fun chttpd_auth:jwt_authentication_handler/1}};
+authentication_handler_map("proxy") ->
+ {true, {~"proxy", fun chttpd_auth:proxy_authentication_handler/1}};
+% deprecated
+authentication_handler_map("{chttpd_auth,jwt_authentication_handler}") ->
+ {true, {~"jwt", fun chttpd_auth:jwt_authentication_handler/1}};
+authentication_handler_map("{chttpd_auth,cookie_authentication_handler}") ->
+ {true, {~"cookie", fun chttpd_auth:cookie_authentication_handler/1}};
+authentication_handler_map("{chttpd_auth,default_authentication_handler}") ->
+ {true, {~"default", fun chttpd_auth:default_authentication_handler/1}};
+authentication_handler_map("{chttpd_auth,proxy_authentication_handler}") ->
+ {true, {~"proxy", fun chttpd_auth:proxy_authentication_handler/1}};
+% reject everything else
+authentication_handler_map(_) ->
+ false.
authenticate_request(Req) ->
{ok, AuthenticationFuns} = application:get_env(chttpd, auth_handlers),
diff --git a/src/chttpd/test/eunit/chttpd_auth_tests.erl b/src/chttpd/test/eunit/chttpd_auth_tests.erl
index 6745bb31c..81291564b 100644
--- a/src/chttpd/test/eunit/chttpd_auth_tests.erl
+++ b/src/chttpd/test/eunit/chttpd_auth_tests.erl
@@ -28,7 +28,7 @@ teardown(_Url) ->
ok.
setup_proxy_auth() ->
- {StartCtx, ProxyCfgFile} = start_couch_with_cfg("{chttpd_auth, proxy_authentication_handler}"),
+ {StartCtx, ProxyCfgFile} = start_couch_with_cfg("proxy"),
config:set("chttpd", "require_valid_user", "false", false),
config:set("chttpd_auth", "hash_algorithms", ?WORKING_HASHES, false),
config:set("chttpd_auth", "proxy_use_secret", "true", false),
diff --git a/src/chttpd/test/eunit/chttpd_util_test.erl b/src/chttpd/test/eunit/chttpd_util_test.erl
index c649393a1..dc06e1075 100644
--- a/src/chttpd/test/eunit/chttpd_util_test.erl
+++ b/src/chttpd/test/eunit/chttpd_util_test.erl
@@ -26,8 +26,7 @@ setup() ->
ok = config:set(
"httpd",
"authentication_handlers",
- "{couch_httpd_auth, cookie_authentication_handler}, "
- "{couch_httpd_auth, default_authentication_handler}",
+ "cookie, default",
Persist
),
ok = config:set("chttpd", "require_valid_user", "false", Persist),
diff --git a/src/couch/src/couch_httpd.erl b/src/couch/src/couch_httpd.erl
index f38375959..f02bd052b 100644
--- a/src/couch/src/couch_httpd.erl
+++ b/src/couch/src/couch_httpd.erl
@@ -21,7 +21,6 @@
-export([header_value/2, header_value/3, qs_value/2, qs_value/3, qs/1, qs_json_value/3]).
-export([path/1, absolute_uri/2, body_length/1]).
-export([verify_is_server_admin/1, unquote/1, quote/1, recv/2, recv_chunked/4, error_info/1]).
--export([make_fun_spec_strs/1]).
-export([make_arity_1_fun/1, make_arity_2_fun/1, make_arity_3_fun/1]).
-export([parse_form/1, json_body/1, json_body_obj/1, body/1]).
-export([doc_etag/1, doc_etag/3, make_etag/1, etag_match/2, etag_respond/3, etag_maybe/2]).
@@ -46,15 +45,13 @@
-export([handle_request/1]).
-export([set_auth_handlers/0]).
-export([maybe_decompress/2, maybe_decompress/3]).
+-export([authentication_handler_match/1]).
-define(HANDLER_NAME_IN_MODULE_POS, 6).
-define(MAX_DRAIN_BYTES, 1048576).
-define(MAX_DRAIN_TIME_MSEC, 1000).
-define(DEFAULT_SOCKET_OPTIONS, "[{sndbuf, 262144}]").
--define(DEFAULT_AUTHENTICATION_HANDLERS,
- "{couch_httpd_auth, cookie_authentication_handler}, "
- "{couch_httpd_auth, default_authentication_handler}"
-).
+-define(DEFAULT_AUTHENTICATION_HANDLERS, "cookie, default").
start_link() ->
start_link(http).
@@ -185,26 +182,48 @@ stop() ->
catch mochiweb_http:stop(https).
set_auth_handlers() ->
- AuthenticationSrcs = make_fun_spec_strs(
- config:get(
- "httpd",
- "authentication_handlers",
- ?DEFAULT_AUTHENTICATION_HANDLERS
- )
+ AuthenticationHandlers0 = config:get(
+ "httpd", "authentication_handlers", ?DEFAULT_AUTHENTICATION_HANDLERS
),
- AuthHandlers = lists:map(
- fun(A) -> {auth_handler_name(A), make_arity_1_fun(A)} end, AuthenticationSrcs
+ AuthenticationHandlers1 = authentication_handler_match(AuthenticationHandlers0),
+ AuthenticationHandlers2 = lists:filtermap(
+ fun authentication_handler_map/1, AuthenticationHandlers1
),
AuthenticationFuns =
- AuthHandlers ++
+ AuthenticationHandlers2 ++
[
%% must be last
fun couch_httpd_auth:party_mode_handler/1
],
ok = application:set_env(couch, auth_handlers, AuthenticationFuns).
-auth_handler_name(SpecStr) ->
- lists:nth(?HANDLER_NAME_IN_MODULE_POS, re:split(SpecStr, "[\\W_]", [])).
+authentication_handler_match(Str) ->
+ case
+ re:run(couch_util:remove_whitespace(Str), "({[a-z0-9_]+,[a-z0-9_]+}|[a-z0-9_]+)", [
+ global, {capture, first, list}
+ ])
+ of
+ {match, Matches} ->
+ [M || [M] <- Matches];
+ nomatch ->
+ []
+ end.
+
+% aliases
+authentication_handler_map("cookie") ->
+ {true, {~"cookie", fun couch_httpd_auth:cookie_authentication_handler/1}};
+authentication_handler_map("default") ->
+ {true, {~"default", fun couch_httpd_auth:default_authentication_handler/1}};
+% deprecated or tests
+authentication_handler_map("{couch_httpd_auth,cookie_authentication_handler}") ->
+ {true, {~"cookie", fun couch_httpd_auth:cookie_authentication_handler/1}};
+authentication_handler_map("{couch_httpd_auth,default_authentication_handler}") ->
+ {true, {~"default", fun couch_httpd_auth:default_authentication_handler/1}};
+authentication_handler_map("{couch_httpd_auth,special_test_authentication_handler}") ->
+ {true, {~"special_test", fun couch_httpd_auth:special_test_authentication_handler/1}};
+% reject everything else
+authentication_handler_map(_) ->
+ false.
get_httpd_handlers() ->
{ok, HttpdGlobalHandlers} = application:get_env(couch, httpd_global_handlers),
@@ -266,10 +285,6 @@ make_arity_3_fun(SpecStr) ->
fun(Arg1, Arg2, Arg3) -> Mod:Fun(Arg1, Arg2, Arg3) end
end.
-% SpecStr is "{my_module, my_fun}, {my_module2, my_fun2}"
-make_fun_spec_strs(SpecStr) ->
- re:split(SpecStr, "(?<=})\\s*,\\s*(?={)", [{return, list}]).
-
handle_request(MochiReq) ->
Body = proplists:get_value(body, MochiReq:get(opts)),
erlang:put(mochiweb_request_body, Body),
@@ -1525,4 +1540,13 @@ should_accept_code_and_message(DontLogFlag) ->
?_assertEqual(ok, log_response(500, {json, {[{error, undef}]}}))}
]}.
+authentication_handler_match_test() ->
+ ?assertEqual("", authentication_handler_match("")),
+ ?assertEqual(["foo", "bar"], authentication_handler_match(" foo , bar ")),
+ ?assertEqual(["foo", "bar"], authentication_handler_match("foo,bar")),
+ ?assertEqual(["{foo,bar}"], authentication_handler_match("{foo,bar}")),
+ ?assertEqual(["{foo,bar}"], authentication_handler_match("{foo, bar}")),
+ ?assertEqual(["foo", "{foo,bar}", "bar"], authentication_handler_match("foo, {foo, bar}, bar")),
+ ?assertEqual(["{foo,bar}", "baz"], authentication_handler_match("{foo , bar}, baz")).
+
-endif.
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index b55892244..21b96d525 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -23,7 +23,6 @@
special_test_authentication_handler/1
]).
-export([cookie_authentication_handler/1, cookie_authentication_handler/2]).
--export([null_authentication_handler/1]).
-export([proxy_authentication_handler/1, proxy_authentification_handler/1]).
-export([cookie_auth_header/2]).
-export([handle_session_req/1, handle_session_req/2]).
@@ -154,9 +153,6 @@ default_authentication_handler(Req, AuthModule) ->
end
end.
-null_authentication_handler(Req) ->
- Req#httpd{user_ctx = ?ADMIN_USER}.
-
%% @doc proxy auth handler.
%
% This handler allows creation of a userCtx object from a user authenticated remotly.
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index 7e3dffd61..7a82888f2 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -15,7 +15,7 @@
-export([priv_dir/0, normpath/1, fold_files/5]).
-export([should_flush/0, should_flush/1, to_existing_atom/1]).
-export([rand32/0, implode/2]).
--export([abs_pathname/1, abs_pathname/2, trim/1, drop_dot_couch_ext/1]).
+-export([abs_pathname/1, abs_pathname/2, trim/1, remove_whitespace/1, drop_dot_couch_ext/1]).
-export([encodeBase64Url/1, decodeBase64Url/1]).
-export([validate_utf8/1, to_hex/1, to_hex_bin/1, parse_term/1, dict_find/3]).
-export([get_nested_json_value/2, json_user_ctx/1]).
@@ -351,6 +351,9 @@ trim(String) ->
String2 = lists:dropwhile(fun is_whitespace/1, String),
lists:reverse(lists:dropwhile(fun is_whitespace/1, lists:reverse(String2))).
+remove_whitespace(Str) ->
+ re:replace(Str, "\\s+", "", [global, {return, list}]).
+
drop_dot_couch_ext(DbName) when is_binary(DbName) ->
PrefixLen = size(DbName) - 6,
case DbName of
diff --git a/src/docs/src/api/server/authn.rst b/src/docs/src/api/server/authn.rst
index 7b89a87bd..354dbdebc 100644
--- a/src/docs/src/api/server/authn.rst
+++ b/src/docs/src/api/server/authn.rst
@@ -283,13 +283,13 @@ Proxy Authentication
.. note::
To use this authentication method make sure that the
- ``{chttpd_auth, proxy_authentication_handler}`` value is added to the
+ ``proxy`` value is added to the
list of the active :config:option:`chttpd/authentication_handlers`:
.. code-block:: ini
[chttpd]
- authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, proxy_authentication_handler}, {chttpd_auth, default_authentication_handler}
+ authentication_handlers = cookie, proxy, default
`Proxy authentication` is very useful in case your application already uses
some external authentication service and you don't want to duplicate users and
@@ -372,13 +372,13 @@ JWT Authentication
.. note::
To use this authentication method, make sure that the
- ``{chttpd_auth, jwt_authentication_handler}`` value is added to the
+ ``jwt`` value is added to the
list of the active :config:option:`chttpd/authentication_handlers`:
.. code-block:: ini
[chttpd]
- authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, jwt_authentication_handler}, {chttpd_auth, default_authentication_handler}
+ authentication_handlers = cookie, jwt, default
``JWT authentication`` enables CouchDB to use externally-generated JWT tokens
instead of defining users or roles in the ``_users`` database.
diff --git a/src/docs/src/api/server/configuration.rst b/src/docs/src/api/server/configuration.rst
index d05bdd8af..b938c422d 100644
--- a/src/docs/src/api/server/configuration.rst
+++ b/src/docs/src/api/server/configuration.rst
@@ -88,7 +88,7 @@ interact with the local node's configuration.
"secure_rewrites": "true"
},
"httpd": {
- "authentication_handlers": "{couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}",
+ "authentication_handlers": "cookie, default",
"bind_address": "192.168.0.2",
"max_connections": "2048",
"port": "5984",
@@ -156,7 +156,7 @@ interact with the local node's configuration.
Server: CouchDB (Erlang/OTP)
{
- "authentication_handlers": "{couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}",
+ "authentication_handlers": "cookie, default",
"bind_address": "127.0.0.1",
"default_handler": "{couch_httpd_db, handle_request}",
"port": "5984"
diff --git a/src/docs/src/config/http.rst b/src/docs/src/config/http.rst
index 1ea18f204..e14a97444 100644
--- a/src/docs/src/config/http.rst
+++ b/src/docs/src/config/http.rst
@@ -81,17 +81,15 @@ HTTP Server Options
.. config:option:: authentication_handlers :: Authentication handlers
List of authentication handlers used by CouchDB. You may
- extend them via third-party plugins or remove some of them if you won't
- let users to use one of provided methods::
+ remove some of them if you won't let users to use one of provided methods::
[chttpd]
- authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
+ authentication_handlers = cookie, default
- - ``{chttpd_auth, cookie_authentication_handler}``: used for Cookie auth;
- - ``{chttpd_auth, proxy_authentication_handler}``: used for Proxy auth;
- - ``{chttpd_auth, jwt_authentication_handler}``: used for JWT auth;
- - ``{chttpd_auth, default_authentication_handler}``: used for Basic auth;
- - ``{couch_httpd_auth, null_authentication_handler}``: disables auth, breaks CouchDB.
+ - ``cookie``: used for Cookie auth;
+ - ``proxy``: used for Proxy auth;
+ - ``jwt``: used for JWT auth;
+ - ``default``: used for Basic auth.
.. config:option:: buffer_response :: Buffered response
diff --git a/test/elixir/test/auth_cache_test.exs b/test/elixir/test/auth_cache_test.exs
index 8b7c29c71..62d08ad9b 100644
--- a/test/elixir/test/auth_cache_test.exs
+++ b/test/elixir/test/auth_cache_test.exs
@@ -22,7 +22,7 @@ defmodule AuthCacheTest do
%{
:section => "httpd",
:key => "authentication_handlers",
- :value => "{couch_httpd_auth, default_authentication_handler}"
+ :value => "default"
},
%{
:section => "chttpd_auth",
diff --git a/test/elixir/test/config/test-config.ini b/test/elixir/test/config/test-config.ini
index 423cc492c..9baf53a70 100644
--- a/test/elixir/test/config/test-config.ini
+++ b/test/elixir/test/config/test-config.ini
@@ -2,7 +2,7 @@
level = warn
[chttpd]
-authentication_handlers = {chttpd_auth, jwt_authentication_handler}, {chttpd_auth, proxy_authentication_handler}, {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
+authentication_handlers = jwt, proxy, cookie, default
[dreyfus]
name = clouseau@127.0.0.1
diff --git a/test/elixir/test/jwt_roles_claim_test.exs b/test/elixir/test/jwt_roles_claim_test.exs
index ec3c7e53e..c3ff93e8a 100644
--- a/test/elixir/test/jwt_roles_claim_test.exs
+++ b/test/elixir/test/jwt_roles_claim_test.exs
@@ -5,11 +5,7 @@ defmodule JwtRolesClaimTest do
%{
:section => "chttpd",
:key => "authentication_handlers",
- :value => [
- "{chttpd_auth, jwt_authentication_handler}, ",
- "{chttpd_auth, cookie_authentication_handler}, ",
- "{chttpd_auth, default_authentication_handler})"
- ] |> Enum.join
+ :value => "jwt, cookie, default"
},
%{
:section => "jwt_keys",
diff --git a/test/elixir/test/security_validation_test.exs b/test/elixir/test/security_validation_test.exs
index 8834e17f5..7120a4a69 100644
--- a/test/elixir/test/security_validation_test.exs
+++ b/test/elixir/test/security_validation_test.exs
@@ -61,7 +61,7 @@ defmodule SecurityValidationTest do
configs = [
{"chttpd", "authentication_handlers",
- "{couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}"},
+ "cookie, default"},
{"chttpd_auth", "authentication_db", auth_db_name}
]
(couchdb) 09/12: Use incremental gunzip
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2a539fc058fd03aef276438c19a2c88d8e41db1e
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sat Oct 3 15:18:45 2026 -0400
Use incremental gunzip
Use incremental gunzip so we can apply configured request limits better.
Session endpoint already had an implicit 1MB size limit applied so we apply it
explicitly in case it changes upstream in mochiweb at some point.
Co-Authored-By: Robert Newson <rn...@apache.org>
(cherry picked from commit e0a8e2c3fb089ff856fe90751047c95fb425d797)
---
src/chttpd/src/chttpd.erl | 2 +-
src/couch/src/couch_att.erl | 2 +-
src/couch/src/couch_httpd.erl | 8 ++++++--
src/couch/src/couch_httpd_auth.erl | 8 +++++---
src/couch/src/couch_util.erl | 27 +++++++++++++++++++++++++++
5 files changed, 40 insertions(+), 7 deletions(-)
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 485e012c2..13fad2ffe 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -1447,7 +1447,7 @@ maybe_decompress(Httpd, Body) ->
case header_value(Httpd, "Content-Encoding", "identity") of
"gzip" ->
try
- zlib:gunzip(Body)
+ couch_util:gunzip_with_limit(Body)
catch
error:data_error ->
throw({bad_request, "Request body is not properly gzipped."})
diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index 4c3678b5f..f61acdc98 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -547,7 +547,7 @@ to_json(Att, OutputData, DataToFollow, ShowEncoding) ->
true ->
AttData =
case Enc of
- gzip -> zlib:gunzip(to_binary(Att));
+ gzip -> couch_util:gunzip_with_limit(to_binary(Att));
identity -> to_binary(Att)
end,
[{<<"data">>, base64:encode(AttData)}]
diff --git a/src/couch/src/couch_httpd.erl b/src/couch/src/couch_httpd.erl
index 5253e4d3c..f38375959 100644
--- a/src/couch/src/couch_httpd.erl
+++ b/src/couch/src/couch_httpd.erl
@@ -45,7 +45,7 @@
-export([check_max_request_length/1]).
-export([handle_request/1]).
-export([set_auth_handlers/0]).
--export([maybe_decompress/2]).
+-export([maybe_decompress/2, maybe_decompress/3]).
-define(HANDLER_NAME_IN_MODULE_POS, 6).
-define(MAX_DRAIN_BYTES, 1048576).
@@ -723,9 +723,13 @@ json_body_obj(Httpd) ->
end.
maybe_decompress(Httpd, Body) ->
+ MaxSize = chttpd_util:get_chttpd_config_integer("max_http_request_size", 4294967296),
+ maybe_decompress(Httpd, Body, MaxSize).
+
+maybe_decompress(Httpd, Body, MaxSize) ->
case header_value(Httpd, "Content-Encoding", "identity") of
"gzip" ->
- zlib:gunzip(Body);
+ couch_util:gunzip_with_limit(Body, MaxSize);
"identity" ->
Body;
Else ->
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index 2af2f7e62..b55892244 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -35,13 +35,15 @@
-export([jwt_authentication_handler/1]).
-import(couch_httpd, [
- header_value/2, send_json/2, send_json/4, send_method_not_allowed/2, maybe_decompress/2
+ header_value/2, send_json/2, send_json/4, send_method_not_allowed/2, maybe_decompress/3
]).
-compile({no_auto_import, [integer_to_binary/1, integer_to_binary/2]}).
-define(LOCKOUT_MSG, <<"Account is temporarily locked due to multiple authentication failures">>).
+-define(MAX_SESSION_BODY_SIZE, 1048576).
+
party_mode_handler(Req) ->
case
chttpd_util:get_chttpd_config_boolean(
@@ -471,14 +473,14 @@ handle_session_req(Req) ->
handle_session_req(Req, couch_auth_cache).
handle_session_req(#httpd{method = 'POST', mochi_req = MochiReq} = Req, AuthModule) ->
- ReqBody = MochiReq:recv_body(),
+ ReqBody = MochiReq:recv_body(?MAX_SESSION_BODY_SIZE),
Form =
case MochiReq:get_primary_header_value("content-type") of
% content type should be json
"application/x-www-form-urlencoded" ++ _ ->
mochiweb_util:parse_qs(ReqBody);
"application/json" ++ _ ->
- {Pairs} = ?JSON_DECODE(maybe_decompress(Req, ReqBody)),
+ {Pairs} = ?JSON_DECODE(maybe_decompress(Req, ReqBody, ?MAX_SESSION_BODY_SIZE)),
lists:map(
fun({Key, Value}) ->
{?b2l(Key), ?b2l(Value)}
diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl
index 8ee165666..7e3dffd61 100644
--- a/src/couch/src/couch_util.erl
+++ b/src/couch/src/couch_util.erl
@@ -47,6 +47,7 @@
-export([ejson_to_map/1]).
-export([new_set/0, set_from_list/1]).
-export([hibernate_after/1]).
+-export([gunzip_with_limit/1, gunzip_with_limit/2]).
-include_lib("couch/include/couch_db.hrl").
@@ -819,3 +820,29 @@ hibernate_after(Module) when is_atom(Module) ->
Timeout when is_integer(Timeout) ->
[{hibernate_after, Timeout}]
end.
+
+gunzip_with_limit(Data) ->
+ Limit = chttpd_util:get_chttpd_config_integer("max_http_request_size", 4294967296),
+ gunzip_with_limit(Data, Limit).
+
+gunzip_with_limit(Data, Limit) when is_integer(Limit), Limit >= 0 ->
+ Z = zlib:open(),
+ try
+ ok = zlib:inflateInit(Z, 16 + 15),
+ Result = gunzip_loop(Z, Limit, zlib:safeInflate(Z, Data), 0, []),
+ ok = zlib:inflateEnd(Z),
+ iolist_to_binary(lists:reverse(Result))
+ after
+ zlib:close(Z)
+ end.
+
+gunzip_loop(Z, Limit, {Action, Output}, AccLen0, AccChunks) ->
+ AccLen1 = AccLen0 + iolist_size(Output),
+ case AccLen1 > Limit of
+ true ->
+ exit({body_too_large, AccLen1});
+ false when Action == continue ->
+ gunzip_loop(Z, Limit, zlib:safeInflate(Z, []), AccLen1, [Output | AccChunks]);
+ false when Action == finished ->
+ [Output | AccChunks]
+ end.
(couchdb) 08/12: Fix proxy config default
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 78675601d1b5a30c4c3fd511193de8fc9eafdea5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sat Oct 3 02:08:29 2026 -0400
Fix proxy config default
Use nil as a better value when users use proxy auth.
Co-Authored-By: Ronny Berndt <ro...@apache.org>
(cherry picked from commit 42de08d2c197eb4618a634253ab19429ded81453)
---
src/couch/src/couch_httpd_auth.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl
index f10b5f455..2af2f7e62 100644
--- a/src/couch/src/couch_httpd_auth.erl
+++ b/src/couch/src/couch_httpd_auth.erl
@@ -208,7 +208,7 @@ proxy_auth_user(Req) ->
true ->
case chttpd_util:get_chttpd_auth_config("secret") of
undefined ->
- Req#httpd{user_ctx = #user_ctx{name = ?l2b(UserName), roles = Roles}};
+ nil;
_Secret ->
Token =
try
(couchdb) 04/12: Fixup nouveau out-of-order purge PR
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1d6fcc14a37d51ee2fefafa32f99b2815df194eb
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Oct 2 14:41:19 2026 -0400
Fixup nouveau out-of-order purge PR
Implement the fixups from https://github.com/apache/couchdb/pull/6138
(cherry picked from commit dd14b1ade8d5fccecb074cc116391a6a47a5f953)
---
src/nouveau/src/nouveau_index_updater.erl | 43 ++++++++++++++++++++-----------
test/elixir/test/nouveau_test.exs | 13 ++++++++--
2 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl
index 9f9de5844..5b03f35ab 100644
--- a/src/nouveau/src/nouveau_index_updater.erl
+++ b/src/nouveau/src/nouveau_index_updater.erl
@@ -244,29 +244,42 @@ purge_index(Db, Index, StartUpdateSeq, #purge_acc{} = PurgeAcc0) ->
case couch_db:get_full_doc_info(Db, Id) of
not_found ->
queue_purge(Id, PurgeSeq, PurgeAcc1);
- #full_doc_info{} = FDI ->
- #doc_info{high_seq = Seq} = couch_doc:to_doc_info(FDI),
+ #full_doc_info{update_seq = Seq} ->
+ %% FDI still exists. If it changed after index last saw it
+ %% (eg. doc was re-created, or a leaf was purged and all
+ %% leafs were re-labeled with new sequences), remove it and
+ %% let the changes phase index the winner (if there is a
+ %% new one). If the FDI didn't change, then the index
+ %% already has its current winner (this may be when purging
+ %% a non-leaf or non-existent revision).
case Seq > StartUpdateSeq of
- true ->
- queue_purge(Id, PurgeSeq, PurgeAcc1);
- false ->
- PurgeAcc1
+ true -> queue_purge(Id, PurgeSeq, PurgeAcc1);
+ false -> PurgeAcc1
end
end,
update_task(1),
- maybe_flush_batch(PurgeAcc2)
+ case maybe_flush_batch(PurgeAcc2) of
+ {ok, PurgeAcc3} ->
+ {ok, PurgeAcc3};
+ {error, Reason} ->
+ exit({error, Reason})
+ end
end,
- {ok, #purge_acc{} = PurgeAcc3} = couch_db:fold_purge_infos(
+ {ok, #purge_acc{} = PurgeAcc4} = couch_db:fold_purge_infos(
Db, PurgeAcc0#purge_acc.index_purge_seq, FoldFun, PurgeAcc0, []
),
- {ok, PurgeAcc4} = flush_batch(PurgeAcc3),
- DbPurgeSeq = couch_db:get_purge_seq(Db),
- ok = nouveau_api:set_purge_seq(
- Index, PurgeAcc4#purge_acc.index_purge_seq, DbPurgeSeq
- ),
- update_local_doc(Db, Index, DbPurgeSeq),
- ok.
+ case flush_batch(PurgeAcc4) of
+ {ok, PurgeAcc5} ->
+ DbPurgeSeq = couch_db:get_purge_seq(Db),
+ ok = nouveau_api:set_purge_seq(
+ Index, PurgeAcc5#purge_acc.index_purge_seq, DbPurgeSeq
+ ),
+ update_local_doc(Db, Index, DbPurgeSeq),
+ ok;
+ {error, Reason} ->
+ exit({error, Reason})
+ end.
queue_purge(Id, PurgeSeq, #purge_acc{} = PurgeAcc) ->
Item = nouveau_api:make_purge(
diff --git a/test/elixir/test/nouveau_test.exs b/test/elixir/test/nouveau_test.exs
index 9a589602b..8f52dba5f 100644
--- a/test/elixir/test/nouveau_test.exs
+++ b/test/elixir/test/nouveau_test.exs
@@ -721,14 +721,14 @@ defmodule NouveauTest do
assert get_total_hits(resp) == 4
# purge two docs in separate requests
- doc_a = (Enum.at(create_resp.body, 0))
+ doc_a = Enum.at(create_resp.body, 0)
resp =
Couch.post("/#{db_name}/_purge",
body: %{doc_a["id"] => [doc_a["rev"]]}
)
assert_status_code(resp, 201)
- doc_b = (Enum.at(create_resp.body, 1))
+ doc_b = Enum.at(create_resp.body, 1)
resp =
Couch.post("/#{db_name}/_purge",
body: %{doc_b["id"] => [doc_b["rev"]]}
@@ -751,6 +751,15 @@ defmodule NouveauTest do
assert_status_code(resp, 200)
assert get_total_hits(resp) == 5
+ # b (doc3) should have "foo" instead of "bar"
+ resp = Couch.get(search_url, query: %{q: "foo:bar"})
+ assert_status_code(resp, 200)
+ assert get_total_hits(resp) == 0
+
+ resp = Couch.get(search_url, query: %{q: "foo:foo"})
+ assert_status_code(resp, 200)
+ assert get_total_hits(resp) == 2
+
# confirm nouveau responds and has correct sequences
resp = Couch.get("/#{db_name}/_design/foo/_nouveau_info/bar")
assert_status_code(resp, 200)
(couchdb) 01/12: Fix purging non-leaf revisions in map-reduce views
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ad461537d74119a79a47ff3a824fe80f29be4291
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Tue Sep 29 17:52:34 2026 -0400
Fix purging non-leaf revisions in map-reduce views
Previously, when we purged a non-leaf revision (either a parent revision or a
completely unknown one) we incorrectly removed the still possibly active live
index entry from that document.
To fix it, check if the doc has changed since the last time we indexed it. Only
remove index entries in the purge phase if the FDI is gone or has been updated
since the last time it was indexed. In the latter case, since we bump updated
sequences on all leafs when we purge a revision leaf (see couch_db_updater),
the regular index update logic may now re-add the entries back into the index
with possibly a new winner.
After adding tests also clean up the tests to use the TDEF_FE macro and sigils
for binaries.
Fix: https://github.com/apache/couchdb/issues/6139
(cherry picked from commit 621c0602014688d2c6a764e92e1a530d935e9277)
---
src/couch_mrview/src/couch_mrview_updater.erl | 19 +-
.../test/eunit/couch_mrview_purge_docs_tests.erl | 1025 +++++++++++---------
2 files changed, 591 insertions(+), 453 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview_updater.erl b/src/couch_mrview/src/couch_mrview_updater.erl
index d642ceb5c..bebfc25ef 100644
--- a/src/couch_mrview/src/couch_mrview_updater.erl
+++ b/src/couch_mrview/src/couch_mrview_updater.erl
@@ -69,14 +69,22 @@ start_update(Partial, State, NumChanges, NumChangesDone) ->
{ok, InitState}.
-purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
+purge(Db, PurgeSeq, PurgedIdRevs, State) ->
#mrst{
id_btree = IdBtree,
views = Views,
partitioned = Partitioned
} = State,
- Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
+ % Keep index entries for docs which still exist and didn't change since the
+ % last index update. An FDI which still exists and changed since the last
+ % index update will have a newer update sequence (because of mrview updater
+ % sequence relabeling) or the doc was re-created, will be processed by the
+ % main indexing update path.
+ Ids0 = [Id || {Id, _Revs} <- PurgedIdRevs],
+ FDIs = couch_db:get_full_doc_infos(Db, Ids0),
+ IdxSeq = State#mrst.update_seq,
+ Ids = [Id || {Id, FDI} <- lists:zip(Ids0, FDIs), gone_or_changed_since(FDI, IdxSeq)],
{ok, Lookups, IdBtree2} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
MakeDictFun = fun
@@ -119,6 +127,13 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
purge_seq = PurgeSeq
}}.
+% For purging figure out if the FDI was completely purged or has changed since
+% the given index sequence (the last seq index processed).
+gone_or_changed_since(not_found, _IdxSeq) ->
+ true;
+gone_or_changed_since(#full_doc_info{update_seq = Seq}, IdxSeq) ->
+ Seq > IdxSeq.
+
process_doc(Doc, Seq, #mrst{doc_acc = Acc} = State) when length(Acc) > 100 ->
couch_work_queue:queue(State#mrst.doc_queue, lists:reverse(Acc)),
process_doc(Doc, Seq, State#mrst{doc_acc = []});
diff --git a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl
index c4ac9ed13..bda1ae5cc 100644
--- a/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl
+++ b/src/couch_mrview/test/eunit/couch_mrview_purge_docs_tests.erl
@@ -41,490 +41,613 @@ view_purge_test_() ->
fun setup/0,
fun teardown/1,
[
- fun test_purge_single/1,
- fun test_purge_partial/1,
- fun test_purge_complete/1,
- fun test_purge_nochange/1,
- fun test_purge_index_reset/1,
- fun test_purge_compact_size_check/1,
- fun test_purge_single_for_docid_with_list/1,
- fun test_purge_complete_for_docid_with_list/1,
- fun test_purge_compact_for_stale_purge_cp_without_client/1,
- fun test_purge_compact_for_stale_purge_cp_with_client/1
+ ?TDEF_FE(test_purge_single),
+ ?TDEF_FE(test_purge_partial),
+ ?TDEF_FE(test_purge_complete),
+ ?TDEF_FE(test_purge_nochange),
+ ?TDEF_FE(test_purge_nochange_non_leaf_rev),
+ ?TDEF_FE(test_purge_nochange_unknown_rev),
+ ?TDEF_FE(test_purge_partial_indexed_winner),
+ ?TDEF_FE(test_purge_recreate_indexed),
+ ?TDEF_FE(test_purge_index_reset),
+ ?TDEF_FE(test_purge_compact_size_check),
+ ?TDEF_FE(test_purge_single_for_docid_with_list),
+ ?TDEF_FE(test_purge_complete_for_docid_with_list),
+ ?TDEF_FE(test_purge_compact_for_stale_purge_cp_without_client),
+ ?TDEF_FE(test_purge_compact_for_stale_purge_cp_with_client)
]
}
}
}.
test_purge_single(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev = get_rev(FDI),
- {ok, [{ok, _PRevs}]} = couch_db:purge_docs(
- Db,
- [{<<"UUID1">>, <<"1">>, [Rev]}]
- ),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 4}, {offset, 0}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev = get_rev(FDI),
+ {ok, [{ok, _PRevs}]} = couch_db:purge_docs(
+ Db,
+ [{~"UUID1", ~"1", [Rev]}]
+ ),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 4}, {offset, 0}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_single_for_docid_with_list(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev = get_rev(FDI),
- {ok, [{ok, _PRevs}]} = couch_db:purge_docs(
- Db,
- [{<<"UUID1">>, "1", [Rev]}]
- ),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 4}, {offset, 0}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev = get_rev(FDI),
+ {ok, [{ok, _PRevs}]} = couch_db:purge_docs(
+ Db,
+ [{~"UUID1", "1", [Rev]}]
+ ),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 4}, {offset, 0}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_partial(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI1 = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev1 = get_rev(FDI1),
- Update =
- {[
- {'_id', <<"1">>},
- {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(<<"1.2">>)]})},
- {'val', 1.2}
- ]},
- {ok, [_Rev2]} = save_docs(Db, [Update], [?REPLICATED_CHANGES]),
-
- PurgeInfos = [{<<"UUID1">>, <<"1">>, [Rev1]}],
-
- {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1.2}, {value, 1.2}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+ Update =
+ {[
+ {'_id', ~"1"},
+ {'_rev', couch_doc:rev_to_str({1, [couch_hash:md5_hash(~"1.2")]})},
+ {'val', 1.2}
+ ]},
+ {ok, [_Rev2]} = save_docs(Db, [Update], [?REPLICATED_CHANGES]),
+
+ PurgeInfos = [{~"UUID1", ~"1", [Rev1]}],
+
+ {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1.2}, {value, 1.2}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_complete(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI1 = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev1 = get_rev(FDI1),
- FDI2 = couch_db:get_full_doc_info(Db, <<"2">>),
- Rev2 = get_rev(FDI2),
- FDI5 = couch_db:get_full_doc_info(Db, <<"5">>),
- Rev5 = get_rev(FDI5),
-
- PurgeInfos = [
- {<<"UUID1">>, <<"1">>, [Rev1]},
- {<<"UUID2">>, <<"2">>, [Rev2]},
- {<<"UUID5">>, <<"5">>, [Rev5]}
- ],
- {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 2}, {offset, 0}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+ FDI2 = couch_db:get_full_doc_info(Db, ~"2"),
+ Rev2 = get_rev(FDI2),
+ FDI5 = couch_db:get_full_doc_info(Db, ~"5"),
+ Rev5 = get_rev(FDI5),
+
+ PurgeInfos = [
+ {~"UUID1", ~"1", [Rev1]},
+ {~"UUID2", ~"2", [Rev2]},
+ {~"UUID5", ~"5", [Rev5]}
+ ],
+ {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 2}, {offset, 0}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_complete_for_docid_with_list(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI1 = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev1 = get_rev(FDI1),
- FDI2 = couch_db:get_full_doc_info(Db, <<"2">>),
- Rev2 = get_rev(FDI2),
- FDI5 = couch_db:get_full_doc_info(Db, <<"5">>),
- Rev5 = get_rev(FDI5),
-
- PurgeInfos = [
- {<<"UUID1">>, "1", [Rev1]},
- {<<"UUID2">>, "2", [Rev2]},
- {<<"UUID5">>, "5", [Rev5]}
- ],
- {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 2}, {offset, 0}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+ FDI2 = couch_db:get_full_doc_info(Db, ~"2"),
+ Rev2 = get_rev(FDI2),
+ FDI5 = couch_db:get_full_doc_info(Db, ~"5"),
+ Rev5 = get_rev(FDI5),
+
+ PurgeInfos = [
+ {~"UUID1", "1", [Rev1]},
+ {~"UUID2", "2", [Rev2]},
+ {~"UUID5", "5", [Rev5]}
+ ],
+ {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 2}, {offset, 0}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_nochange(Db) ->
- ?_test(begin
- Result = run_query(Db, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- FDI1 = couch_db:get_full_doc_info(Db, <<"1">>),
- Rev1 = get_rev(FDI1),
-
- PurgeInfos = [
- {<<"UUID1">>, <<"6">>, [Rev1]}
- ],
- {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
- {ok, Db2} = couch_db:reopen(Db),
-
- Result2 = run_query(Db2, []),
- Expect2 =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect2, Result2)
- end).
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+
+ PurgeInfos = [
+ {~"UUID1", ~"6", [Rev1]}
+ ],
+ {ok, _} = couch_db:purge_docs(Db, PurgeInfos),
+ {ok, Db2} = couch_db:reopen(Db),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
+
+test_purge_nochange_non_leaf_rev(Db) ->
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+ Update =
+ {[
+ {'_id', ~"1"},
+ {'_rev', couch_doc:rev_to_str(Rev1)},
+ {'val', 42}
+ ]},
+ {ok, [_Rev2]} = save_docs(Db, [Update], []),
+ {ok, Db1} = couch_db:reopen(Db),
+
+ Result = run_query(Db1, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]},
+ {row, [{id, ~"1"}, {key, 42}, {value, 42}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ % Rev1 is not a leaf any longer so nothing is purged, but the
+ % purge request is still recorded and the index has to process it
+ PurgeInfos = [{~"UUID1", ~"1", [Rev1]}],
+ ?assertEqual({ok, [{ok, []}]}, couch_db:purge_docs(Db1, PurgeInfos)),
+ {ok, Db2} = couch_db:reopen(Db1),
+ ?assertEqual(1, couch_db:get_purge_seq(Db2)),
+
+ Result2 = run_query(Db2, []),
+ ?assertEqual(Expect, Result2),
+ ?assertEqual(1, get_local_purge_seq(Db2)).
+
+test_purge_nochange_unknown_rev(Db) ->
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ % A revision which never existed, so nothing is purged
+ UnknownRev = {1, couch_hash:md5_hash(~"unknown")},
+ PurgeInfos = [{~"UUID1", ~"1", [UnknownRev]}],
+ ?assertEqual({ok, [{ok, []}]}, couch_db:purge_docs(Db, PurgeInfos)),
+ {ok, Db2} = couch_db:reopen(Db),
+ ?assertEqual(1, couch_db:get_purge_seq(Db2)),
+
+ Result2 = run_query(Db2, []),
+ ?assertEqual(Expect, Result2),
+ ?assertEqual(1, get_local_purge_seq(Db2)).
+
+test_purge_partial_indexed_winner(Db) ->
+ % Add a longer conflicting branch so it becomes the winner
+ WinningRev = {2, couch_hash:md5_hash(~"1.2-2")},
+ Conflict = #doc{
+ id = ~"1",
+ revs = {2, [element(2, WinningRev), couch_hash:md5_hash(~"1.2-1")]},
+ body = {[{~"val", 1.2}]}
+ },
+ {ok, []} = couch_db:update_docs(Db, [Conflict], [], ?REPLICATED_CHANGES),
+ {ok, Db1} = couch_db:reopen(Db),
+
+ Result = run_query(Db1, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1.2}, {value, 1.2}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ % Purge the indexed winner so the original revision wins again
+ PurgeInfos = [{~"UUID1", ~"1", [WinningRev]}],
+ ?assertEqual({ok, [{ok, [WinningRev]}]}, couch_db:purge_docs(Db1, PurgeInfos)),
+ {ok, Db2} = couch_db:reopen(Db1),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
+
+test_purge_recreate_indexed(Db) ->
+ Result = run_query(Db, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ FDI1 = couch_db:get_full_doc_info(Db, ~"1"),
+ Rev1 = get_rev(FDI1),
+ PurgeInfos = [{~"UUID1", ~"1", [Rev1]}],
+ ?assertEqual({ok, [{ok, [Rev1]}]}, couch_db:purge_docs(Db, PurgeInfos)),
+ {ok, Db1} = couch_db:reopen(Db),
+
+ % Recreate doc before index processes the purge
+ {ok, [_]} = save_docs(Db1, [{[{'_id', ~"1"}, {'val', 42}]}], []),
+ {ok, Db2} = couch_db:reopen(Db1),
+
+ Result2 = run_query(Db2, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]},
+ {row, [{id, ~"1"}, {key, 42}, {value, 42}]}
+ ]},
+ ?assertEqual(Expect2, Result2).
test_purge_index_reset(Db) ->
- ?_test(begin
- ok = couch_db:set_purge_infos_limit(Db, 2),
- {ok, Db1} = couch_db:reopen(Db),
-
- Result = run_query(Db1, []),
- Expect =
- {ok, [
- {meta, [{total, 5}, {offset, 0}]},
- {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
- {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
- {row, [{id, <<"3">>}, {key, 3}, {value, 3}]},
- {row, [{id, <<"4">>}, {key, 4}, {value, 4}]},
- {row, [{id, <<"5">>}, {key, 5}, {value, 5}]}
- ]},
- ?assertEqual(Expect, Result),
-
- PurgeInfos = lists:map(
- fun(I) ->
- DocId = integer_to_binary(I),
- FDI = couch_db:get_full_doc_info(Db, DocId),
- Rev = get_rev(FDI),
- {couch_uuids:random(), DocId, [Rev]}
- end,
- lists:seq(1, 5)
- ),
- {ok, _} = couch_db:purge_docs(Db1, PurgeInfos),
-
- {ok, Db2} = couch_db:reopen(Db1),
-
- % Forcibly set the purge doc to a newer purge
- % sequence to force an index reset. This should
- % never happen in real life but the reset
- % is required for correctness.
- {ok, #doc{body = {OldProps}} = LocalDoc} = get_local_purge_doc(Db2),
- NewPurgeSeq = {<<"purge_seq">>, 5},
- NewProps = lists:keyreplace(<<"purge_seq">>, 1, OldProps, NewPurgeSeq),
- RewindDoc = LocalDoc#doc{body = {NewProps}},
- {ok, _} = couch_db:update_doc(Db2, RewindDoc, []),
-
- % Compact the database to remove purge infos
- {ok, _} = couch_db:start_compact(Db2),
- wait_compaction(couch_db:name(Db), "database", ?LINE),
-
- {ok, Db3} = couch_db:reopen(Db2),
- Result2 = run_query(Db3, []),
- Expect2 =
- {ok, [
- {meta, [{total, 0}, {offset, 0}]}
- ]},
- ?assertEqual(Expect2, Result2),
-
- % Assert that we had a reset
- meck:wait(
- 1,
- couch_index_updater,
- handle_info,
- [{'EXIT', '_', {reset, '_'}}, '_'],
- 5000
- )
- end).
+ ok = couch_db:set_purge_infos_limit(Db, 2),
+ {ok, Db1} = couch_db:reopen(Db),
+
+ Result = run_query(Db1, []),
+ Expect =
+ {ok, [
+ {meta, [{total, 5}, {offset, 0}]},
+ {row, [{id, ~"1"}, {key, 1}, {value, 1}]},
+ {row, [{id, ~"2"}, {key, 2}, {value, 2}]},
+ {row, [{id, ~"3"}, {key, 3}, {value, 3}]},
+ {row, [{id, ~"4"}, {key, 4}, {value, 4}]},
+ {row, [{id, ~"5"}, {key, 5}, {value, 5}]}
+ ]},
+ ?assertEqual(Expect, Result),
+
+ PurgeInfos = lists:map(
+ fun(I) ->
+ DocId = integer_to_binary(I),
+ FDI = couch_db:get_full_doc_info(Db, DocId),
+ Rev = get_rev(FDI),
+ {couch_uuids:random(), DocId, [Rev]}
+ end,
+ lists:seq(1, 5)
+ ),
+ {ok, _} = couch_db:purge_docs(Db1, PurgeInfos),
+
+ {ok, Db2} = couch_db:reopen(Db1),
+
+ % Forcibly set the purge doc to a newer purge
+ % sequence to force an index reset. This should
+ % never happen in real life but the reset
+ % is required for correctness.
+ {ok, #doc{body = {OldProps}} = LocalDoc} = get_local_purge_doc(Db2),
+ NewPurgeSeq = {~"purge_seq", 5},
+ NewProps = lists:keyreplace(~"purge_seq", 1, OldProps, NewPurgeSeq),
+ RewindDoc = LocalDoc#doc{body = {NewProps}},
+ {ok, _} = couch_db:update_doc(Db2, RewindDoc, []),
+
+ % Compact the database to remove purge infos
+ {ok, _} = couch_db:start_compact(Db2),
+ wait_compaction(couch_db:name(Db), "database", ?LINE),
+
+ {ok, Db3} = couch_db:reopen(Db2),
+ Result2 = run_query(Db3, []),
+ Expect2 =
+ {ok, [
+ {meta, [{total, 0}, {offset, 0}]}
+ ]},
+ ?assertEqual(Expect2, Result2),
+
+ % Assert that we had a reset
+ meck:wait(
+ 1,
+ couch_index_updater,
+ handle_info,
+ [{'EXIT', '_', {reset, '_'}}, '_'],
+ 5000
+ ).
test_purge_compact_size_check(Db) ->
- ?_test(begin
- DbName = couch_db:name(Db),
- Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
- {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
- _Result = run_query(Db1, []),
- DiskSizeBefore = db_disk_size(DbName),
-
- PurgedDocsNum = 150,
- IdsRevs = lists:foldl(
- fun(Id, CIdRevs) ->
- Id1 = docid(Id),
- FDI1 = couch_db:get_full_doc_info(Db1, Id1),
- Rev1 = get_rev(FDI1),
- UUID1 = uuid(Id),
- [{UUID1, Id1, [Rev1]} | CIdRevs]
- end,
- [],
- lists:seq(1, PurgedDocsNum)
- ),
- {ok, _} = couch_db:purge_docs(Db1, IdsRevs),
-
- {ok, Db2} = couch_db:reopen(Db1),
- _Result1 = run_query(Db2, []),
- {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
- Db2,
- 0,
- fun fold_fun/2,
- [],
- []
- ),
- ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)),
- config:set("couchdb", "file_compression", "snappy", false),
-
- {ok, Db3} = couch_db:open_int(DbName, []),
- {ok, _CompactPid} = couch_db:start_compact(Db3),
- wait_compaction(DbName, "database", ?LINE),
- ok = couch_db:close(Db3),
- DiskSizeAfter = db_disk_size(DbName),
- ?assert(DiskSizeBefore > DiskSizeAfter)
- end).
+ DbName = couch_db:name(Db),
+ Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
+ {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
+ _Result = run_query(Db1, []),
+ DiskSizeBefore = db_disk_size(DbName),
+
+ PurgedDocsNum = 150,
+ IdsRevs = lists:foldl(
+ fun(Id, CIdRevs) ->
+ Id1 = docid(Id),
+ FDI1 = couch_db:get_full_doc_info(Db1, Id1),
+ Rev1 = get_rev(FDI1),
+ UUID1 = uuid(Id),
+ [{UUID1, Id1, [Rev1]} | CIdRevs]
+ end,
+ [],
+ lists:seq(1, PurgedDocsNum)
+ ),
+ {ok, _} = couch_db:purge_docs(Db1, IdsRevs),
+
+ {ok, Db2} = couch_db:reopen(Db1),
+ _Result1 = run_query(Db2, []),
+ {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
+ Db2,
+ 0,
+ fun fold_fun/2,
+ [],
+ []
+ ),
+ ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)),
+ config:set("couchdb", "file_compression", "snappy", false),
+
+ {ok, Db3} = couch_db:open_int(DbName, []),
+ {ok, _CompactPid} = couch_db:start_compact(Db3),
+ wait_compaction(DbName, "database", ?LINE),
+ ok = couch_db:close(Db3),
+ DiskSizeAfter = db_disk_size(DbName),
+ ?assert(DiskSizeBefore > DiskSizeAfter).
test_purge_compact_for_stale_purge_cp_without_client(Db) ->
- ?_test(begin
- DbName = couch_db:name(Db),
- % add more documents to database for purge
- Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
- {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
-
- % change PurgedDocsLimit to 10 from 1000 to
- % avoid timeout of eunit test
- PurgedDocsLimit = 10,
- couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit),
-
- % purge 150 documents
- PurgedDocsNum = 150,
- PurgeInfos = lists:foldl(
- fun(Id, CIdRevs) ->
- Id1 = docid(Id),
- FDI1 = couch_db:get_full_doc_info(Db1, Id1),
- Rev1 = get_rev(FDI1),
- UUID1 = uuid(Id),
- [{UUID1, Id1, [Rev1]} | CIdRevs]
- end,
- [],
- lists:seq(1, PurgedDocsNum)
- ),
- {ok, _} = couch_db:purge_docs(Db1, PurgeInfos),
-
- {ok, Db2} = couch_db:reopen(Db1),
- {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
- Db2,
- 0,
- fun fold_fun/2,
- [],
- []
- ),
- ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)),
-
- % run compaction to trigger pruning of purge tree
- {ok, Db3} = couch_db:open_int(DbName, []),
- {ok, _CompactPid} = couch_db:start_compact(Db3),
- wait_compaction(DbName, "database", ?LINE),
- ok = couch_db:close(Db3),
-
- % check the remaining purge requests in purge tree
- {ok, Db4} = couch_db:reopen(Db3),
- OldestPSeq = couch_db:get_oldest_purge_seq(Db4),
- {ok, PurgedIdRevs2} = couch_db:fold_purge_infos(
- Db4,
- OldestPSeq - 1,
- fun fold_fun/2,
- [],
- []
- ),
- ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2))
- end).
+ DbName = couch_db:name(Db),
+ % add more documents to database for purge
+ Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
+ {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
+
+ % change PurgedDocsLimit to 10 from 1000 to
+ % avoid timeout of eunit test
+ PurgedDocsLimit = 10,
+ couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit),
+
+ % purge 150 documents
+ PurgedDocsNum = 150,
+ PurgeInfos = lists:foldl(
+ fun(Id, CIdRevs) ->
+ Id1 = docid(Id),
+ FDI1 = couch_db:get_full_doc_info(Db1, Id1),
+ Rev1 = get_rev(FDI1),
+ UUID1 = uuid(Id),
+ [{UUID1, Id1, [Rev1]} | CIdRevs]
+ end,
+ [],
+ lists:seq(1, PurgedDocsNum)
+ ),
+ {ok, _} = couch_db:purge_docs(Db1, PurgeInfos),
+
+ {ok, Db2} = couch_db:reopen(Db1),
+ {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
+ Db2,
+ 0,
+ fun fold_fun/2,
+ [],
+ []
+ ),
+ ?assertEqual(PurgedDocsNum, length(PurgedIdRevs)),
+
+ % run compaction to trigger pruning of purge tree
+ {ok, Db3} = couch_db:open_int(DbName, []),
+ {ok, _CompactPid} = couch_db:start_compact(Db3),
+ wait_compaction(DbName, "database", ?LINE),
+ ok = couch_db:close(Db3),
+
+ % check the remaining purge requests in purge tree
+ {ok, Db4} = couch_db:reopen(Db3),
+ OldestPSeq = couch_db:get_oldest_purge_seq(Db4),
+ {ok, PurgedIdRevs2} = couch_db:fold_purge_infos(
+ Db4,
+ OldestPSeq - 1,
+ fun fold_fun/2,
+ [],
+ []
+ ),
+ ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)).
test_purge_compact_for_stale_purge_cp_with_client(Db) ->
- ?_test(begin
- DbName = couch_db:name(Db),
- % add more documents to database for purge
- Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
- {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
-
- % change PurgedDocsLimit to 10 from 1000 to
- % avoid timeout of eunit test
- PurgedDocsLimit = 10,
- couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit),
- _Result = run_query(Db1, []),
-
- % first purge 30 documents
- PurgedDocsNum1 = 30,
- IdsRevs = lists:foldl(
- fun(Id, CIdRevs) ->
- Id1 = docid(Id),
- FDI1 = couch_db:get_full_doc_info(Db1, Id1),
- Rev1 = get_rev(FDI1),
- UUID1 = uuid(Id),
- [{UUID1, Id1, [Rev1]} | CIdRevs]
- end,
- [],
- lists:seq(1, PurgedDocsNum1)
- ),
- {ok, _} = couch_db:purge_docs(Db1, IdsRevs),
-
- {ok, Db2} = couch_db:reopen(Db1),
- % run query again to reflect purge request to mrview
- _Result1 = run_query(Db2, []),
- {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
- Db2,
- 0,
- fun fold_fun/2,
- [],
- []
- ),
- ?assertEqual(PurgedDocsNum1, length(PurgedIdRevs)),
-
- % then purge 120 documents
- PurgedDocsNum2 = 150,
- IdsRevs2 = lists:foldl(
- fun(Id, CIdRevs) ->
- Id1 = docid(Id),
- FDI1 = couch_db:get_full_doc_info(Db1, Id1),
- Rev1 = get_rev(FDI1),
- UUID1 = uuid(Id),
- [{UUID1, Id1, [Rev1]} | CIdRevs]
- end,
- [],
- lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2)
- ),
- {ok, _} = couch_db:purge_docs(Db2, IdsRevs2),
-
- % run compaction to trigger pruning of purge tree
- % only the first 30 purge requests are pruned
- {ok, Db3} = couch_db:open_int(DbName, []),
- {ok, _CompactPid} = couch_db:start_compact(Db3),
- wait_compaction(DbName, "database", ?LINE),
- ok = couch_db:close(Db3),
-
- % check the remaining purge requests in purge tree
- {ok, Db4} = couch_db:reopen(Db3),
- OldestPSeq = couch_db:get_oldest_purge_seq(Db4),
- {ok, PurgedIdRevs2} = couch_db:fold_purge_infos(
- Db4,
- OldestPSeq - 1,
- fun fold_fun/2,
- [],
- []
- ),
- ?assertEqual(PurgedDocsNum2 - PurgedDocsNum1, length(PurgedIdRevs2))
- end).
+ DbName = couch_db:name(Db),
+ % add more documents to database for purge
+ Docs = couch_mrview_test_util:make_docs(normal, 6, 200),
+ {ok, Db1} = couch_mrview_test_util:save_docs(Db, Docs),
+
+ % change PurgedDocsLimit to 10 from 1000 to
+ % avoid timeout of eunit test
+ PurgedDocsLimit = 10,
+ couch_db:set_purge_infos_limit(Db1, PurgedDocsLimit),
+ _Result = run_query(Db1, []),
+
+ % first purge 30 documents
+ PurgedDocsNum1 = 30,
+ IdsRevs = lists:foldl(
+ fun(Id, CIdRevs) ->
+ Id1 = docid(Id),
+ FDI1 = couch_db:get_full_doc_info(Db1, Id1),
+ Rev1 = get_rev(FDI1),
+ UUID1 = uuid(Id),
+ [{UUID1, Id1, [Rev1]} | CIdRevs]
+ end,
+ [],
+ lists:seq(1, PurgedDocsNum1)
+ ),
+ {ok, _} = couch_db:purge_docs(Db1, IdsRevs),
+
+ {ok, Db2} = couch_db:reopen(Db1),
+ % run query again to reflect purge request to mrview
+ _Result1 = run_query(Db2, []),
+ {ok, PurgedIdRevs} = couch_db:fold_purge_infos(
+ Db2,
+ 0,
+ fun fold_fun/2,
+ [],
+ []
+ ),
+ ?assertEqual(PurgedDocsNum1, length(PurgedIdRevs)),
+
+ % then purge 120 documents
+ PurgedDocsNum2 = 150,
+ IdsRevs2 = lists:foldl(
+ fun(Id, CIdRevs) ->
+ Id1 = docid(Id),
+ FDI1 = couch_db:get_full_doc_info(Db1, Id1),
+ Rev1 = get_rev(FDI1),
+ UUID1 = uuid(Id),
+ [{UUID1, Id1, [Rev1]} | CIdRevs]
+ end,
+ [],
+ lists:seq(PurgedDocsNum1 + 1, PurgedDocsNum2)
+ ),
+ {ok, _} = couch_db:purge_docs(Db2, IdsRevs2),
+
+ % run compaction to trigger pruning of purge tree
+ % only the first 30 purge requests are pruned
+ {ok, Db3} = couch_db:open_int(DbName, []),
+ {ok, _CompactPid} = couch_db:start_compact(Db3),
+ wait_compaction(DbName, "database", ?LINE),
+ ok = couch_db:close(Db3),
+
+ % check the remaining purge requests in purge tree
+ {ok, Db4} = couch_db:reopen(Db3),
+ OldestPSeq = couch_db:get_oldest_purge_seq(Db4),
+ {ok, PurgedIdRevs2} = couch_db:fold_purge_infos(
+ Db4,
+ OldestPSeq - 1,
+ fun fold_fun/2,
+ [],
+ []
+ ),
+ ?assertEqual(PurgedDocsNum2 - PurgedDocsNum1, length(PurgedIdRevs2)).
get_local_purge_doc(Db) ->
- {ok, DDoc} = couch_db:open_doc(Db, <<"_design/bar">>, []),
+ {ok, DDoc} = couch_db:open_doc(Db, ~"_design/bar", []),
{ok, IdxState} = couch_mrview_util:ddoc_to_mrst(couch_db:name(Db), DDoc),
Sig = IdxState#mrst.sig,
HexSig = list_to_binary(couch_index_util:hexsig(Sig)),
DocId = couch_mrview_util:get_local_purge_doc_id(HexSig),
couch_db:open_doc(Db, DocId, []).
+get_local_purge_seq(Db) ->
+ {ok, Db1} = couch_db:reopen(Db),
+ {ok, #doc{body = {Props}}} = get_local_purge_doc(Db1),
+ couch_util:get_value(~"purge_seq", Props).
+
run_query(Db, Opts) ->
- couch_mrview:query_view(Db, <<"_design/bar">>, <<"baz">>, Opts).
+ couch_mrview:query_view(Db, ~"_design/bar", ~"baz", Opts).
save_docs(Db, JsonDocs, Options) ->
Docs = lists:map(
(couchdb) 11/12: Improve weatherreport checks
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 99316730e668c9dfc1b1f56f0cb1db16256db903
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sun Oct 4 02:05:58 2026 -0400
Improve weatherreport checks
For better compatibility use `file:write_file_info/2` instead `touch`. Check if
commands exist with `os:find_executable/1` before running them. Use
`spawn_executable` instead of `spawn` to run the commands.
Co-Authored-By: Robert Newson <rn...@apache.org>
(cherry picked from commit e786ccf78efb689132766785c2c7282b79d2a30b)
---
src/weatherreport/src/weatherreport_check_disk.erl | 2 +-
.../src/weatherreport_check_memory_use.erl | 2 +-
.../src/weatherreport_check_tcp_queues.erl | 2 +-
src/weatherreport/src/weatherreport_config.erl | 2 +-
src/weatherreport/src/weatherreport_util.erl | 31 ++++++++++++++--------
5 files changed, 24 insertions(+), 15 deletions(-)
diff --git a/src/weatherreport/src/weatherreport_check_disk.erl b/src/weatherreport/src/weatherreport_check_disk.erl
index 5361ae632..dca84d053 100644
--- a/src/weatherreport/src/weatherreport_check_disk.erl
+++ b/src/weatherreport/src/weatherreport_check_disk.erl
@@ -180,7 +180,7 @@ check_is_file_readable(Directory) ->
%% Check if the directory is mounted with 'noatime'
check_atime(Directory) ->
File = filename:join([Directory, ?TEST_FILE]),
- weatherreport_util:run_command("touch -at 201401010000.00 " ++ File),
+ file:write_file_info(File, #file_info{atime = {{2014, 01, 01}, {0, 0, 0}}}),
{ok, FileInfo1} = file:read_file_info(File),
{ok, S} = file:open(File, [read]),
io:get_line(S, ''),
diff --git a/src/weatherreport/src/weatherreport_check_memory_use.erl b/src/weatherreport/src/weatherreport_check_memory_use.erl
index 04c021381..f18f02bb4 100644
--- a/src/weatherreport/src/weatherreport_check_memory_use.erl
+++ b/src/weatherreport/src/weatherreport_check_memory_use.erl
@@ -52,7 +52,7 @@ valid() ->
-spec check(list()) -> [{atom(), term()}].
check(_Opts) ->
Pid = weatherreport_node:pid(),
- Output = weatherreport_util:run_command("ps -o pmem,rss -p " ++ Pid),
+ Output = weatherreport_util:run_command("ps", ["-o", "pmem,rss", "-p", Pid]),
[_, _, Percent, RealSize | _] = string:tokens(Output, "/n \n"),
Messages = [{info, {process_usage, Percent, RealSize}}],
case weatherreport_util:binary_to_float(list_to_binary(Percent)) >= 90 of
diff --git a/src/weatherreport/src/weatherreport_check_tcp_queues.erl b/src/weatherreport/src/weatherreport_check_tcp_queues.erl
index cc502031b..6ed4ea254 100644
--- a/src/weatherreport/src/weatherreport_check_tcp_queues.erl
+++ b/src/weatherreport/src/weatherreport_check_tcp_queues.erl
@@ -77,7 +77,7 @@ sum_to_message(Sum, Prefix) ->
-spec check(list()) -> [{atom(), term()}].
check(_Opts) ->
- Netstats = weatherreport_util:run_command("netstat"),
+ Netstats = weatherreport_util:run_command("netstat", []),
{SumRecvQ, SumSendQ} = sum_queues(Netstats),
[sum_to_message(SumRecvQ, "recv_q"), sum_to_message(SumSendQ, "send_q")].
diff --git a/src/weatherreport/src/weatherreport_config.erl b/src/weatherreport/src/weatherreport_config.erl
index 6cf9fd533..47a81ab08 100644
--- a/src/weatherreport/src/weatherreport_config.erl
+++ b/src/weatherreport/src/weatherreport_config.erl
@@ -81,7 +81,7 @@ get_vm_env(Key) ->
%% @doc Determines the user/uid that the script is running as.
-spec user() -> string().
user() ->
- case weatherreport_util:run_command("whoami") of
+ case weatherreport_util:run_command("whoami", []) of
[] ->
undefined;
Resp ->
diff --git a/src/weatherreport/src/weatherreport_util.erl b/src/weatherreport/src/weatherreport_util.erl
index c4f9fb42a..8817f8d73 100644
--- a/src/weatherreport/src/weatherreport_util.erl
+++ b/src/weatherreport/src/weatherreport_util.erl
@@ -30,7 +30,7 @@
-module(weatherreport_util).
-export([
short_name/1,
- run_command/1,
+ run_command/2,
binary_to_float/1,
flush_stdout/0,
check_proc_count/3
@@ -46,16 +46,25 @@ short_name(Mod) when is_atom(Mod) ->
%% @doc Runs a shell command and returns the output. stderr is
%% redirected to stdout so its output will be included.
--spec run_command(Command :: iodata()) -> StdOut :: iodata().
-run_command(Command) ->
- weatherreport_log:log(
- node(),
- debug,
- "Running shell command: ~s",
- [Command]
- ),
- Port = open_port({spawn, Command}, [exit_status, stderr_to_stdout]),
- do_read(Port, []).
+-spec run_command(Command :: string(), Args :: [string() | binary()]) -> StdOut :: iodata().
+run_command(Command, Args) ->
+ case os:find_executable(Command) of
+ false ->
+ [];
+ FileName ->
+ weatherreport_log:log(
+ node(),
+ debug,
+ "Running shell command: ~s with args: ~p",
+ [FileName, Args]
+ ),
+ Port = open_port({spawn_executable, FileName}, [
+ {args, Args},
+ exit_status,
+ stderr_to_stdout
+ ]),
+ do_read(Port, [])
+ end.
do_read(Port, Acc) ->
receive
(couchdb) 07/12: Fix suffix check for replicator and users dbs
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a4cfbb22b666930911dfd78e79586decbfcef5a9
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sat Oct 3 02:32:09 2026 -0400
Fix suffix check for replicator and users dbs
Match the plain users and replicator db behavior.
Co-Authored-By: Jan Lehnardt <ja...@apache.org>
(cherry picked from commit 014bbf069794b45ae3b748f1bcecd3b4fc121975)
---
src/chttpd/src/chttpd_auth_request.erl | 147 ++++++++++++++++++++++++++++++++-
1 file changed, 143 insertions(+), 4 deletions(-)
diff --git a/src/chttpd/src/chttpd_auth_request.erl b/src/chttpd/src/chttpd_auth_request.erl
index b62958721..7ce638ad9 100644
--- a/src/chttpd/src/chttpd_auth_request.erl
+++ b/src/chttpd/src/chttpd_auth_request.erl
@@ -41,9 +41,15 @@ authorize_request_int(#httpd{path_parts = [<<"_replicator">>], method = 'PUT'} =
require_admin(Req);
authorize_request_int(#httpd{path_parts = [<<"_replicator">>], method = 'DELETE'} = Req) ->
require_admin(Req);
-authorize_request_int(#httpd{path_parts = [<<"_replicator">>, <<"_all_docs">> | _]} = Req) ->
+authorize_request_int(#httpd{path_parts = [DbName, <<"_all_docs">> | _]} = Req) when
+ DbName =:= <<"_replicator">>;
+ binary_part(DbName, {byte_size(DbName), -12}) =:= <<"/_replicator">>
+->
require_admin(Req);
-authorize_request_int(#httpd{path_parts = [<<"_replicator">>, <<"_changes">> | _]} = Req) ->
+authorize_request_int(#httpd{path_parts = [DbName, <<"_changes">> | _]} = Req) when
+ DbName =:= <<"_replicator">>;
+ binary_part(DbName, {byte_size(DbName), -12}) =:= <<"/_replicator">>
+->
require_admin(Req);
authorize_request_int(#httpd{path_parts = [<<"_replicator">> | _]} = Req) ->
db_authorization_check(Req);
@@ -53,9 +59,15 @@ authorize_request_int(#httpd{path_parts = [<<"_users">>], method = 'PUT'} = Req)
require_admin(Req);
authorize_request_int(#httpd{path_parts = [<<"_users">>], method = 'DELETE'} = Req) ->
require_admin(Req);
-authorize_request_int(#httpd{path_parts = [<<"_users">>, <<"_all_docs">> | _]} = Req) ->
+authorize_request_int(#httpd{path_parts = [DbName, <<"_all_docs">> | _]} = Req) when
+ DbName =:= <<"_users">>;
+ binary_part(DbName, {byte_size(DbName), -7}) =:= <<"/_users">>
+->
require_admin(Req);
-authorize_request_int(#httpd{path_parts = [<<"_users">>, <<"_changes">> | _]} = Req) ->
+authorize_request_int(#httpd{path_parts = [DbName, <<"_changes">> | _]} = Req) when
+ DbName =:= <<"_users">>;
+ binary_part(DbName, {byte_size(DbName), -7}) =:= <<"/_users">>
+->
require_admin(Req);
authorize_request_int(#httpd{path_parts = [<<"_users">> | _]} = Req) ->
db_authorization_check(Req);
@@ -166,3 +178,130 @@ maybe_admin_only_dbs(Req) ->
true -> require_admin(Req);
false -> Req
end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+users_all_docs_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"_users">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+users_all_docs_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"_users">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+foo_users_all_docs_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"foo/_users">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+foo_users_all_docs_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"foo/_users">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+users_changes_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"_users">>, <<"_changes">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+users_changes_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"_users">>, <<"_changes">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+foo_users_changes_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"foo/_users">>, <<"_changes">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+foo_users_changes_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"foo/_users">>, <<"_changes">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+%%%
+
+replicator_all_docs_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"_replicator">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+replicator_all_docs_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"_replicator">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+foo_replicator_all_docs_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"foo/_replicator">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+foo_replicator_all_docs_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"foo/_replicator">>, <<"_all_docs">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+replicator_changes_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"_replicator">>, <<"_changes">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+replicator_changes_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"_replicator">>, <<"_changes">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+foo_replicator_changes_no_admin_test() ->
+ Throw = {unauthorized, <<"You are not a server admin.">>},
+ Req = #httpd{
+ path_parts = [<<"foo/_replicator">>, <<"_changes">>],
+ user_ctx = #user_ctx{}
+ },
+ ?assertThrow(Throw, authorize_request_int(Req)).
+
+foo_replicator_changes_admin_test() ->
+ Req = #httpd{
+ path_parts = [<<"foo/_replicator">>, <<"_changes">>],
+ user_ctx = #user_ctx{roles = [<<"_admin">>]}
+ },
+ ?assertEqual(Req, authorize_request_int(Req)).
+
+-endif.
(couchdb) 12/12: Update whatsnew
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8274740df358b94c1aa928b549dd3328e49298f0
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Sun Oct 4 12:58:33 2026 -0400
Update whatsnew
Bring in a few more fixes for 3.5.3
---
src/docs/src/whatsnew/3.5.rst | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/docs/src/whatsnew/3.5.rst b/src/docs/src/whatsnew/3.5.rst
index 58e1a2439..3ee7cae37 100644
--- a/src/docs/src/whatsnew/3.5.rst
+++ b/src/docs/src/whatsnew/3.5.rst
@@ -76,6 +76,7 @@ Bugfixes
verified, in view, Dreyfus and Nouveau purge checkpoints.
* :ghissue:`6130`: Preserve view purge sequences when compacting. No longer
skip purges that happen right before a view is compacted.
+* :ghissue:`6140`: Fix purging non-leaf revisions in map-reduce views.
* :ghissue:`6129`: Fix attachment revision positions when re-creating a deleted
document. Attachments re-created with the same name will now replicate from
source to target when previously they weren't.
@@ -87,6 +88,7 @@ Bugfixes
source shard check.
* :ghissue:`6102`: Fix ``group_by_proximity`` returning an empty same-zone
shard list when the coordinator does not hold a copy of the shard.
+* :ghissue:`6145`: Fix spurious ``500`` during concurrent database deletion.
* :ghissue:`5948`: Handle ``{bad_request, Error, Reason}`` triples from
design document validators in ``fabric_doc_update``.
* :ghissue:`5839`: Only increment the ``vdu_rejects`` counter on actual VDU
@@ -100,6 +102,7 @@ Bugfixes
in all cases in the index manager.
* :ghissue:`6125`: Nouveau: fix string sorting order to match Java's
lexicographic order.
+* :ghissue:`6138`: Nouveau: fix purge out of order index update
* :ghissue:`6018`, :ghissue:`6021`, :ghissue:`6023`, :ghissue:`6133`: QuickJS
engine updates.
* :ghissue:`6058`: Update Jiffy to ``2.0.2``
@@ -109,6 +112,16 @@ Bugfixes
* :ghissue:`6068`: Add a query server ``process_count`` gauge.
* :ghissue:`6070`: Supervisor-level replicator auth plugin callbacks
``sup_initialize/0`` and ``sup_cleanup/1``.
+* :ghissue:`6141`: Limit number of redirects in replicator.
+* :ghissue:`6147`: Make ``_search_cleanup`` and ``_nouveau_cleanup`` admin-only
+ like ``_view_cleanup``.
+* :ghissue:`6149`: Fix the ``_users`` and ``_replicator`` database name suffix
+ check in the auth handler.
+* :ghissue:`6148`: Fix default proxy authentication headers config value.
+* :ghissue:`6151`: Use incremental decompression for reques bodies
+* :ghissue:`6152`: Use short names for auth handlers. The previous tuple forms
+ are still accepted.
+* :ghissue:`6153`: Improve command running in ``weatherreport``
Docs
----
(couchdb) 03/12: Fix out of order update error in nouveau
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6d3613a8ffea710f39ddfc58e61343a239700aae
Author: Tom Wier <th...@medic.org>
AuthorDate: Tue Sep 29 12:39:52 2026 +0300
Fix out of order update error in nouveau
(cherry picked from commit 522771806d1daa478574465cca555cc3299c1565)
---
src/nouveau/src/nouveau_index_updater.erl | 114 ++++++++++++------------------
test/elixir/test/config/nouveau.elixir | 1 +
test/elixir/test/nouveau_test.exs | 53 ++++++++++++++
3 files changed, 100 insertions(+), 68 deletions(-)
diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl
index 38e284a95..9f9de5844 100644
--- a/src/nouveau/src/nouveau_index_updater.erl
+++ b/src/nouveau/src/nouveau_index_updater.erl
@@ -32,7 +32,6 @@
proc,
changes_done,
total_changes,
- exclude_idrevs,
update_seq,
batch_size,
batch
@@ -40,8 +39,6 @@
-record(purge_acc, {
index,
- exclude_list = [],
- index_update_seq,
index_purge_seq,
batch_size,
batch
@@ -89,12 +86,11 @@ update(#index{} = Index) ->
PurgeAcc0 = #purge_acc{
index = Index,
- index_update_seq = IndexUpdateSeq,
index_purge_seq = IndexPurgeSeq,
batch_size = config:get_integer("nouveau", "batch_size", 20),
batch = []
},
- {ok, PurgeAcc1} = purge_index(Db, Index, PurgeAcc0),
+ ok = purge_index(Db, Index, IndexUpdateSeq, PurgeAcc0),
NewCurSeq = couch_db:get_update_seq(Db),
Proc = get_os_process(Index#index.def_lang),
@@ -107,8 +103,7 @@ update(#index{} = Index) ->
proc = Proc,
changes_done = 0,
total_changes = TotalChanges,
- exclude_idrevs = PurgeAcc1#purge_acc.exclude_list,
- update_seq = PurgeAcc1#purge_acc.index_update_seq,
+ update_seq = IndexUpdateSeq,
batch_size = config:get_integer("nouveau", "batch_size", 20),
batch = []
},
@@ -137,21 +132,13 @@ load_docs(FDI, #acc{} = Acc1) ->
{progress, (Acc1#acc.changes_done * 100) div Acc1#acc.total_changes}
]),
DI = couch_doc:to_doc_info(FDI),
- #doc_info{id = Id, revs = [#rev_info{rev = Rev} | _]} = DI,
-
- Acc2 =
- case lists:member({Id, Rev}, Acc1#acc.exclude_idrevs) of
- true ->
- Acc1;
- false ->
- Item = update_or_delete_index(
- Acc1#acc.db, Acc1#acc.update_seq, DI, Acc1#acc.proc
- ),
- Acc1#acc{
- batch = [Item | Acc1#acc.batch],
- update_seq = DI#doc_info.high_seq
- }
- end,
+ Item = update_or_delete_index(
+ Acc1#acc.db, Acc1#acc.update_seq, DI, Acc1#acc.proc
+ ),
+ Acc2 = Acc1#acc{
+ batch = [Item | Acc1#acc.batch],
+ update_seq = DI#doc_info.high_seq
+ },
case maybe_flush_batch(Acc2) of
{ok, Acc3} ->
{ok, Acc3};
@@ -251,53 +238,44 @@ index_definition(#index{} = Index, InitialPurgeSeq) ->
<<"initial_purge_seq">> => InitialPurgeSeq
}.
-purge_index(Db, Index, #purge_acc{} = PurgeAcc0) ->
- Proc = get_os_process(Index#index.def_lang),
- try
- true = proc_prompt(Proc, [<<"add_fun">>, Index#index.def, <<"nouveau">>]),
- FoldFun = fun({PurgeSeq, _UUID, Id, _Revs}, #purge_acc{} = PurgeAcc1) ->
- PurgeAcc3 =
- case couch_db:get_full_doc_info(Db, Id) of
- not_found ->
- Item = nouveau_api:make_purge(
- Id, PurgeAcc1#purge_acc.index_purge_seq, PurgeSeq
- ),
- PurgeAcc2 = PurgeAcc1#purge_acc{batch = [Item | PurgeAcc1#purge_acc.batch]},
- PurgeAcc2#purge_acc{index_purge_seq = PurgeSeq};
- FDI ->
- DI = couch_doc:to_doc_info(FDI),
- #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{rev = Rev} | _]} = DI,
- case lists:member({Id, Rev}, PurgeAcc1#purge_acc.exclude_list) of
- true ->
- PurgeAcc1;
- false ->
- Item = update_or_delete_index(
- Db, PurgeAcc1#purge_acc.index_update_seq, DI, Proc
- ),
- PurgeAcc1#purge_acc{
- batch = [Item | PurgeAcc1#purge_acc.batch],
- exclude_list = [{Id, Rev} | PurgeAcc1#purge_acc.exclude_list],
- index_update_seq = Seq
- }
- end
- end,
- update_task(1),
- maybe_flush_batch(PurgeAcc3)
- end,
+purge_index(Db, Index, StartUpdateSeq, #purge_acc{} = PurgeAcc0) ->
+ FoldFun = fun({PurgeSeq, _UUID, Id, _Revs}, #purge_acc{} = PurgeAcc1) ->
+ PurgeAcc2 =
+ case couch_db:get_full_doc_info(Db, Id) of
+ not_found ->
+ queue_purge(Id, PurgeSeq, PurgeAcc1);
+ #full_doc_info{} = FDI ->
+ #doc_info{high_seq = Seq} = couch_doc:to_doc_info(FDI),
+ case Seq > StartUpdateSeq of
+ true ->
+ queue_purge(Id, PurgeSeq, PurgeAcc1);
+ false ->
+ PurgeAcc1
+ end
+ end,
+ update_task(1),
+ maybe_flush_batch(PurgeAcc2)
+ end,
- {ok, #purge_acc{} = PurgeAcc3} = couch_db:fold_purge_infos(
- Db, PurgeAcc0#purge_acc.index_purge_seq, FoldFun, PurgeAcc0, []
- ),
- {ok, PurgeAcc4} = flush_batch(PurgeAcc3),
- DbPurgeSeq = couch_db:get_purge_seq(Db),
- ok = nouveau_api:set_purge_seq(
- Index, PurgeAcc4#purge_acc.index_purge_seq, DbPurgeSeq
- ),
- update_local_doc(Db, Index, DbPurgeSeq),
- {ok, PurgeAcc4}
- after
- ret_os_process(Proc)
- end.
+ {ok, #purge_acc{} = PurgeAcc3} = couch_db:fold_purge_infos(
+ Db, PurgeAcc0#purge_acc.index_purge_seq, FoldFun, PurgeAcc0, []
+ ),
+ {ok, PurgeAcc4} = flush_batch(PurgeAcc3),
+ DbPurgeSeq = couch_db:get_purge_seq(Db),
+ ok = nouveau_api:set_purge_seq(
+ Index, PurgeAcc4#purge_acc.index_purge_seq, DbPurgeSeq
+ ),
+ update_local_doc(Db, Index, DbPurgeSeq),
+ ok.
+
+queue_purge(Id, PurgeSeq, #purge_acc{} = PurgeAcc) ->
+ Item = nouveau_api:make_purge(
+ Id, PurgeAcc#purge_acc.index_purge_seq, PurgeSeq
+ ),
+ PurgeAcc#purge_acc{
+ batch = [Item | PurgeAcc#purge_acc.batch],
+ index_purge_seq = PurgeSeq
+ }.
update_task(NumChanges) ->
[Changes, Total] = couch_task_status:get([changes_done, total_changes]),
diff --git a/test/elixir/test/config/nouveau.elixir b/test/elixir/test/config/nouveau.elixir
index 7fd6ff195..5d620080a 100644
--- a/test/elixir/test/config/nouveau.elixir
+++ b/test/elixir/test/config/nouveau.elixir
@@ -31,6 +31,7 @@
"delete",
"purge",
"purge with conflicts",
+ "purge with out of order edits",
"index same field with different field types",
"index not found",
"index function throws",
diff --git a/test/elixir/test/nouveau_test.exs b/test/elixir/test/nouveau_test.exs
index c6d1103ef..9a589602b 100644
--- a/test/elixir/test/nouveau_test.exs
+++ b/test/elixir/test/nouveau_test.exs
@@ -706,6 +706,59 @@ defmodule NouveauTest do
assert seq(db_info["purge_seq"]) == resp.body["search_index"]["purge_seq"]
end
+ @tag :with_db
+ @tag query: %{q: 1}
+ test "purge with out of order edits", context do
+ db_name = context[:db_name]
+ create_resp = create_search_docs(db_name)
+ create_ddoc(db_name)
+
+ search_url = "/#{db_name}/_design/foo/_nouveau/bar"
+
+ # confirm all hits
+ resp = Couch.get(search_url, query: %{q: "*:*", include_docs: true})
+ assert_status_code(resp, 200)
+ assert get_total_hits(resp) == 4
+
+ # purge two docs in separate requests
+ doc_a = (Enum.at(create_resp.body, 0))
+ resp =
+ Couch.post("/#{db_name}/_purge",
+ body: %{doc_a["id"] => [doc_a["rev"]]}
+ )
+ assert_status_code(resp, 201)
+
+ doc_b = (Enum.at(create_resp.body, 1))
+ resp =
+ Couch.post("/#{db_name}/_purge",
+ body: %{doc_b["id"] => [doc_b["rev"]]}
+ )
+ assert_status_code(resp, 201)
+
+ # add another doc to catch if update seq has skipped docs
+ resp = Couch.put("/#{db_name}/docC", body: %{"foo" => "c", "bar" => 1})
+ assert_status_code(resp, 201)
+
+ # recreate the purged docs in a different order
+ resp = Couch.put("/#{db_name}/#{doc_b["id"]}", body: %{"foo" => "foo", "bar" => 42})
+ assert_status_code(resp, 201)
+
+ resp = Couch.put("/#{db_name}/#{doc_a["id"]}", body: %{"foo" => "foo", "bar" => 42})
+ assert_status_code(resp, 201)
+
+ # confirm a and b are back
+ resp = Couch.get(search_url, query: %{q: "*:*", include_docs: true})
+ assert_status_code(resp, 200)
+ assert get_total_hits(resp) == 5
+
+ # confirm nouveau responds and has correct sequences
+ resp = Couch.get("/#{db_name}/_design/foo/_nouveau_info/bar")
+ assert_status_code(resp, 200)
+ db_info = info(db_name)
+ assert seq(db_info["update_seq"]) == resp.body["search_index"]["update_seq"]
+ assert seq(db_info["purge_seq"]) == resp.body["search_index"]["purge_seq"]
+ end
+
@tag :with_db
test "index same field with different field types", context do
db_name = context[:db_name]
(couchdb) 05/12: Improve redirects in replicator
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch 3.5.x-a-few-more-fixes
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a9010b449a200966783d0476a24d0bf9bbd507b5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Wed Sep 30 01:06:16 2026 -0400
Improve redirects in replicator
Limit replicator redirects to a few tries.
Co-Authored-By: Robert Newson <rn...@apache.org>
(cherry picked from commit 8a9c2bdcf44b35d45852d47ce002c094106f6555)
---
src/couch_replicator/src/couch_replicator_httpc.erl | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl
index fe81e65ea..37c7b1d10 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -28,6 +28,7 @@
-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
-define(MAX_WAIT, 5 * 60 * 1000).
+-define(MAX_REDIRECTS, 5).
-define(STREAM_STATUS, ibrowse_stream_status).
-define(STOP_HTTP_WORKER, stop_http_worker).
@@ -458,10 +459,16 @@ query_args_to_string([], Acc) ->
query_args_to_string([{K, V} | Rest], Acc) ->
query_args_to_string(Rest, [K ++ "=" ++ couch_httpd:quote(V) | Acc]).
-do_redirect(_Worker, Code, Headers, #httpdb{url = Url} = HttpDb, Params, _Cb) ->
- RedirectUrl = redirect_url(Headers, Url),
- {HttpDb2, Params2} = after_redirect(RedirectUrl, Code, HttpDb, Params),
- throw({retry, HttpDb2, Params2}).
+do_redirect(Worker, Code, Headers, #httpdb{url = Url} = HttpDb, Params, _Cb) ->
+ Redirects = get_value(redirects, Params, 0),
+ case Redirects >= ?MAX_REDIRECTS of
+ true ->
+ report_error(Worker, HttpDb, Params, {error, too_many_redirects});
+ false ->
+ RedirectUrl = redirect_url(Headers, Url),
+ {HttpDb2, Params2} = after_redirect(RedirectUrl, Code, HttpDb, Params),
+ throw({retry, HttpDb2, ?replace(Params2, redirects, Redirects + 1)})
+ end.
redirect_url(RespHeaders, OrigUrl) ->
MochiHeaders = mochiweb_headers:make(RespHeaders),