You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2023/02/24 22:07:35 UTC
[couchdb] branch main updated (301797c56 -> ac600d1bb)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
from 301797c56 Update Erlang 24 for CI
new 7a2705ac6 hide shard-sync and purge documents from `_local_docs`
new b702a5edf inspect the document more precisely to avoid skipping user docs
new 8f519ba04 add tests to cover hiding resp showing system docs
new ac600d1bb add a notice that purge documents are only visible with `include_system=true`
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd_db.erl | 9 +-
src/chttpd/test/eunit/chttpd_local_docs_tests.erl | 198 ++++++++++++++++++++++
src/couch_mrview/src/couch_mrview.erl | 84 +++++----
src/docs/src/cluster/purging.rst | 6 +-
4 files changed, 265 insertions(+), 32 deletions(-)
create mode 100644 src/chttpd/test/eunit/chttpd_local_docs_tests.erl
[couchdb] 02/04: inspect the document more precisely to avoid skipping user docs
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b702a5edff969abac0628e0cf16ef6412ba5c292
Author: Christian Kruse <cj...@defunct.ch>
AuthorDate: Tue Feb 21 20:45:19 2023 +0100
inspect the document more precisely to avoid skipping user docs
---
src/couch_mrview/src/couch_mrview.erl | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index 3616af7df..c0f6ff49b 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -531,13 +531,11 @@ map_fold({{Key, Id}, Val}, _Offset, Acc) ->
user_acc = UAcc1,
last_go = Go
}};
-map_fold(#doc{id = <<"_local/", IdTail/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
+map_fold(#doc{id = <<"_local/", _/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
IncludeSys = couch_util:get_value(include_system, Acc#mracc.args#mrargs.extra),
- case {IncludeSys, IdTail} of
- {false, <<"purge-", _/binary>>} ->
- {ok, Acc};
- {false, <<"shard-sync", _/binary>>} ->
+ case should_skip_local_document(IncludeSys, Doc) of
+ true ->
{ok, Acc};
_ ->
#mracc{
@@ -571,6 +569,25 @@ map_fold(#doc{id = <<"_local/", IdTail/binary>>} = Doc, _Offset, #mracc{} = Acc)
}}
end.
+should_skip_local_document(_IncludeSys = false, #doc{
+ id = <<"_local/purge-", _/binary>>, body = {Props}
+}) ->
+ contains_fields([<<"purge_seq">>], Props);
+should_skip_local_document(_IncludeSys = false, #doc{
+ id = <<"_local/shard-sync", _/binary>>, body = {Props}
+}) ->
+ contains_fields([<<"target_uuid">>, <<"seq">>], Props);
+should_skip_local_document(_IncludeSys, _Doc) ->
+ false.
+
+contains_fields([], _Props) ->
+ true;
+contains_fields([Field | Fields], Props) ->
+ case couch_util:get_value(Field, Props) of
+ undefined -> false;
+ _ -> contains_fields(Fields, Props)
+ end.
+
red_fold(Db, {NthRed, _Lang, View} = RedView, Args, Callback, UAcc) ->
Finalizer =
case couch_util:get_value(finalizer, Args#mrargs.extra) of
[couchdb] 03/04: add tests to cover hiding resp showing system docs
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8f519ba04f90ab32c19109034c584e966c7633eb
Author: Christian Kruse <cj...@defunct.ch>
AuthorDate: Fri Feb 24 21:56:31 2023 +0100
add tests to cover hiding resp showing system docs
---
src/chttpd/test/eunit/chttpd_local_docs_tests.erl | 198 ++++++++++++++++++++++
1 file changed, 198 insertions(+)
diff --git a/src/chttpd/test/eunit/chttpd_local_docs_tests.erl b/src/chttpd/test/eunit/chttpd_local_docs_tests.erl
new file mode 100644
index 000000000..e30a70bb4
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_local_docs_tests.erl
@@ -0,0 +1,198 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_local_docs_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_local_docs_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(JSON, {"Content-Type", "application/json"}).
+
+setup() ->
+ Hashed = couch_passwords:hash_admin_password(?PASS),
+ ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false),
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Db = binary_to_list(?tempdb()),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Url = lists:concat(["http://", Addr, ":", Port, "/"]),
+ ok = create_db(Url, Db),
+ {Url, Db}.
+
+teardown({Url, Db}) ->
+ delete_db(Url, Db),
+ ok = config:delete("admins", ?USER, _Persist = false).
+
+start_couch() ->
+ test_util:start_couch([chttpd]).
+
+stop_couch(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+chttpd_local_docs_test_() ->
+ {
+ "chttpd _local_docs tests",
+ {
+ setup,
+ fun start_couch/0,
+ fun stop_couch/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(t_local_docs_doesnt_include_system_docs),
+ ?TDEF_FE(t_local_docs_does_include_system_docs_w_param)
+ ]
+ }
+ }
+ }.
+
+t_local_docs_doesnt_include_system_docs({Top, Db}) ->
+ Path = Top ++ Db,
+ Docs = [
+ #{
+ <<"_id">> => <<"_local/shard-sync-foo-bar">>,
+ <<"_rev">> => <<"0-1">>,
+ <<"seq">> => 1,
+ <<"target_uuid">> => <<"4f3eb3181db904d62176fb81626f84d0">>
+ },
+ #{
+ <<"_id">> => <<"_local/purge-mrview-foo-bar">>,
+ <<"_rev">> => <<"0-1">>,
+ <<"ddoc_id">> => <<"_design/ddoc1">>,
+ <<"purge_seq">> => 0,
+ <<"signature">> => [1, 2, 3],
+ <<"type">> => <<"mrview">>,
+ <<"updated_on">> => 1676995502
+ },
+ #{
+ <<"_id">> => <<"_local/1-foo-bar">>
+ },
+ #{
+ <<"_id">> => <<"_local/2-foo-bar">>
+ }
+ ],
+ Body = #{<<"docs">> => Docs},
+ {Code, _} = req(post, Path ++ "/_bulk_docs", Body),
+ ?assertEqual(201, Code),
+ {Code1, Res} = req(get, Path ++ "/_local_docs"),
+ ?assertEqual(200, Code1),
+ ?assertEqual(
+ #{
+ <<"total_rows">> => null,
+ <<"offset">> => null,
+ <<"rows">> => [
+ #{
+ <<"id">> => <<"_local/1-foo-bar">>,
+ <<"key">> => <<"_local/1-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-1">>}
+ },
+ #{
+ <<"id">> => <<"_local/2-foo-bar">>,
+ <<"key">> => <<"_local/2-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-1">>}
+ }
+ ]
+ },
+ Res
+ ).
+
+t_local_docs_does_include_system_docs_w_param({Top, Db}) ->
+ Path = Top ++ Db,
+ Docs = [
+ #{
+ <<"_id">> => <<"_local/shard-sync-foo-bar">>,
+ <<"_rev">> => <<"0-1">>,
+ <<"seq">> => 1,
+ <<"target_uuid">> => <<"4f3eb3181db904d62176fb81626f84d0">>
+ },
+ #{
+ <<"_id">> => <<"_local/purge-mrview-foo-bar">>,
+ <<"_rev">> => <<"0-1">>,
+ <<"ddoc_id">> => <<"_design/ddoc1">>,
+ <<"purge_seq">> => 0,
+ <<"signature">> => [1, 2, 3],
+ <<"type">> => <<"mrview">>,
+ <<"updated_on">> => 1676995502
+ },
+ #{
+ <<"_id">> => <<"_local/1-foo-bar">>
+ },
+ #{
+ <<"_id">> => <<"_local/2-foo-bar">>
+ }
+ ],
+ Body = #{<<"docs">> => Docs},
+ {Code, _} = req(post, Path ++ "/_bulk_docs", Body),
+ ?assertEqual(201, Code),
+ {Code1, Res} = req(get, Path ++ "/_local_docs?include_system=true"),
+ ?assertEqual(200, Code1),
+ ?assertEqual(
+ #{
+ <<"total_rows">> => null,
+ <<"offset">> => null,
+ <<"rows">> => [
+ #{
+ <<"id">> => <<"_local/1-foo-bar">>,
+ <<"key">> => <<"_local/1-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-1">>}
+ },
+ #{
+ <<"id">> => <<"_local/2-foo-bar">>,
+ <<"key">> => <<"_local/2-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-1">>}
+ },
+ #{
+ <<"id">> => <<"_local/purge-mrview-foo-bar">>,
+ <<"key">> => <<"_local/purge-mrview-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-2">>}
+ },
+ #{
+ <<"id">> => <<"_local/shard-sync-foo-bar">>,
+ <<"key">> => <<"_local/shard-sync-foo-bar">>,
+ <<"value">> => #{<<"rev">> => <<"0-2">>}
+ }
+ ]
+ },
+ Res
+ ).
+
+create_db(Top, Db) ->
+ case req(put, Top ++ Db) of
+ {201, #{}} ->
+ ok;
+ Error ->
+ error({failed_to_create_test_db, Db, Error})
+ end.
+
+delete_db(Top, Db) ->
+ case req(delete, Top ++ Db) of
+ {200, #{}} ->
+ ok;
+ Error ->
+ error({failed_to_delete_test_db, Db, Error})
+ end.
+
+req(Method, Url) ->
+ Headers = [?JSON, ?AUTH],
+ {ok, Code, _, Res} = test_request:request(Method, Url, Headers),
+ {Code, jiffy:decode(Res, [return_maps])}.
+
+req(Method, Url, #{} = Body) ->
+ req(Method, Url, jiffy:encode(Body));
+req(Method, Url, Body) ->
+ Headers = [?JSON, ?AUTH],
+ {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body),
+ {Code, jiffy:decode(Res, [return_maps])}.
[couchdb] 01/04: hide shard-sync and purge documents from `_local_docs`
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7a2705ac67fe4ac6072224f87174fa41da7e274e
Author: Christian Kruse <cj...@defunct.ch>
AuthorDate: Sun Feb 19 09:41:45 2023 +0100
hide shard-sync and purge documents from `_local_docs`
It hides the shard-sync and purge documents from the response unless a `?include_system=true` is given.
Fixes #3930
---
src/chttpd/src/chttpd_db.erl | 9 ++++-
src/couch_mrview/src/couch_mrview.erl | 69 ++++++++++++++++++++---------------
2 files changed, 47 insertions(+), 31 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 748b356fd..db506e8e2 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -938,10 +938,11 @@ all_docs_view(Req, Db, Keys, OP) ->
Args1 = Args0#mrargs{view_type = map},
Args2 = fabric_util:validate_all_docs_args(Db, Args1),
Args3 = set_namespace(OP, Args2),
+ Args4 = set_include_sysdocs(OP, Req, Args3),
Options = [{user_ctx, Req#httpd.user_ctx}],
Max = chttpd:chunked_response_buffer_size(),
VAcc = #vacc{db = Db, req = Req, threshold = Max},
- {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args3),
+ {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args4),
{ok, Resp#vacc.resp}.
view_cb({row, Row} = Msg, Acc) ->
@@ -2286,6 +2287,12 @@ set_namespace(<<"_design_docs">>, Args) ->
set_namespace(NS, #mrargs{} = Args) ->
couch_mrview_util:set_extra(Args, namespace, NS).
+set_include_sysdocs(<<"_local_docs">>, Req, Args) ->
+ Val = chttpd:qs_value(Req, "include_system", "false") == "true",
+ couch_mrview_util:set_extra(Args, include_system, Val);
+set_include_sysdocs(_OP, _Req, Args) ->
+ Args.
+
%% /db/_bulk_get stuff
bulk_get_is_multipart(#httpd{mochi_req = MochiReq}) ->
diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl
index d8640c903..3616af7df 100644
--- a/src/couch_mrview/src/couch_mrview.erl
+++ b/src/couch_mrview/src/couch_mrview.erl
@@ -531,36 +531,45 @@ map_fold({{Key, Id}, Val}, _Offset, Acc) ->
user_acc = UAcc1,
last_go = Go
}};
-map_fold(#doc{id = <<"_local/", _/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
- #mracc{
- limit = Limit,
- callback = Callback,
- user_acc = UAcc0,
- args = Args
- } = Acc,
- #doc{
- id = DocId,
- revs = {Pos, [RevId | _]}
- } = Doc,
- Rev = {Pos, RevId},
- Row =
- [
- {id, DocId},
- {key, DocId},
- {value, {[{rev, couch_doc:rev_to_str(Rev)}]}}
- ] ++
- if
- not Args#mrargs.include_docs -> [];
- true -> [{doc, couch_doc:to_json_obj(Doc, Args#mrargs.doc_options)}]
- end,
- {Go, UAcc1} = Callback({row, Row}, UAcc0),
- {Go, Acc#mracc{
- limit = Limit - 1,
- reduce_fun = undefined,
- doc_info = undefined,
- user_acc = UAcc1,
- last_go = Go
- }}.
+map_fold(#doc{id = <<"_local/", IdTail/binary>>} = Doc, _Offset, #mracc{} = Acc) ->
+ IncludeSys = couch_util:get_value(include_system, Acc#mracc.args#mrargs.extra),
+
+ case {IncludeSys, IdTail} of
+ {false, <<"purge-", _/binary>>} ->
+ {ok, Acc};
+ {false, <<"shard-sync", _/binary>>} ->
+ {ok, Acc};
+ _ ->
+ #mracc{
+ limit = Limit,
+ callback = Callback,
+ user_acc = UAcc0,
+ args = Args
+ } = Acc,
+ #doc{
+ id = DocId,
+ revs = {Pos, [RevId | _]}
+ } = Doc,
+ Rev = {Pos, RevId},
+ Row =
+ [
+ {id, DocId},
+ {key, DocId},
+ {value, {[{rev, couch_doc:rev_to_str(Rev)}]}}
+ ] ++
+ if
+ not Args#mrargs.include_docs -> [];
+ true -> [{doc, couch_doc:to_json_obj(Doc, Args#mrargs.doc_options)}]
+ end,
+ {Go, UAcc1} = Callback({row, Row}, UAcc0),
+ {Go, Acc#mracc{
+ limit = Limit - 1,
+ reduce_fun = undefined,
+ doc_info = undefined,
+ user_acc = UAcc1,
+ last_go = Go
+ }}
+ end.
red_fold(Db, {NthRed, _Lang, View} = RedView, Args, Callback, UAcc) ->
Finalizer =
[couchdb] 04/04: add a notice that purge documents are only visible with `include_system=true`
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ac600d1bb9c9b5335abecc954abf9dd496a2f846
Author: Christian Kruse <cj...@defunct.ch>
AuthorDate: Fri Feb 24 22:11:05 2023 +0100
add a notice that purge documents are only visible with `include_system=true`
---
src/docs/src/cluster/purging.rst | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/docs/src/cluster/purging.rst b/src/docs/src/cluster/purging.rst
index 2d8f00ed5..cc9087d89 100644
--- a/src/docs/src/cluster/purging.rst
+++ b/src/docs/src/cluster/purging.rst
@@ -66,8 +66,10 @@ Local Purge Checkpoint Documents
Indexes and internal replications of the database with purges create and
periodically update local checkpoint purge documents:
``_local/purge-{type}-{hash}``. These documents report the last ``purge_seq``
-processed by them and the timestamp of the last processing. An example of a
-local checkpoint purge document:
+processed by them and the timestamp of the last processing. These documents
+are only visible in ``_local_docs`` when you add a ``include_system=true``
+parameter, so e.g. ``/test-db/_local_docs?include_system=true``. An example
+of a local checkpoint purge document:
.. code-block:: json