You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2026/09/29 18:56:58 UTC
(couchdb) branch nouveau-purge-fix created (now 38b3db862)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at 38b3db862 remove unused field
This branch includes the following new commits:
new de9168a3e Preserve view purge sequences when compacting
new c89fcabbc Implement _index_info endpoint
new f3be3a0e8 if purged doc is recreated, just add it to the exclusion list
new a94757d91 remove unused Proc
new 38b3db862 remove unused field
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb) 04/05: remove unused Proc
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a94757d910c111fed384e89a9879eead797d951e
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Sep 29 19:50:44 2026 +0100
remove unused Proc
---
src/nouveau/src/nouveau_index_updater.erl | 74 ++++++++++++++-----------------
1 file changed, 34 insertions(+), 40 deletions(-)
diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl
index 290ce9e7f..32bf6ab0e 100644
--- a/src/nouveau/src/nouveau_index_updater.erl
+++ b/src/nouveau/src/nouveau_index_updater.erl
@@ -252,47 +252,41 @@ index_definition(#index{} = Index, InitialPurgeSeq) ->
}.
purge_index(Db, Index, #purge_acc{} = PurgeAcc0) ->
- Proc = get_os_process(Index#index.def_lang),
- try
- true = proc_prompt(Proc, [<<"add_fun">>, Index#index.def, <<"nouveau">>]),
- FoldFun = fun({PurgeSeq, _UUID, Id, _Revs}, #purge_acc{} = PurgeAcc1) ->
- PurgeAcc3 =
- case couch_db:get_full_doc_info(Db, Id) of
- not_found ->
- Item = nouveau_api:make_purge(
- Id, PurgeAcc1#purge_acc.index_purge_seq, PurgeSeq
- ),
- PurgeAcc2 = PurgeAcc1#purge_acc{batch = [Item | PurgeAcc1#purge_acc.batch]},
- PurgeAcc2#purge_acc{index_purge_seq = PurgeSeq};
- FDI ->
- DI = couch_doc:to_doc_info(FDI),
- #doc_info{id = Id, revs = [#rev_info{rev = Rev} | _]} = DI,
- case lists:member({Id, Rev}, PurgeAcc1#purge_acc.exclude_list) of
- true ->
- PurgeAcc1;
- false ->
- PurgeAcc1#purge_acc{
- exclude_list = [{Id, Rev} | PurgeAcc1#purge_acc.exclude_list]
- }
- end
- end,
- update_task(1),
- maybe_flush_batch(PurgeAcc3)
- end,
+ FoldFun = fun({PurgeSeq, _UUID, Id, _Revs}, #purge_acc{} = PurgeAcc1) ->
+ PurgeAcc3 =
+ case couch_db:get_full_doc_info(Db, Id) of
+ not_found ->
+ Item = nouveau_api:make_purge(
+ Id, PurgeAcc1#purge_acc.index_purge_seq, PurgeSeq
+ ),
+ PurgeAcc2 = PurgeAcc1#purge_acc{batch = [Item | PurgeAcc1#purge_acc.batch]},
+ PurgeAcc2#purge_acc{index_purge_seq = PurgeSeq};
+ FDI ->
+ DI = couch_doc:to_doc_info(FDI),
+ #doc_info{id = Id, revs = [#rev_info{rev = Rev} | _]} = DI,
+ case lists:member({Id, Rev}, PurgeAcc1#purge_acc.exclude_list) of
+ true ->
+ PurgeAcc1;
+ false ->
+ PurgeAcc1#purge_acc{
+ exclude_list = [{Id, Rev} | PurgeAcc1#purge_acc.exclude_list]
+ }
+ end
+ end,
+ update_task(1),
+ maybe_flush_batch(PurgeAcc3)
+ end,
- {ok, #purge_acc{} = PurgeAcc3} = couch_db:fold_purge_infos(
- Db, PurgeAcc0#purge_acc.index_purge_seq, FoldFun, PurgeAcc0, []
- ),
- {ok, PurgeAcc4} = flush_batch(PurgeAcc3),
- DbPurgeSeq = couch_db:get_purge_seq(Db),
- ok = nouveau_api:set_purge_seq(
- Index, PurgeAcc4#purge_acc.index_purge_seq, DbPurgeSeq
- ),
- update_local_doc(Db, Index, DbPurgeSeq),
- {ok, PurgeAcc4}
- after
- ret_os_process(Proc)
- end.
+ {ok, #purge_acc{} = PurgeAcc3} = couch_db:fold_purge_infos(
+ Db, PurgeAcc0#purge_acc.index_purge_seq, FoldFun, PurgeAcc0, []
+ ),
+ {ok, PurgeAcc4} = flush_batch(PurgeAcc3),
+ DbPurgeSeq = couch_db:get_purge_seq(Db),
+ ok = nouveau_api:set_purge_seq(
+ Index, PurgeAcc4#purge_acc.index_purge_seq, DbPurgeSeq
+ ),
+ update_local_doc(Db, Index, DbPurgeSeq),
+ {ok, PurgeAcc4}.
update_task(NumChanges) ->
[Changes, Total] = couch_task_status:get([changes_done, total_changes]),
(couchdb) 01/05: Preserve view purge sequences when compacting
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit de9168a3e92d2ae4d63ced697be9884c582d4f1a
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Sep 24 02:45:58 2026 -0400
Preserve view purge sequences when compacting
Previously, the view's new state was initialized with the purge sequence of the
db. But the view might not have yet processed all the purges up until that db
purge sequence. That lead to us possibly skipping over purges in the view (the
ones that occurred right before the compaction started). To avoid that,
preserve the view's old purge sequence and carry that over to the new state.
After compaction swaps it can continue processing those purges where it left
off.
Fix: https://github.com/apache/couchdb/issues/6128
---
src/couch_mrview/src/couch_mrview_compactor.erl | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl
index 4686d77eb..fa8ce71fe 100644
--- a/src/couch_mrview/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview/src/couch_mrview_compactor.erl
@@ -40,6 +40,7 @@ compact(State) ->
idx_name = IdxName,
sig = Sig,
update_seq = Seq,
+ purge_seq = PurgeSeq,
id_btree = IdBtree,
views = Views
} = State,
@@ -126,10 +127,16 @@ compact(State) ->
),
unlink(EmptyState#mrst.fd),
+ % EmptyState from reset_index/3 has the purge_seq set to db's purge
+ % sequence. However the B-trees we just copied are from the old state, and
+ % we should also carry over the old purge_seq of this view, so it can
+ % continue processsing where it left off right before the compaction
+ % started.
{ok, EmptyState#mrst{
id_btree = NewIdBtree,
views = NewViews,
- update_seq = Seq
+ update_seq = Seq,
+ purge_seq = PurgeSeq
}}.
recompact(State) ->
(couchdb) 02/05: Implement _index_info endpoint
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c89fcabbca9b7a41e63c0599df4f6ab758d03f05
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Tue Sep 15 12:11:37 2026 -0400
Implement _index_info endpoint
Implement a new `/{db}/_index_info` endpoint to return the status of all the
indexes in the database in a single endpoint. It matches the general pattern of
`_dbs_info` where we return the db info for all databases in a cluster. Just
like with _db_infos, where it doesn't return anything new just a better and
more optimized summary of all dbs, the new endpoint returns all the index
status in a single endpoint. User could, for example, fetch all the design
docs, parse them, and with an if/else check call a per-index specific _info
endpoint such as /{db}/_design/{ddoc}/_info,
/{db}/_design/{ddoc}/_nouveau_info/{index} etc. to get all this information,
but that's not very ergonomic and it's slower.
One of the main uses of this endpoint is to find out when all the indexes in
the db have been built. Each index returns an `updates_pending` object with the
pending bounds across all copies of each range. With a minimum (the best copy),
maximum (the worst copy) and a count of how many live copies returned a valid
result. This information is more accurate than what we might get with the view
group _info calls. It turns out that for view groups _info we broke the
minimum/preferred/total `updates_pending` stats some point.
That happened when we switched the collection to work as a ring (the
COUCHDB-3053 dedup fix), so then only the first responding copy of each range
is aggregated, total = minimum and preferred only counts the responding shards.
The new endpoint on the other hand, waits and aggregates stats from all copies
of the index.
To fix the COUCHDB-3053 dedup issue update the indivdual indexes _info
endpoints to return the same correct `updates_pending` structure for all
indexes. To avoid changing the API response this PR only adds fields and
doesn't remove or change existing ones.
The endpoint is driven by the list of design docs so it borrows the
start/end/limit traversal parameters from _design_docs. To improve traversal
instead of `offset` the endpoint returns a `bookmark` result. User may use the
bookmark as the `startkey` to paginate results.
Just like the `_dbs_info` endpoint, add a max limit parameter configured as
`[chttpd] max_ddoc_number_for_index_info_req`. If a request selects more design
docs than that then a 400 `too_many_design_docs` is returned.
---
rel/overlay/etc/default.ini | 3 +
src/chttpd/src/chttpd_db.erl | 183 +++++++++++++
src/chttpd/src/chttpd_httpd_handlers.erl | 1 +
src/docs/src/api/database/index.rst | 1 +
src/docs/src/api/database/index_info.rst | 457 +++++++++++++++++++++++++++++++
src/docs/src/api/ddoc/common.rst | 29 ++
src/docs/src/api/ddoc/nouveau.rst | 19 +-
src/docs/src/api/ddoc/search.rst | 18 +-
src/dreyfus/src/dreyfus_fabric_info.erl | 178 ++++++++----
src/dreyfus/src/dreyfus_index.erl | 33 +++
src/dreyfus/src/dreyfus_rpc.erl | 18 +-
src/fabric/src/fabric.erl | 57 +++-
src/fabric/src/fabric_group_info.erl | 187 ++++++++-----
src/fabric/src/fabric_index_info.erl | 436 +++++++++++++++++++++++++++++
src/fabric/src/fabric_rpc.erl | 126 ++++++++-
src/fabric/src/fabric_util.erl | 70 +++++
src/fabric/test/eunit/fabric_tests.erl | 24 +-
src/nouveau/src/nouveau_fabric_info.erl | 176 ++++++++----
src/nouveau/src/nouveau_rpc.erl | 12 +-
src/nouveau/src/nouveau_util.erl | 31 +++
test/elixir/lib/couch/dbtest.ex | 7 +
test/elixir/test/config/nouveau.elixir | 3 +-
test/elixir/test/config/search.elixir | 2 +
test/elixir/test/config/suite.elixir | 7 +
test/elixir/test/design_docs_test.exs | 40 +++
test/elixir/test/index_info_test.exs | 391 ++++++++++++++++++++++++++
test/elixir/test/nouveau_test.exs | 47 ++++
test/elixir/test/search_test.exs | 63 +++++
28 files changed, 2405 insertions(+), 214 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 57263c8a9..0840adac1 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -256,6 +256,9 @@ bind_address = 127.0.0.1
; _dbs_info in a request
;max_db_number_for_dbs_info_req = 100
+; Limit maximum number of design docs for the _index_info request.
+;max_ddoc_number_for_index_info_req = 1000
+
; set to true to delay the start of a response until the end has been calculated
;buffer_response = false
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 20f41e04a..9bafed01c 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -26,6 +26,7 @@
handle_changes_req/2,
update_doc_result_to_json/1, update_doc_result_to_json/2,
handle_design_info_req/3,
+ handle_index_info_req/2,
handle_view_cleanup_req/2,
update_doc/4,
http_code_from_status/1,
@@ -81,6 +82,8 @@
T == <<"_explain">>)
).
+-define(MAX_DDOC_NUM_FOR_INDEX_INFO, 1000).
+
% Database request handlers
handle_request(#httpd{path_parts = [DbName | RestParts], method = Method} = Req) ->
case {Method, RestParts} of
@@ -464,6 +467,101 @@ handle_design_info_req(#httpd{method = 'GET'} = Req, Db, #doc{} = DDoc) ->
handle_design_info_req(Req, _Db, _DDoc) ->
send_method_not_allowed(Req, "GET").
+% Get index status info for all design docs in the database. Takes
+% `type=view|search|nouveau` as a filtering parameter as well as all the
+% _design_doc paging parameters: startkey, endkey, limit, etc.
+handle_index_info_req(#httpd{method = 'GET', path_parts = [_, _]} = Req, Db) ->
+ Args = couch_mrview_http:parse_params(Req, undefined),
+ index_info_req(Req, Db, Args);
+handle_index_info_req(#httpd{method = 'POST', path_parts = [_, _]} = Req, Db) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ Keys = couch_mrview_util:get_view_keys(chttpd:json_body_obj(Req)),
+ Args = couch_mrview_http:parse_body_and_query(Req, Keys),
+ index_info_req(Req, Db, Args);
+handle_index_info_req(#httpd{path_parts = [_, _]} = Req, _Db) ->
+ send_method_not_allowed(Req, "GET,HEAD,POST");
+handle_index_info_req(Req, _Db) ->
+ chttpd:send_error(Req, not_found).
+
+index_info_req(Req, Db, #mrargs{keys = Keys} = Args) ->
+ Types = parse_types(Req),
+ Default = ?MAX_DDOC_NUM_FOR_INDEX_INFO,
+ Max = config:get_integer("chttpd", "max_ddoc_number_for_index_info_req", Default),
+ case is_list(Keys) andalso length(Keys) > Max of
+ true -> throw({bad_request, too_many_keys});
+ false -> ok
+ end,
+ DbName = couch_db:name(Db),
+ case fabric:get_index_info(DbName, Types, Args, [{max_ddocs, Max}]) of
+ {ok, Meta, ByType} ->
+ send_json(Req, 200, {meta_to_json(Meta) ++ index_info_to_json(ByType)});
+ {error, too_many_design_docs} ->
+ throw({bad_request, too_many_design_docs});
+ {error, Reason} ->
+ throw(Reason)
+ end.
+
+% Since we're gathering index info based on design docs, total_rows counts
+% design docs. The bookmark is the doc ID of the design doc after the page so
+% it can be used as the startkey of the next page, or null when there is none,
+% like for a `keys` request, for example.
+meta_to_json(Meta) ->
+ [
+ {total_rows, couch_util:get_value(total, Meta, null)},
+ {bookmark, couch_util:get_value(bookmark, Meta, null)}
+ ].
+
+% Types for _index_info. Must be one or more of view, search, nouveau
+parse_types(Req) ->
+ TypeArgs = [V || {K, V} <- chttpd:qs(Req), K == "type"],
+ case TypeArgs of
+ [] ->
+ [view, search, nouveau];
+ [_ | _] ->
+ Tokens = lists:flatmap(fun(V) -> string:tokens(V, ",") end, TypeArgs),
+ case lists:usort(lists:map(fun parse_type/1, Tokens)) of
+ [] -> throw({query_parse_error, ~"`type` must not be empty"});
+ Types -> [T || T <- [view, search, nouveau], lists:member(T, Types)]
+ end
+ end.
+
+parse_type("view") ->
+ view;
+parse_type("search") ->
+ search;
+parse_type("nouveau") ->
+ nouveau;
+parse_type(Other) ->
+ Msg = io_lib:format("Invalid type: ~s. Must be view, search or nouveau", [Other]),
+ throw({query_parse_error, ?l2b(Msg)}).
+
+% Return one list per index type. Each entry has the ddoc and either a name or
+% views list (view indexes are actually groups of views building together). If
+% index info is successfully returned there is an "ok": true entry. If it
+% couldn't be returned (all copies are not responding or the index subsystem is
+% disabled) "ok":false is returned for that index along with an "error" and
+% "reason" field.
+index_info_to_json(ByType) ->
+ [{idx_type(Type), idx_entries(Type, ByType)} || Type <- [view, search, nouveau]].
+
+idx_entries(Type, ByType) ->
+ [idx_info(E) || E <- couch_util:get_value(Type, ByType, [])].
+
+idx_type(view) -> view_indexes;
+idx_type(search) -> search_indexes;
+idx_type(nouveau) -> nouveau_indexes.
+
+% view and search infos are {[...]} ejson objects, nouveau's is a map. Both
+% encode as they are.
+idx_info({Ident, {ok, Info}}) ->
+ {Ident ++ [{ok, true}, {info, Info}]};
+idx_info({Ident, {error, Error}}) ->
+ {Ident ++ [{ok, false}] ++ errobj(Error)}.
+
+errobj(Error) ->
+ {_Code, ErrorStr, ReasonStr} = chttpd:error_info(Error),
+ [{error, ErrorStr}, {reason, ReasonStr}].
+
create_db_req(#httpd{} = Req, DbName) ->
couch_httpd:verify_is_server_admin(Req),
ShardsOpt = parse_shards_opt(Req),
@@ -2660,6 +2758,91 @@ monitor_attachments_test_() ->
?_assertEqual([], monitor_attachments(Atts))
end}.
+index_info_to_json_test_() ->
+ % {error, Reason} become {"ok": false, "error":.., "reason":..}
+ % {ok, Info} entries will have an "info" object returned
+ Oops = {service_unavailable, ~"Search is not available"},
+ Pending = [
+ {minimum, 0},
+ {preferred, 0},
+ {total, 2},
+ {maximum, 2},
+ {copies, 6},
+ {copies_expected, 6}
+ ],
+ D = ~"_design/d",
+ View = {[{updates_pending, {Pending}}, {update_seq, 1}]},
+ Search = {[{updates_pending, {Pending}}, {doc_count, 2}]},
+ Nouveau = #{num_docs => 3, updates_pending => maps:from_list(Pending)},
+ Res = [
+ {view, [
+ {[{ddoc, D}, {views, [~"v"]}], {ok, View}},
+ {[{ddoc, ~"_design/e"}, {views, []}], {error, not_found}}
+ ]},
+ {search, [
+ {[{ddoc, D}, {name, ~"i"}], {ok, Search}},
+ {[{ddoc, D}, {name, ~"j"}], {error, Oops}}
+ ]},
+ {nouveau, [{[{ddoc, D}, {name, ~"n"}], {ok, Nouveau}}]}
+ ],
+ Expected = [
+ {view_indexes, [
+ {[{ddoc, D}, {views, [~"v"]}, {ok, true}, {info, View}]},
+ {[
+ {ddoc, ~"_design/e"},
+ {views, []},
+ {ok, false},
+ {error, ~"not_found"},
+ {reason, ~"missing"}
+ ]}
+ ]},
+ {search_indexes, [
+ {[{ddoc, D}, {name, ~"i"}, {ok, true}, {info, Search}]},
+ {[
+ {ddoc, D},
+ {name, ~"j"},
+ {ok, false},
+ {error, ~"service unavailable"},
+ {reason, ~"Search is not available"}
+ ]}
+ ]},
+ {nouveau_indexes, [
+ {[{ddoc, D}, {name, ~"n"}, {ok, true}, {info, Nouveau}]}
+ ]}
+ ],
+ ?_assertEqual(Expected, index_info_to_json(Res)).
+
+meta_to_json_test_() ->
+ [
+ ?_assertEqual(
+ [{total_rows, 3}, {bookmark, ~"_design/d"}],
+ meta_to_json([{bookmark, ~"_design/d"}, {total, 3}, {offset, 1}])
+ ),
+ ?_assertEqual(
+ [{total_rows, 3}, {bookmark, null}], meta_to_json([{bookmark, null}, {total, 3}])
+ ),
+ ?_assertEqual([{total_rows, null}, {bookmark, null}], meta_to_json([]))
+ ].
+
+index_info_type_filter_test_() ->
+ ByType = [{search, [{[{ddoc, ~"_design/d"}, {name, ~"i"}], {error, not_found}}]}],
+ ?_assertEqual(
+ [
+ {view_indexes, []},
+ {search_indexes, [
+ {[
+ {ddoc, ~"_design/d"},
+ {name, ~"i"},
+ {ok, false},
+ {error, ~"not_found"},
+ {reason, ~"missing"}
+ ]}
+ ]},
+ {nouveau_indexes, []}
+ ],
+ index_info_to_json(ByType)
+ ).
+
parse_partitioned_opt_test_() ->
{
foreach,
diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl
index 3e499b72d..4251cbfbc 100644
--- a/src/chttpd/src/chttpd_httpd_handlers.erl
+++ b/src/chttpd/src/chttpd_httpd_handlers.erl
@@ -30,6 +30,7 @@ url_handler(<<"_up">>) -> fun chttpd_misc:handle_up_req/1;
url_handler(_) -> no_match.
db_handler(<<"_view_cleanup">>) -> fun chttpd_db:handle_view_cleanup_req/2;
+db_handler(<<"_index_info">>) -> fun chttpd_db:handle_index_info_req/2;
db_handler(<<"_compact">>) -> fun chttpd_db:handle_compact_req/2;
db_handler(<<"_design">>) -> fun chttpd_db:handle_design_req/2;
db_handler(<<"_partition">>) -> fun chttpd_db:handle_partition_req/2;
diff --git a/src/docs/src/api/database/index.rst b/src/docs/src/api/database/index.rst
index 914c0d6ff..8b5f23c1c 100644
--- a/src/docs/src/api/database/index.rst
+++ b/src/docs/src/api/database/index.rst
@@ -44,5 +44,6 @@ Where ``{db}`` is the name of any database.
changes
compact
cleanup
+ index_info
security
misc
diff --git a/src/docs/src/api/database/index_info.rst b/src/docs/src/api/database/index_info.rst
new file mode 100644
index 000000000..9d17abc04
--- /dev/null
+++ b/src/docs/src/api/database/index_info.rst
@@ -0,0 +1,457 @@
+.. Licensed under the Apache License, Version 2.0 (the "License"); you may not
+.. use this file except in compliance with the License. You may obtain a copy of
+.. the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+.. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+.. License for the specific language governing permissions and limitations under
+.. the License.
+
+.. _api/db/index_info:
+
+=====================
+``/{db}/_index_info``
+=====================
+
+.. versionadded:: 3.6
+
+.. http:get:: /{db}/_index_info
+ :synopsis: Get the status of all the indexes in the database
+
+ Return index status grouped by type: ``view_indexes``, ``search_indexes``
+ and ``nouveau_indexes``. Each entry identifies the index by its design
+ document (``ddoc``) and, for search and nouveau indexes, its ``name``, like
+ :get:`/{db}/_index`. In the ``info`` field return the same object returned
+ as we get from the individual design doc ``_info`` endpoints:
+ :get:`/{db}/_design/{ddoc}/_info` or the ``search_index``,
+ :get:`/{db}/_design/{ddoc}/_search_info/{index}` and
+ :get:`/{db}/_design/{ddoc}/_nouveau_info/{index}`. See
+ :ref:`api/db/index_info/structure`.
+
+ This endpoint may be used when one needs to know whether a database's
+ indexes are fully built, for example before switching traffic to a replica.
+
+ :param db: Database name
+ :query string type: Filter by type. Can be one of these values:
+ ``view``, ``search`` and ``nouveau``, or several of them comma separated.
+ *Optional*, default: all types.
+ :query json startkey: Return indexes of design documents with a doc ID
+ greater than or equal to this key. Same as for
+ :get:`/{db}/_design_docs`. To get the next page, pass the ``bookmark``
+ of the previous response as the ``startkey``. see `Paging`_. *Optional*
+ :query json start_key: Alias for ``startkey``. *Optional*
+ :query json endkey: Return indexes of design documents with a doc ID less
+ than or equal to this key. *Optional*
+ :query json end_key: Alias for ``endkey``. *Optional*
+ :query boolean inclusive_end: Whether the design document with the
+ ``endkey`` doc ID is included. Default is ``true``. *Optional*
+ :query boolean descending: Traverse design documents in descending order,
+ as :get:`/{db}/_design_docs` does. Default is ``false``. *Optional*
+ :query number limit: Return indexes of at most this many design documents.
+ Design documents without indexes count towards the limit too. A request
+ may traverse at most ``max_ddoc_number_for_index_info_req`` design
+ documents, see `Paging`_. *Optional*
+ :query number skip: How many design documents to skip before starting to
+ return results. For paging it's best to use
+ ``startkey`` with the ``bookmark`` of the previous
+ response, see `Paging`_. *Optional*
+ :query json keys: Return indexes for the design documents with these doc
+ IDs only. Should be a JSON array of strings. :post:`/{db}/_index_info`
+ can take the same ID list in the request body. *Optional*
+ :<header Accept: - :mimetype:`application/json`
+ - :mimetype:`text/plain`
+ :>header Content-Type: - :mimetype:`application/json`
+ - :mimetype:`text/plain; charset=utf-8`
+ :>json number total_rows: Total number of design docs. Same as returned in
+ :get:`/{db}/_design_docs`. Can be used for paging
+ to see how many design document to expect
+ overall. Note: a design document may have more
+ than on index defined.
+ :>json string bookmark: The doc ID of the first design document after the
+ ones from the result. Use it as the the ``startkey`` of the next page
+ to paginate. It will be ``null`` when there are no more design
+ documents, or when ``keys`` are used. See `Paging`_.
+ :>json array view_indexes: One entry per design doc for views.
+ See :ref:`api/db/index_info/structure`.
+ :>json array search_indexes: One entry per search index.
+ :>json array nouveau_indexes: One entry per nouveau index.
+ :code 200: Request was successful
+ :code 400: Invalid db name or parameter. Or, total design document selected
+ is over the ``max_ddoc_number_for_index_info_req`` limit.
+ :code 401: Unauthorized request
+ :code 403: Insufficient permissions / :ref:`Too many requests with invalid credentials<error/403>`
+ :code 404: Database doesn't exist
+
+ **Request**:
+
+ .. code-block:: http
+
+ GET /recipes/_index_info HTTP/1.1
+ Accept: application/json
+ Host: localhost:5984
+
+ **Response**:
+
+ .. code-block:: http
+
+ HTTP/1.1 200 OK
+ Cache-Control: must-revalidate
+ Content-Type: application/json
+ Date: Mon, 01 Sep 2025 15:42:11 GMT
+ Server: CouchDB (Erlang/OTP)
+
+ {
+ "total_rows": 2,
+ "bookmark": null,
+ "view_indexes": [
+ {
+ "ddoc": "_design/8c2a4caf8ea1b581ac43a062fd43a876dee1382d",
+ "views": [
+ "by-rating"
+ ],
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 54,
+ "preferred": 54,
+ "total": 324,
+ "maximum": 54,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "collator_versions": [
+ "153.136"
+ ],
+ "compact_running": false,
+ "language": "query",
+ "purge_seq": 0,
+ "signature": "b77547252cb8b19ff12831973b576c0f",
+ "sizes": {
+ "active": 0,
+ "external": 10,
+ "file": 102
+ },
+ "update_seq": 0,
+ "updater_running": false,
+ "waiting_clients": 0,
+ "waiting_commit": false
+ }
+ },
+ {
+ "ddoc": "_design/cookbook",
+ "views": [
+ "by_ingredient",
+ "by_title"
+ ],
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "collator_versions": [
+ "153.136"
+ ],
+ "compact_running": false,
+ "language": "javascript",
+ "purge_seq": 0,
+ "signature": "a6d97b0199e54a1eb56e4becb1322587",
+ "sizes": {
+ "active": 1825,
+ "external": 1355,
+ "file": 16750
+ },
+ "update_seq": 54,
+ "updater_running": false,
+ "waiting_clients": 0,
+ "waiting_commit": false
+ }
+ }
+ ],
+ "search_indexes": [
+ {
+ "ddoc": "_design/cookbook",
+ "name": "ingredients",
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "committed_seq": 54,
+ "disk_size": 3960,
+ "doc_count": 50,
+ "doc_del_count": 0,
+ "pending_seq": 54,
+ "signature": "0b4ba635d5eb4fcbb2f6c9c2247460ec"
+ }
+ },
+ {
+ "ddoc": "_design/cookbook",
+ "name": "steps",
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 12,
+ "maximum": 12,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "committed_seq": 42,
+ "disk_size": 2210,
+ "doc_count": 38,
+ "doc_del_count": 0,
+ "pending_seq": 42,
+ "signature": "efc82e779a1f10ec90b2fbf0bc5246dd"
+ }
+ }
+ ],
+ "nouveau_indexes": [
+ {
+ "ddoc": "_design/cookbook",
+ "name": "ingredients",
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "disk_size": 6324,
+ "num_docs": 50,
+ "purge_seq": 0,
+ "signature": "ea87fe8f9517403691850f51d0a1ce3e5afaf89347204dd79430252a0591e503",
+ "update_seq": 54
+ }
+ }
+ ]
+ }
+
+ Here the ``cookbook`` design doc has two views, two search indexes and a
+ nouveau index. All are built on all 6 copies except the ``steps`` search
+ index, which lags 12 updates behind. The first index is a query (Mango)
+ which wasn't built yet, so all copies are behind.
+
+.. http:post:: /{db}/_index_info
+ :synopsis: Get the status of indexes from the selected design doc set.
+
+ Same as :get:`/{db}/_index_info`, with the parameters given in the body,
+ like :post:`/{db}/_design_docs`. A ``keys`` array selects the design docs
+ to report by ID, like :post:`/_dbs_info`.
+
+ :param db: Database name
+ :<header Accept: - :mimetype:`application/json`
+ - :mimetype:`text/plain`
+ :<header Content-Type: :mimetype:`application/json`
+ :<json array keys: Doc IDs of the design docs to report, for example
+ ``["_design/cookbook"]``. Other :get:`/{db}/_index_info`
+ parameters can provided in the body as well. *Optional*
+ :>header Content-Type: - :mimetype:`application/json`
+ - :mimetype:`text/plain; charset=utf-8`
+ :>json number total_rows: Number of design docs in the database. Same as
+ for :get:`/{db}/_index_info`
+ :>json string bookmark: Same as for :get:`/{db}/_index_info`. ``null``
+ for a ``keys`` request, which has no next page
+ :>json array view_indexes: Same as :get:`/{db}/_index_info`
+ :>json array search_indexes: Same as :get:`/{db}/_index_info`
+ :>json array nouveau_indexes: Same as :get:`/{db}/_index_info`
+ :code 200: Request was successful
+ :code 400: Invalid db name or parameter, or more keys than the
+ ``max_ddoc_number_for_index_info_req`` limit.
+ :code 401: Unauthorized request
+ :code 403: Insufficient permissions / :ref:`Too many requests with invalid credentials<error/403>`
+ :code 404: Database doesn't exist
+
+ **Request**:
+
+ .. code-block:: http
+
+ POST /recipes/_index_info?type=search HTTP/1.1
+ Accept: application/json
+ Content-Type: application/json
+ Host: localhost:5984
+
+ {
+ "keys": [
+ "_design/cookbook"
+ ]
+ }
+
+ **Response**:
+
+ .. code-block:: http
+
+ HTTP/1.1 200 OK
+ Cache-Control: must-revalidate
+ Content-Type: application/json
+ Date: Mon, 01 Sep 2025 15:42:11 GMT
+ Server: CouchDB (Erlang/OTP)
+
+ {
+ "total_rows": 2,
+ "bookmark": null,
+ "view_indexes": [],
+ "search_indexes": [
+ {
+ "ddoc": "_design/cookbook",
+ "name": "ingredients",
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "committed_seq": 54,
+ "disk_size": 3960,
+ "doc_count": 50,
+ "doc_del_count": 0,
+ "pending_seq": 54,
+ "signature": "0b4ba635d5eb4fcbb2f6c9c2247460ec"
+ }
+ },
+ {
+ "ddoc": "_design/cookbook",
+ "name": "steps",
+ "ok": true,
+ "info": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 12,
+ "maximum": 12,
+ "copies": 6,
+ "copies_expected": 6
+ },
+ "committed_seq": 42,
+ "disk_size": 2210,
+ "doc_count": 38,
+ "doc_del_count": 0,
+ "pending_seq": 42,
+ "signature": "efc82e779a1f10ec90b2fbf0bc5246dd"
+ }
+ }
+ ],
+ "nouveau_indexes": []
+ }
+
+ We filter to return only ``search`` type indexes.
+
+.. _api/db/index_info/structure:
+
+Index Information
+=================
+
+All three types of index lists are returned. Each one may be empty if not
+indexes are found or they are filtered based on the ``type`` parameter. Each
+element of the list will contain:
+
+* **ddoc** (*string*): The design doc ID.
+* **name** (*string*): Index name for search and nouveau indexes.
+* **views** (*array*): A list of map-reduce views for a view group
+* **ok** (*boolean*): ``true`` if we successfully fetched info about that index
+ and ``false`` if we couldn't (see `Errors`_ below).
+* **info** (*object*): Info object of the index. This is the same as the
+ returned by individual index ``_info`` endpoints:
+ :ref:`api/ddoc/view_index_info`,
+ :get:`/{db}/_design/{ddoc}/_search_info/{index}` and
+ :get:`/{db}/_design/{ddoc}/_nouveau_info/{index}`.
+
+Paging
+------
+
+The same traversal parameters as for :get:`/{db}/_design_docs` can be used.
+Paging is per design document, not per index. ``total_rows`` is the number of
+design documents in the database including design documents which may not have
+any indexes.
+
+To page, request ``limit=N`` and pass the ``bookmark`` of each response as the
+``startkey`` of the next request, until it is ``null``.
+
+.. code-block:: text
+
+ GET /recipes/_index_info?limit=100
+
+ {
+ "total_rows": 250,
+ "bookmark": "_design/d0100",
+ "view_indexes": [ ... ],
+ "search_indexes": [ ... ],
+ "nouveau_indexes": [ ... ]
+ }
+
+ GET /recipes/_index_info?limit=100&startkey="_design/d0100"
+
+A single request can process at most ``[chttpd]
+max_ddoc_number_for_index_info_req`` design docs. This is similar to
+``max_db_number_for_dbs_info_req`` for :post:`/_dbs_info`. If a request
+selects more than that, the request will fail with an error:
+
+.. code-block:: javascript
+
+ {
+ "error": "bad_request",
+ "reason": "too_many_design_docs"
+ }
+
+Pending updates
+---------------
+
+The ``info`` object of every entry with ``ok`` set to ``true`` has an ``updates_pending``
+object. Its fields are described in :ref:`api/ddoc/view_index_info`. In summary:
+
+* **minimum** will be ``0`` when at least one complete copy of the index exists
+ for every range.
+* **maximum** will be ``0`` when all shard copies are completely built.
+* **preferred** and **total** are the backlogs of the preferred shard copies and of
+ all shard copies added together.
+* **copies** how many shard copies returned results. This may be smaller than
+ **copies_expected** for example if some nodes are not reachable or in
+ maintenance mode
+
+Errors
+------
+
+If index info could not be obtained for some then the index entry will have
+``ok`` set to ``false``. And the index entry will have ``error`` and ``reason``
+fields instead of ``info``. This is the same as how :post:`/{db}/_bulk_docs`
+reports per document errors. For example if clouseau (search indexing system)
+is down:
+
+.. code-block:: javascript
+
+ {
+ "ddoc": "_design/cookbook",
+ "name": "ingredients",
+ "ok": false,
+ "error": "service unavailable",
+ "reason": "Search is not available"
+ }
+
+Invalid ``type`` values return a ``400 Bad Request``:
+
+.. code-block:: javascript
+
+ {
+ "error": "query_parse_error",
+ "reason": "Invalid type: foo. Must be view, search or nouveau"
+ }
diff --git a/src/docs/src/api/ddoc/common.rst b/src/docs/src/api/ddoc/common.rst
index 961af135b..bf40b3b87 100644
--- a/src/docs/src/api/ddoc/common.rst
+++ b/src/docs/src/api/ddoc/common.rst
@@ -182,6 +182,14 @@
{
"name": "recipe",
"view_index": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
"compact_running": false,
"language": "python",
"purge_seq": 0,
@@ -222,3 +230,24 @@ The response from :get:`/{db}/_design/{ddoc}/_info` contains
this design document
* **waiting_commit** (*boolean*): Indicates if there are outstanding commits
to the underlying database that need to processed
+* **updates_pending** (*object*): Information about the pending updates for
+ this index calculated over all available copies of all the shards.
+
+ * **minimum** (*number*): Backlog of the most up to date copy of each
+ range. ``0`` means at least one complete copy exists for every range.
+ * **preferred** (*number*): Backlog of the preferred shard copies, the ones
+ queries are routed to by default.
+ * **total** (*number*): Backlog over all copies added up.
+ * **maximum** (*number*): Backlog of the least up to date copy of each
+ range. ``0`` means the index is fully built on every copy.
+ * **copies** (*number*): The number of shard copies which reported an
+ answer. Copies which are unreachable or in maintenance mode are not
+ included. Compare with the ``copies_expected`` number.
+ * **copies_expected** (*number*): Expected number of shard responses. If
+ ``copies`` is smaller than this, the results may be partial and
+ ``maximum`` for instance may not represent the true backlog.
+
+ .. versionchanged:: 3.6
+ Information is now gathered from every shard copy instead of the first
+ responding copy of each range. ``maximum``, ``copies`` and
+ ``copies_expected`` were added and ``total`` now adds up all copies.
diff --git a/src/docs/src/api/ddoc/nouveau.rst b/src/docs/src/api/ddoc/nouveau.rst
index fa9935e97..09964a5ac 100644
--- a/src/docs/src/api/ddoc/nouveau.rst
+++ b/src/docs/src/api/ddoc/nouveau.rst
@@ -143,6 +143,23 @@
"search_index": {
"num_docs": 1000,
"update_seq": 5000,
- "disk_size": 1048576
+ "purge_seq": 0,
+ "disk_size": 1048576,
+ "signature": "0d6d58264aca53a894136931cc5fd2625258f2200b7a7b5606bb3fe3a4a31a94",
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ }
}
}
+
+.. versionadded:: 3.6
+ ``updates_pending`` reports how far the index lags behind the database
+ across all copies of every shard range, with the same fields as the
+ ``updates_pending`` object of :ref:`api/ddoc/view_index_info`. To obtain
+ it for every index of a database in one request see
+ :get:`/{db}/_index_info`.
diff --git a/src/docs/src/api/ddoc/search.rst b/src/docs/src/api/ddoc/search.rst
index 427a9b507..e5a2d61cb 100644
--- a/src/docs/src/api/ddoc/search.rst
+++ b/src/docs/src/api/ddoc/search.rst
@@ -165,10 +165,26 @@
{
"name": "_design/cookbook/ingredients",
"search_index": {
+ "updates_pending": {
+ "minimum": 0,
+ "preferred": 0,
+ "total": 0,
+ "maximum": 0,
+ "copies": 6,
+ "copies_expected": 6
+ },
"pending_seq": 7125496,
"doc_del_count": 129180,
"doc_count": 1066173,
"disk_size": 728305827,
- "committed_seq": 7125496
+ "committed_seq": 7125496,
+ "signature": "efc82e779a1f10ec90b2fbf0bc5246dd"
}
}
+
+.. versionadded:: 3.6
+ ``updates_pending`` reports how far the index lags behind the database
+ across all copies of every shard range, with the same fields as the
+ ``updates_pending`` object of :ref:`api/ddoc/view_index_info`. To obtain
+ it for every index of a database in one request see
+ :get:`/{db}/_index_info`.
diff --git a/src/dreyfus/src/dreyfus_fabric_info.erl b/src/dreyfus/src/dreyfus_fabric_info.erl
index 2925768b1..592fcc5c3 100644
--- a/src/dreyfus/src/dreyfus_fabric_info.erl
+++ b/src/dreyfus/src/dreyfus_fabric_info.erl
@@ -17,6 +17,8 @@
-include_lib("mem3/include/mem3.hrl").
-export([go/4]).
+% Used by fabric_index_info
+-export([build_final_response/3]).
go(DbName, DDocId, IndexName, InfoLevel) when is_binary(DDocId) ->
{ok, DDoc} = fabric:open_doc(DbName, <<"_design/", DDocId/binary>>, []),
@@ -26,60 +28,94 @@ go(DbName, DDoc, IndexName, InfoLevel) ->
DesignName = dreyfus_util:get_design_docid(DDoc),
dreyfus_util:maybe_deny_index(DbName, DesignName, IndexName),
Shards = mem3:shards(DbName),
+ Ushards = mem3:ushards(DbName),
+ USet = couch_util:set_from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
Workers = fabric_util:submit_jobs(Shards, dreyfus_rpc, InfoLevel, [DDoc, IndexName]),
RexiMon = fabric_util:create_monitors(Shards),
- Acc0 = {fabric_dict:init(Workers, nil), []},
- try
- fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0)
+ Acc0 = {fabric_dict:init(Workers, nil), [], USet, length(Workers), undefined},
+ try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of
+ {timeout, {WorkersDict, Resps, _, Expected, _}} ->
+ DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil),
+ fabric_util:log_timeout(DefunctWorkers, "search_info"),
+ fabric_util:cleanup(DefunctWorkers),
+ finish(Resps, USet, Expected, timeout);
+ Else ->
+ Else
after
rexi_monitor:stop(RexiMon)
end.
-handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, {Counters, Acc}) ->
- case fabric_util:remove_down_workers(Counters, NodeRef) of
- {ok, NewCounters} ->
- {ok, {NewCounters, Acc}};
- error ->
- {error, {nodedown, <<"progress not possible">>}}
- end;
-handle_message({rexi_EXIT, Reason}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, Reason}
- end;
-handle_message({ok, Info}, Worker, {Counters, Acc}) ->
- case fabric_dict:lookup_element(Worker, Counters) of
- undefined ->
- % already heard from someone else in this range
- {ok, {Counters, Acc}};
- nil ->
- C1 = fabric_dict:store(Worker, ok, Counters),
- C2 = fabric_view:remove_overlapping_shards(Worker, C1),
- case fabric_dict:any(nil, C2) of
- true ->
- {ok, {C2, [Info | Acc]}};
- false ->
- {stop, merge_results(lists:flatten([Info | Acc]))}
- end
- end;
-handle_message({error, Reason}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, Reason}
- end;
-handle_message({'EXIT', _}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, {nodedown, <<"progress not possible">>}}
+handle_message({ok, Info}, Worker, {Counters, Resps, USet, Expected, LastErr}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, [{Worker, Info} | Resps], USet, Expected, LastErr);
+handle_message(
+ {rexi_DOWN, _, {_, NodeRef}, _}, _Worker, {Counters, Resps, USet, Expected, LastErr}
+) ->
+ Counters1 = fabric_dict:filter(fun(#shard{node = N}, _) -> N =/= NodeRef end, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, LastErr);
+handle_message({rexi_EXIT, Reason}, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason);
+handle_message({error, Reason}, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason);
+handle_message(Reason, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason).
+
+maybe_stop(Counters, Resps, USet, Expected, FinalErr) ->
+ case fabric_dict:size(Counters) of
+ 0 ->
+ Error =
+ case FinalErr of
+ undefined -> {nodedown, <<"progress not possible">>};
+ _ -> FinalErr
+ end,
+ case finish(Resps, USet, Expected, Error) of
+ {ok, Result} -> {stop, Result};
+ {error, _} = E -> E
+ end;
+ _ ->
+ {ok, {Counters, Resps, USet, Expected, FinalErr}}
+ end.
+
+% Every range must have at least one result
+finish(Resps, USet, Expected, Error) ->
+ Ranges = [{B, E} || {#shard{range = [B, E]}, _} <- Resps],
+ case mem3_util:get_ring(Ranges) of
+ [] -> {error, Error};
+ _ -> {ok, build_final_response(USet, Expected, Resps)}
+ end.
+
+build_final_response(USet, Expected, Resps) ->
+ ByRange = lists:foldl(
+ fun({#shard{range = Range, name = Id, node = Node}, Info}, Acc) ->
+ orddict:append(Range, {sets:is_element({Id, Node}, USet), Info}, Acc)
+ end,
+ orddict:new(),
+ Resps
+ ),
+ RangeCopies = [Copies || {_Range, Copies} <- ByRange],
+ Reps = [representative(Copies) || Copies <- RangeCopies],
+ PerRange = [
+ [{Pref, couch_util:get_value(pending_updates, Info)} || {Pref, Info} <- Copies]
+ || Copies <- RangeCopies
+ ],
+ Merged = merge_results(lists:append(Reps)),
+ case fabric_util:aggregate_pending(PerRange, Expected) of
+ undefined -> Merged;
+ Pending -> [{updates_pending, {Pending}} | Merged]
+ end.
+
+% Try to pick the ushard copy as a stable representative so result doesn't flop
+% and forth with each request as much
+representative(Copies) ->
+ case [Info || {true, Info} <- Copies] of
+ [Info | _] ->
+ Info;
+ [] ->
+ [{_Pref, Info} | _] = Copies,
+ Info
end.
merge_results(Info) ->
@@ -108,3 +144,49 @@ merge_results(Info) ->
[],
Dict
).
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+shard([B, E] = Range, Node) ->
+ Name = integer_to_list(B) ++ "-" ++ integer_to_list(E),
+ #shard{name = list_to_binary(Name), node = Node, range = Range}.
+
+info(Docs, Pending) ->
+ [{doc_count, Docs}, {pending_updates, Pending}, {signature, ~"s"}].
+
+final_resp_test() ->
+ R1 = [0, 10],
+ R2 = [11, 20],
+ R1N1 = shard(R1, n1),
+ R2N2 = shard(R2, n2),
+ USet = couch_util:set_from_list([
+ {R1N1#shard.name, n1}, {R2N2#shard.name, n2}
+ ]),
+ R1N2 = shard(R1, n2),
+ R2N3 = shard(R2, n3),
+ Resps = [
+ {R1N1, info(100, 0)},
+ {R1N2, info(94, 6)},
+ {R2N2, info(98, 2)},
+ {R2N3, info(96, 4)}
+ ],
+ Result = build_final_response(USet, 6, Resps),
+ ?assertEqual(198, couch_util:get_value(doc_count, Result)),
+ ?assertEqual(undefined, couch_util:get_value(pending_updates, Result)),
+ ?assertEqual(
+ {[
+ {minimum, 2},
+ {preferred, 2},
+ {total, 12},
+ {maximum, 10},
+ {copies, 4},
+ {copies_expected, 6}
+ ]},
+ couch_util:get_value(updates_pending, Result)
+ ),
+ % No backlog in this case so no updates_pending either
+ NoPending = [{R1N1, [{disk_size, 5}]}, {R2N2, [{disk_size, 7}]}],
+ ?assertEqual([{disk_size, 12}], build_final_response(USet, 6, NoPending)).
+
+-endif.
diff --git a/src/dreyfus/src/dreyfus_index.erl b/src/dreyfus/src/dreyfus_index.erl
index d75405001..9c4414874 100644
--- a/src/dreyfus/src/dreyfus_index.erl
+++ b/src/dreyfus/src/dreyfus_index.erl
@@ -26,6 +26,7 @@
await/2,
search/2,
info/1,
+ info/3,
group1/2,
group2/2,
design_doc_to_indexes/2
@@ -79,6 +80,38 @@ info(Pid0) ->
MFA = {?MODULE, info_int, [Pid]},
dreyfus_util:time([index, info], MFA).
+% Shard local index info from ddoc and index
+info(DbName, DDoc, IndexName) ->
+ case design_doc_to_index(DbName, DDoc, IndexName) of
+ {ok, Index} ->
+ case dreyfus_index_manager:get_index(DbName, Index) of
+ {ok, Pid} ->
+ case info(Pid) of
+ {ok, Fields} ->
+ Info = [
+ {signature, Index#index.sig},
+ {pending_updates, pending_updates(DbName, Fields)}
+ | Fields
+ ],
+ {ok, Info};
+ Else ->
+ Else
+ end;
+ Error ->
+ Error
+ end;
+ Error ->
+ Error
+ end.
+
+% Works like view pending_updates from couch_index. Return a difference
+% betweenn committed db seq and index
+pending_updates(DbName, Fields) ->
+ GetCommSeq = fun(Db) -> couch_db:get_committed_update_seq(Db) end,
+ CommittedSeq = couch_util:with_db(DbName, GetCommSeq),
+ IndexSeq = couch_util:get_value(pending_seq, Fields, 0),
+ max(CommittedSeq - IndexSeq, 0).
+
%% We either have a dreyfus_index gen_server pid or the remote
%% clouseau pid.
to_index_pid(Pid) ->
diff --git a/src/dreyfus/src/dreyfus_rpc.erl b/src/dreyfus/src/dreyfus_rpc.erl
index 5fbe194dc..b00af49e6 100644
--- a/src/dreyfus/src/dreyfus_rpc.erl
+++ b/src/dreyfus/src/dreyfus_rpc.erl
@@ -80,23 +80,7 @@ info(DbName, DDoc, IndexName) ->
info_int(DbName, DDoc, IndexName) ->
erlang:put(io_priority, {search, DbName}),
check_interactive_mode(),
- case dreyfus_index:design_doc_to_index(DbName, DDoc, IndexName) of
- {ok, Index} ->
- case dreyfus_index_manager:get_index(DbName, Index) of
- {ok, Pid} ->
- case dreyfus_index:info(Pid) of
- {ok, Fields} ->
- Info = [{signature, Index#index.sig} | Fields],
- rexi:reply({ok, Info});
- Else ->
- rexi:reply(Else)
- end;
- Error ->
- rexi:reply(Error)
- end;
- Error ->
- rexi:reply(Error)
- end.
+ rexi:reply(dreyfus_index:info(DbName, DDoc, IndexName)).
disk_size(DbName, DDoc, IndexName) ->
erlang:put(io_priority, {search, DbName}),
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index ab575f135..5397df507 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -59,12 +59,13 @@
changes/4,
query_view/3, query_view/4, query_view/6, query_view/7,
get_view_group_info/2,
+ get_index_info/2, get_index_info/3, get_index_info/4,
end_changes/0
]).
% miscellany
-export([
- design_docs/1,
+ design_docs/1, design_docs/2,
reset_validation_funs/1,
cleanup_index_files_all_nodes/0,
cleanup_index_files_all_nodes/1,
@@ -590,40 +591,70 @@ query_view(Db, Options, DDoc, ViewName, Callback, Acc0, QueryArgs0) ->
{minimum, non_neg_integer()}
| {preferred, non_neg_integer()}
| {total, non_neg_integer()}
+ | {maximum, non_neg_integer()}
+ | {copies, non_neg_integer()}
+ | {copies_expected, non_neg_integer()}
]}
]}.
get_view_group_info(DbName, DesignId) ->
fabric_group_info:go(dbname(DbName), design_doc(DesignId)).
+%% @doc get index info of all indexes: view, search and nouveau etc
+get_index_info(DbName, Types) ->
+ get_index_info(DbName, Types, #mrargs{}).
+
+%% @doc same as get_index_info/2 restricted to the design docs selected by
+%% the all_docs style arguments (start_key, end_key, limit, skip, keys)
+get_index_info(DbName, Types, #mrargs{} = Args) ->
+ fabric_index_info:go(dbname(DbName), Types, Args).
+
+%% @doc same as get_index_info/3 with options. {max_ddocs, N} rejects a
+%% request with {error, too_many_design_docs} if we pick more than N design docs
+get_index_info(DbName, Types, #mrargs{} = Args, Opts) ->
+ fabric_index_info:go(dbname(DbName), Types, Args, Opts).
+
-spec end_changes() -> ok.
end_changes() ->
fabric_view_changes:increment_changes_epoch().
-%% @doc retrieve all the design docs from a database
+%% @doc retrieve all the design docs from a database and the meta info
-spec design_docs(dbname()) -> {ok, [json_obj()]} | {error, Reason :: term()}.
design_docs(DbName) ->
- Extra0 = [{view_row_map, true}],
+ case design_docs(DbName, #mrargs{}) of
+ {ok, _Meta, DDocs} -> {ok, DDocs};
+ {error, Reason} -> {error, Reason}
+ end.
+
+design_docs(DbName, #mrargs{extra = Extra0} = Args0) ->
+ Extra1 = [{view_row_map, true} | Extra0],
Extra =
case get(io_priority) of
- undefined -> Extra0;
- Else -> [{io_priority, Else} | Extra0]
+ undefined -> Extra1;
+ Else -> [{io_priority, Else} | Extra1]
end,
- QueryArgs0 = #mrargs{
+ QueryArgs0 = Args0#mrargs{
include_docs = true,
extra = Extra
},
QueryArgs = set_namespace(<<"_design">>, QueryArgs0),
Callback = fun
- ({meta, _}, []) ->
- {ok, []};
- ({row, Props}, Acc) ->
- {ok, [couch_util:get_value(doc, Props) | Acc]};
- (complete, Acc) ->
- {ok, lists:reverse(Acc)};
+ ({meta, Meta}, {_, Acc}) ->
+ {ok, {Meta, Acc}};
+ ({row, Props}, {Meta, Acc}) ->
+ % For "keys" rows of missing docs won't have a "doc" property
+ case couch_util:get_value(doc, Props) of
+ {_} = DDoc -> {ok, {Meta, [DDoc | Acc]}};
+ _ -> {ok, {Meta, Acc}}
+ end;
+ (complete, {Meta, Acc}) ->
+ {ok, {Meta, lists:reverse(Acc)}};
({error, Reason}, _Acc) ->
{error, Reason}
end,
- fabric:all_docs(dbname(DbName), [?ADMIN_CTX], Callback, [], QueryArgs).
+ case fabric:all_docs(dbname(DbName), [?ADMIN_CTX], Callback, {[], []}, QueryArgs) of
+ {ok, {Meta, DDocs}} -> {ok, Meta, DDocs};
+ {error, Reason} -> {error, Reason}
+ end.
%% @doc forces a reload of validation functions, this is performed after
%% design docs are update
diff --git a/src/fabric/src/fabric_group_info.erl b/src/fabric/src/fabric_group_info.erl
index 29f4835a0..317240f7f 100644
--- a/src/fabric/src/fabric_group_info.erl
+++ b/src/fabric/src/fabric_group_info.erl
@@ -13,6 +13,8 @@
-module(fabric_group_info).
-export([go/2]).
+% Used by fabric_index_info
+-export([build_final_response/3]).
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch/include/couch_db.hrl").
@@ -26,87 +28,85 @@ go(DbName, #doc{id = DDocId}) ->
Workers = fabric_util:submit_jobs(Shards, group_info, [DDocId]),
RexiMon = fabric_util:create_monitors(Shards),
USet = couch_util:set_from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
- Acc = {fabric_dict:init(Workers, nil), [], USet},
+ Acc = {fabric_dict:init(Workers, nil), [], USet, length(Workers), undefined},
try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc) of
- {timeout, {WorkersDict, _, _}} ->
+ {timeout, {WorkersDict, Resps, _, Expected, _}} ->
DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil),
fabric_util:log_timeout(DefunctWorkers, "group_info"),
- {error, timeout};
+ fabric_util:cleanup(DefunctWorkers),
+ finish(Resps, USet, Expected, timeout);
Else ->
Else
after
rexi_monitor:stop(RexiMon)
end.
-handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, {Counters, Resps, USet}) ->
- case fabric_ring:node_down(NodeRef, Counters, Resps) of
- {ok, Counters1} -> {ok, {Counters1, Resps, USet}};
- error -> {error, {nodedown, <<"progress not possible">>}}
- end;
-handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps, USet}) ->
- case fabric_ring:handle_error(Shard, Counters, Resps) of
- {ok, Counters1} -> {ok, {Counters1, Resps, USet}};
- error -> {error, Reason}
- end;
-handle_message({ok, Info}, Shard, {Counters, Resps, USet}) ->
- case fabric_ring:handle_response(Shard, Info, Counters, Resps) of
- {ok, {Counters1, Resps1}} ->
- {ok, {Counters1, Resps1, USet}};
- {stop, Resps1} ->
- {stop, build_final_response(USet, Resps1)}
- end;
-handle_message(Reason, Shard, {Counters, Resps, USet}) ->
- case fabric_ring:handle_error(Shard, Counters, Resps) of
- {ok, Counters1} -> {ok, {Counters1, Resps, USet}};
- error -> {error, Reason}
+handle_message({ok, Info}, Shard, {Counters, Resps, USet, Expected, LastErr}) ->
+ Counters1 = fabric_dict:erase(Shard, Counters),
+ maybe_stop(Counters1, [{Shard, Info} | Resps], USet, Expected, LastErr);
+handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _, {Counters, Resps, USet, Expected, LastErr}) ->
+ Counters1 = fabric_dict:filter(fun(#shard{node = N}, _) -> N =/= NodeRef end, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, LastErr);
+handle_message({rexi_EXIT, Reason}, Shard, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Shard, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason);
+handle_message(Reason, Shard, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Shard, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason).
+
+maybe_stop(Counters, Resps, USet, Expected, FinalErr) ->
+ case fabric_dict:size(Counters) of
+ 0 ->
+ Error =
+ case FinalErr of
+ undefined -> {nodedown, <<"progress not possible">>};
+ _ -> FinalErr
+ end,
+ case finish(Resps, USet, Expected, Error) of
+ {ok, Result} -> {stop, Result};
+ {error, _} = E -> E
+ end;
+ _ ->
+ {ok, {Counters, Resps, USet, Expected, FinalErr}}
+ end.
+
+% Every range needs at least one reporting copy.
+finish(Resps, USet, Expected, Error) ->
+ Ranges = [{B, E} || {#shard{range = [B, E]}, _} <- Resps],
+ case mem3_util:get_ring(Ranges) of
+ [] -> {error, Error};
+ _ -> {ok, build_final_response(USet, Expected, Resps)}
end.
-build_final_response(USet, Responses) ->
- AccF = fabric_dict:fold(
- fun(#shard{name = Id, node = Node}, Info, Acc) ->
- IsPreferred = sets:is_element({Id, Node}, USet),
- dict:append(Id, {Node, IsPreferred, Info}, Acc)
+build_final_response(USet, Expected, Resps) ->
+ ByRange = lists:foldl(
+ fun({#shard{range = Range, name = Id, node = Node}, Info}, Acc) ->
+ orddict:append(Range, {sets:is_element({Id, Node}, USet), Info}, Acc)
end,
- dict:new(),
- Responses
+ orddict:new(),
+ Resps
),
- Pending = aggregate_pending(AccF),
- Infos = get_infos(AccF),
- [{updates_pending, {Pending}} | merge_results(Infos)].
-
-get_infos(Acc) ->
- Values = [V || {_, V} <- dict:to_list(Acc)],
- lists:flatten([Info || {_Node, _Pref, Info} <- lists:flatten(Values)]).
-
-aggregate_pending(Dict) ->
- {Preferred, Total, Minimum} =
- dict:fold(
- fun(_Name, Results, {P, T, M}) ->
- {Preferred, Total, Minimum} = calculate_pending(Results),
- {P + Preferred, T + Total, M + Minimum}
- end,
- {0, 0, 0},
- Dict
- ),
- [
- {minimum, Minimum},
- {preferred, Preferred},
- {total, Total}
- ].
-
-calculate_pending(Results) ->
- lists:foldl(
- fun
- ({_Node, true, Info}, {P, T, V}) ->
- Pending = couch_util:get_value(pending_updates, Info),
- {P + Pending, T + Pending, min(Pending, V)};
- ({_Node, false, Info}, {P, T, V}) ->
- Pending = couch_util:get_value(pending_updates, Info),
- {P, T + Pending, min(Pending, V)}
- end,
- {0, 0, infinity},
- Results
- ).
+ RangeCopies = [Copies || {_Range, Copies} <- ByRange],
+ Reps = [representative(Copies) || Copies <- RangeCopies],
+ PerRange = [
+ [{Pref, couch_util:get_value(pending_updates, Info)} || {Pref, Info} <- Copies]
+ || Copies <- RangeCopies
+ ],
+ Merged = merge_results(lists:append(Reps)),
+ case fabric_util:aggregate_pending(PerRange, Expected) of
+ undefined -> Merged;
+ Pending -> [{updates_pending, {Pending}} | Merged]
+ end.
+
+% If there is a ushard pick that so we don't flop as much from one call to the next
+representative(Copies) ->
+ case [Info || {true, Info} <- Copies] of
+ [Info | _] ->
+ Info;
+ [] ->
+ [{_Pref, Info} | _] = Copies,
+ Info
+ end.
merge_results(Info) ->
Dict = lists:foldl(
@@ -161,3 +161,54 @@ merge_object(Objects) ->
[],
Dict
).
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+shard(Range, Node) ->
+ #shard{name = ?l2b(io_lib:format("~p", [Range])), node = Node, range = Range}.
+
+info(Seq, Pending) ->
+ [{update_seq, Seq}, {pending_updates, Pending}, {signature, ~"s"}].
+
+build_final_response_test() ->
+ R1 = [0, 10],
+ R2 = [11, 20],
+ R1N1 = shard(R1, n1),
+ R2N2 = shard(R2, n2),
+ USet = couch_util:set_from_list([
+ {R1N1#shard.name, n1}, {R2N2#shard.name, n2}
+ ]),
+ Resps = [
+ {R1N1, info(100, 0)},
+ {shard(R1, n2), info(94, 6)},
+ {R2N2, info(98, 2)},
+ {shard(R2, n3), info(96, 4)}
+ ],
+ Result = build_final_response(USet, 6, Resps),
+ % update_seq comes from the prefered set of shards only (100+98=198)
+ ?assertEqual(198, couch_util:get_value(update_seq, Result)),
+ ?assertEqual(~"s", couch_util:get_value(signature, Result)),
+ ?assertEqual(undefined, couch_util:get_value(pending_updates, Result)),
+ ?assertEqual(
+ {[
+ {minimum, 2},
+ {preferred, 2},
+ {total, 12},
+ {maximum, 10},
+ {copies, 4},
+ {copies_expected, 6}
+ ]},
+ couch_util:get_value(updates_pending, Result)
+ ).
+
+finish_requires_complete_ring_test() ->
+ A = [0, 2147483647],
+ B = [2147483648, 4294967295],
+ Info = info(1, 0),
+ ?assertEqual({error, boom}, finish([{shard(A, n1), Info}], sets:new(), 4, boom)),
+ ?assertMatch(
+ {ok, [_ | _]}, finish([{shard(A, n1), Info}, {shard(B, n2), Info}], sets:new(), 4, boom)
+ ).
+
+-endif.
diff --git a/src/fabric/src/fabric_index_info.erl b/src/fabric/src/fabric_index_info.erl
new file mode 100644
index 000000000..fbb24cdef
--- /dev/null
+++ b/src/fabric/src/fabric_index_info.erl
@@ -0,0 +1,436 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% Get index status. Gather design docs and then send them all to the workers
+% and they locally gather index info in parallel. We query live shard copies
+% and merge the result for each index. Fields like size, seqs and counts are
+% returned from one copy per range and updates_pending object is computed from
+% all the responding copies.
+%
+% Results are grouped per index type: [{Type, [{Ident, {ok, Info} | {error,
+% Err}}]}]. Type = view|search|nouveau. Ident identifies and index as [{ddoc,
+% Id}] and {name, Name} for search and nouveau indexes, or {views, Names} for
+% view groups.
+
+-module(fabric_index_info).
+
+-export([go/2, go/3, go/4]).
+
+-include_lib("mem3/include/mem3.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
+-define(UNAVAILABLE, {unavailable, ~"Index info is missing"}).
+
+go(DbName, Types) ->
+ go(DbName, Types, #mrargs{}).
+
+go(DbName, Types, #mrargs{} = Args) ->
+ go(DbName, Types, Args, []).
+
+% Args select the design docs like they do for _design_docs (start_key,
+% end_key, limit, skip, keys). Also {max_ddocs, N} specifies the maximum number
+% of ddocs. Specifying more than that will return an error. Use pagination to
+% get more results. To help paginate we get one extra design doc past the end
+% then we use it's doc ID as the bookmark (startkey) of the next page.
+go(DbName, Types, #mrargs{} = Args, Opts) ->
+ Max = couch_util:get_value(max_ddocs, Opts, infinity),
+ case fabric:design_docs(DbName, fetch_args(Args, Max)) of
+ {ok, Meta, Fetched} ->
+ case page(Fetched, Args, Max) of
+ {ok, DDocs, Bookmark} ->
+ {Ctx, Responses} = index_info(DbName, DDocs, Types),
+ ByType = order(merge(DDocs, Types, Responses, Ctx), Args),
+ {ok, [{bookmark, Bookmark} | Meta], ByType};
+ {error, _} = Error ->
+ Error
+ end;
+ {error, _} = Error ->
+ Error
+ end.
+
+% Fetch one more design doc to let user paginate easily
+fetch_args(#mrargs{limit = Limit} = Args, Max) ->
+ Args#mrargs{limit = page_size(Limit, Max) + 1}.
+
+page_size(Limit, infinity) ->
+ Limit;
+page_size(Limit, Max) ->
+ min(Limit, Max).
+
+page(Fetched, #mrargs{limit = Limit, keys = Keys}, Max) ->
+ Size = page_size(Limit, Max),
+ case length(Fetched) > Size of
+ true when Limit > Max ->
+ {error, too_many_design_docs};
+ true ->
+ {Page, [Next | _]} = lists:split(Size, Fetched),
+ {ok, Page, bookmark(Next, Keys)};
+ false ->
+ {ok, Fetched, null}
+ end.
+
+bookmark(_Next, Keys) when is_list(Keys) ->
+ null;
+bookmark(Next, undefined) ->
+ ddoc_id(Next).
+
+% Handle descending order
+order(ByType, #mrargs{direction = rev}) ->
+ [{Type, lists:reverse(Entries)} || {Type, Entries} <- ByType];
+order(ByType, #mrargs{}) ->
+ ByType.
+
+% Returns {Ctx, Responses}. Ctx is the set of preferred (ushard) copies and the
+% number of copies. Merge will need the number of copies to build
+% updates_pending object.
+index_info(DbName, DDocs, Types) ->
+ Shards = mem3:shards(DbName),
+ Ushards = mem3:ushards(DbName),
+ USet = couch_util:set_from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
+ Workers = fabric_util:submit_jobs(Shards, fabric_rpc, index_info, [DDocs, Types]),
+ RexiMon = fabric_util:create_monitors(Workers),
+ Ctx = {USet, length(Workers)},
+ Acc0 = {fabric_dict:init(Workers, nil), []},
+ try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of
+ {ok, Responses} ->
+ {Ctx, Responses};
+ {timeout, {WorkersDict, Responses}} ->
+ DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil),
+ fabric_util:log_timeout(DefunctWorkers, "index_info"),
+ fabric_util:cleanup(DefunctWorkers),
+ {Ctx, Responses};
+ {error, Error} ->
+ fabric_util:cleanup(Workers),
+ throw(Error)
+ after
+ rexi_monitor:stop(RexiMon)
+ end.
+
+handle_message({ok, Result}, Worker, {Counters, Acc}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, [{Worker, Result} | Acc]);
+handle_message({rexi_DOWN, _, {_, NRef}, _}, _Worker, {Counters, Acc}) ->
+ Counters1 = fabric_dict:filter(fun(#shard{node = N}, _) -> N =/= NRef end, Counters),
+ maybe_stop(Counters1, Acc);
+handle_message(_Error, Worker, {Counters, Acc}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Acc).
+
+maybe_stop(Counters, Acc) ->
+ case fabric_dict:size(Counters) of
+ 0 -> {stop, Acc};
+ _ -> {ok, {Counters, Acc}}
+ end.
+
+% Response = {Shard, [{DDocId, Sections}]}
+% Sections: {error, Err} | [
+% {view_index, Res},
+% {search_indexes, [{Name, Res}]},
+% {nouveau_indexes, [{Name, Res}]}
+% ]
+% Res: {ok, Info} | {error, Err}
+%
+% Note: view ddoc is a group so there only one per ddoc
+%
+merge(DDocs, Types, Responses, Ctx) ->
+ {DDocErrors, ByIndex} = collect(Responses),
+ [{Type, merge_type(Type, DDocs, DDocErrors, ByIndex, Ctx)} || Type <- Types].
+
+collect(Responses) ->
+ lists:foldl(
+ fun({Shard, PerDDoc}, Acc0) ->
+ lists:foldl(
+ fun
+ ({DDocId, {error, Error}}, {Errs, Idx}) ->
+ {maps:merge(#{DDocId => Error}, Errs), Idx};
+ ({DDocId, Sect}, {Errs, Idx}) ->
+ Fun = fun({K, V}, A) -> add_section(DDocId, K, V, Shard, A) end,
+ {Errs, lists:foldl(Fun, Idx, Sect)}
+ end,
+ Acc0,
+ PerDDoc
+ )
+ end,
+ {#{}, orddict:new()},
+ Responses
+ ).
+
+add_section(DDocId, view_index, Res, Shard, Idx) ->
+ orddict:append({view, DDocId, undefined}, {Shard, Res}, Idx);
+add_section(DDocId, search_indexes, Named, Shard, Idx) ->
+ add_named(search, DDocId, Named, Shard, Idx);
+add_section(DDocId, nouveau_indexes, Named, Shard, Idx) ->
+ add_named(nouveau, DDocId, Named, Shard, Idx).
+
+add_named(Type, DDocId, Named, Shard, Idx) ->
+ lists:foldl(
+ fun({Name, Res}, A) -> orddict:append({Type, DDocId, Name}, {Shard, Res}, A) end,
+ Idx,
+ Named
+ ).
+
+% Sort entries by ddoc and then index name
+merge_type(Type, DDocs, DDocErrors, ByIndex, Ctx) ->
+ ByDDocId = maps:from_list([{ddoc_id(DDoc), DDoc} || DDoc <- DDocs]),
+ Keys = lists:usort(lists:flatmap(fun(DDoc) -> declared(Type, DDoc) end, DDocs)),
+ [{ident(Key, ByDDocId), merge_index(Type, Key, DDocErrors, ByIndex, Ctx)} || Key <- Keys].
+
+% Works like fabric_rpc:section/5
+declared(view, {Props} = DDoc) ->
+ case couch_util:get_value(~"views", Props) of
+ {[_ | _]} -> [{view, ddoc_id(DDoc), undefined}];
+ _ -> []
+ end;
+declared(search, {Props} = DDoc) ->
+ named(search, ddoc_id(DDoc), couch_util:get_value(~"indexes", Props));
+declared(nouveau, {Props} = DDoc) ->
+ named(nouveau, ddoc_id(DDoc), couch_util:get_value(~"nouveau", Props)).
+
+named(Type, DDocId, {[_ | _] = Indexes}) ->
+ [{Type, DDocId, Name} || {Name, _} <- Indexes];
+named(_Type, _DDocId, _) ->
+ [].
+
+ddoc_id({Props}) ->
+ couch_util:get_value(~"_id", Props).
+
+% Identity fields of an entry. For consistency these are borrowed from GET
+% /db/_index response. A view group is per ddoc so instead of a name it lists
+% its views.
+ident({view, DDocId, undefined}, ByDDocId) ->
+ [{ddoc, DDocId}, {views, view_names(maps:get(DDocId, ByDDocId))}];
+ident({_Type, DDocId, IndexName}, _ByDDocId) ->
+ [{ddoc, DDocId}, {name, IndexName}].
+
+view_names({Props}) ->
+ {Views} = couch_util:get_value(~"views", Props),
+ lists:usort([Name || {Name, _} <- Views, Name =/= ~"lib"]).
+
+% Show an error if there is one. No results at all is also an error.
+merge_index(Type, {_, DDocId, _} = Key, DDocErrors, ByIndex, Ctx) ->
+ case DDocErrors of
+ #{DDocId := Error} ->
+ {error, Error};
+ #{} ->
+ case orddict:find(Key, ByIndex) of
+ {ok, Results} -> merge_leaf(Type, Results, Ctx);
+ error -> {error, ?UNAVAILABLE}
+ end
+ end.
+
+% Results: {Shard, {ok, Info} | {error, Err}} per copy. Errors "win"
+merge_leaf(Type, Results, Ctx) ->
+ case [Error || {_, {error, Error}} <- Results] of
+ [Error | _] -> {error, Error};
+ [] -> merge_oks(Type, Results, Ctx)
+ end.
+
+% The per index info endpoints do the merging so the entry is exactly what
+% they would return: additive fields from one (preferred) copy per range and
+% updates_pending across all the copies. Just like there, the merged fields
+% must cover the whole database so every range needs at least one reporting
+% copy.
+merge_oks(Type, Results, {USet, CopiesExpected}) ->
+ Resps = [{Shard, Info} || {Shard, {ok, Info}} <- Results],
+ Ranges = [{B, E} || {#shard{range = [B, E]}, _} <- Resps],
+ case mem3_util:get_ring(Ranges) of
+ [] -> {error, ?UNAVAILABLE};
+ _ -> {ok, merge_infos(Type, USet, CopiesExpected, Resps)}
+ end.
+
+merge_infos(view, USet, CopiesExpected, Resps) ->
+ {fabric_group_info:build_final_response(USet, CopiesExpected, Resps)};
+merge_infos(search, USet, CopiesExpected, Resps) ->
+ {dreyfus_fabric_info:build_final_response(USet, CopiesExpected, Resps)};
+merge_infos(nouveau, USet, CopiesExpected, Resps) ->
+ nouveau_fabric_info:build_final_response(USet, CopiesExpected, Resps).
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+% Two ranges which form a complete ring (q=2) and a single range one (q=1)
+-define(R1, [0, 2147483647]).
+-define(R2, [2147483648, 4294967295]).
+-define(RALL, [0, 4294967295]).
+
+shard(Range, Node) ->
+ #shard{name = list_to_binary(io_lib:format("~p", [Range])), node = Node, range = Range}.
+
+% n1 is the preferred copy of R1 and RALL, n2 of R2; 6 copies expected
+ctx() ->
+ USet = couch_util:set_from_list([
+ {(shard(?R1, n1))#shard.name, n1},
+ {(shard(?RALL, n1))#shard.name, n1},
+ {(shard(?R2, n2))#shard.name, n2}
+ ]),
+ {USet, 6}.
+
+pending(Min, Pref, Total, Max, Copies) ->
+ [
+ {minimum, Min},
+ {preferred, Pref},
+ {total, Total},
+ {maximum, Max},
+ {copies, Copies},
+ {copies_expected, 6}
+ ].
+
+viewres(Pending, Seq) ->
+ {ok, [{signature, ~"sig"}, {pending_updates, Pending}, {update_seq, Seq}]}.
+
+% A ddoc with a two view group (plus a lib), one search index and one nouveau
+% index
+ddoc() ->
+ {[
+ {~"_id", ~"_design/d"},
+ {
+ ~"views",
+ {[
+ {~"v2", {[{~"map", ~"function(d){}"}]}},
+ {~"lib", {[{~"x", ~"exports.x = 1"}]}},
+ {~"v1", {[{~"map", ~"function(d){}"}]}}
+ ]}
+ },
+ {~"indexes", {[{~"s", {[{~"index", ~"function(d){}"}]}}]}},
+ {~"nouveau", {[{~"n", {[{~"index", ~"function(d){}"}]}}]}}
+ ]}.
+
+view_ident() -> [{ddoc, ~"_design/d"}, {views, [~"v1", ~"v2"]}].
+
+merge_responses_test() ->
+ % Most fields come from a preferred cover. Pending computed from all shards.
+ Res = [
+ {shard(?R1, n1), [{~"_design/d", [{view_index, viewres(0, 100)}]}]},
+ {shard(?R1, n2), [{~"_design/d", [{view_index, viewres(6, 94)}]}]},
+ {shard(?R2, n2), [{~"_design/d", [{view_index, viewres(2, 98)}]}]},
+ {shard(?R2, n3), [{~"_design/d", [{view_index, viewres(4, 96)}]}]}
+ ],
+ [{view, [{Ident, {ok, {Props}}}]}] = merge([ddoc()], [view], Res, ctx()),
+ ?assertEqual(view_ident(), Ident),
+ ?assertEqual(198, couch_util:get_value(update_seq, Props)),
+ ?assertEqual(undefined, couch_util:get_value(pending_updates, Props)),
+ ?assertEqual({pending(2, 2, 12, 10, 4)}, couch_util:get_value(updates_pending, Props)).
+
+merge_responses_search_test() ->
+ SRes = fun(Pending, Docs) ->
+ {ok, [{signature, ~"s"}, {pending_updates, Pending}, {doc_count, Docs}]}
+ end,
+ Res = [
+ {shard(?RALL, n1), [{~"_design/d", [{search_indexes, [{~"s", SRes(3, 7)}]}]}]},
+ {shard(?RALL, n2), [{~"_design/d", [{search_indexes, [{~"s", SRes(7, 5)}]}]}]}
+ ],
+ [{search, [{Ident, {ok, {Props}}}]}] = merge([ddoc()], [search], Res, ctx()),
+ ?assertEqual([{ddoc, ~"_design/d"}, {name, ~"s"}], Ident),
+ ?assertEqual(7, couch_util:get_value(doc_count, Props)),
+ ?assertEqual(undefined, couch_util:get_value(pending_updates, Props)),
+ ?assertEqual({pending(3, 3, 10, 7, 2)}, couch_util:get_value(updates_pending, Props)).
+
+merge_responses_nouveau_test() ->
+ % Nouveau uses maps for info so add a separate test for it
+ NRes = fun(Pending, Seq) ->
+ {ok, #{signature => ~"s", pending_updates => Pending, ~"update_seq" => Seq}}
+ end,
+ Res = [
+ {shard(?RALL, n1), [{~"_design/d", [{nouveau_indexes, [{~"n", NRes(5, 10)}]}]}]},
+ {shard(?RALL, n2), [{~"_design/d", [{nouveau_indexes, [{~"n", NRes(0, 15)}]}]}]}
+ ],
+ [{nouveau, [{Ident, {ok, Merged}}]}] = merge([ddoc()], [nouveau], Res, ctx()),
+ ?assertEqual([{ddoc, ~"_design/d"}, {name, ~"n"}], Ident),
+ ?assertEqual(10, maps:get(~"update_seq", Merged)),
+ ?assertNot(maps:is_key(pending_updates, Merged)),
+ ?assertEqual(maps:from_list(pending(0, 5, 5, 5, 2)), maps:get(updates_pending, Merged)).
+
+merge_responses_error_wins_test() ->
+ Res = [
+ {shard(?RALL, n1), [{~"_design/d", [{view_index, viewres(0, 100)}]}]},
+ {shard(?RALL, n2), [{~"_design/d", [{view_index, {error, not_found}}]}]}
+ ],
+ ?assertEqual(
+ [{view, [{view_ident(), {error, not_found}}]}],
+ merge([ddoc()], [view], Res, ctx())
+ ).
+
+merge_responses_ddoc_error_test() ->
+ % Error on one copy marks all of that ddoc's indexes
+ Res = [
+ {shard(?RALL, n1), [{~"_design/d", {error, bad_ddoc}}]},
+ {shard(?RALL, n2), [{~"_design/d", [{view_index, viewres(0, 100)}]}]}
+ ],
+ ?assertEqual(
+ [
+ {view, [{view_ident(), {error, bad_ddoc}}]},
+ {search, [{[{ddoc, ~"_design/d"}, {name, ~"s"}], {error, bad_ddoc}}]},
+ {nouveau, [{[{ddoc, ~"_design/d"}, {name, ~"n"}], {error, bad_ddoc}}]}
+ ],
+ merge([ddoc()], [view, search, nouveau], Res, ctx())
+ ).
+
+merge_responses_no_copies_test() ->
+ % If nobody responds we get an error
+ NoViews = {[{~"_id", ~"_design/e"}, {~"indexes", {[{~"s", {[]}}]}}]},
+ Expected = [
+ {view, []},
+ {search, [{[{ddoc, ~"_design/e"}, {name, ~"s"}], {error, ?UNAVAILABLE}}]}
+ ],
+ ?assertEqual(Expected, merge([NoViews], [view, search], [], ctx())).
+
+merge_responses_incomplete_ring_test() ->
+ % Test failure to cover the whole range
+ Res = [
+ {shard(?R1, n1), [{~"_design/d", [{view_index, viewres(0, 100)}]}]},
+ {shard(?R1, n2), [{~"_design/d", [{view_index, viewres(6, 94)}]}]}
+ ],
+ ?assertEqual(
+ [{view, [{view_ident(), {error, ?UNAVAILABLE}}]}],
+ merge([ddoc()], [view], Res, ctx())
+ ).
+
+merge_responses_sorted_test() ->
+ % Entries are sorted by ddoc id then index name
+ DDocA = {[{~"_id", ~"_design/a"}, {~"indexes", {[{~"z", {[]}}, {~"b", {[]}}]}}]},
+ Res = [
+ {shard(?RALL, n1), [
+ {~"_design/d", [{search_indexes, [{~"s", {ok, [{doc_count, 1}]}}]}]},
+ {~"_design/a", [
+ {search_indexes, [
+ {~"z", {ok, [{doc_count, 2}]}}, {~"b", {ok, [{doc_count, 3}]}}
+ ]}
+ ]}
+ ]}
+ ],
+ [{search, Entries}] = merge([ddoc(), DDocA], [search], Res, ctx()),
+ ?assertEqual(
+ [
+ [{ddoc, ~"_design/a"}, {name, ~"b"}],
+ [{ddoc, ~"_design/a"}, {name, ~"z"}],
+ [{ddoc, ~"_design/d"}, {name, ~"s"}]
+ ],
+ [Ident || {Ident, _} <- Entries]
+ ).
+
+page_test() ->
+ DdA = {[{~"_id", ~"_design/a"}]},
+ DdB = {[{~"_id", ~"_design/b"}]},
+ DdC = {[{~"_id", ~"_design/c"}]},
+ Args = fun(L) -> #mrargs{limit = L} end,
+ ?assertEqual(#mrargs{limit = 101}, fetch_args(Args(100), infinity)),
+ ?assertEqual(#mrargs{limit = 3}, fetch_args(Args(100), 2)),
+ ?assertEqual(#mrargs{limit = 2}, fetch_args(Args(1), 2)),
+ ?assertEqual({ok, [DdA, DdB], null}, page([DdA, DdB], Args(2), infinity)),
+ ?assertEqual({ok, [DdA, DdB], null}, page([DdA, DdB], Args(100), 2)),
+ ?assertEqual({ok, [DdA, DdB], ~"_design/c"}, page([DdA, DdB, DdC], Args(2), infinity)),
+ ?assertEqual({ok, [DdA, DdB], ~"_design/c"}, page([DdA, DdB, DdC], Args(2), 2)),
+ ?assertEqual({error, too_many_design_docs}, page([DdA, DdB, DdC], Args(100), 2)),
+ ?assertEqual({ok, [DdA], null}, page([DdA, DdB], #mrargs{limit = 1, keys = [x]}, 2)),
+ ?assertEqual({ok, [], ~"_design/a"}, page([DdA], Args(0), 2)).
+
+-endif.
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index fd06b391b..5760fc54d 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -26,7 +26,7 @@
get_missing_revs/2, get_missing_revs/3,
update_docs/3
]).
--export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]).
+-export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2, index_info/3]).
-export([
create_db/1, create_db/2,
delete_db/1,
@@ -66,6 +66,9 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
+% How many design docs an index_info/3 worker can process concurrently
+-define(INDEX_INFO_CONCURRENCY, 8).
+
%% rpc endpoints
%% call to with_db will supply your M:F with a Db instance
%% and then remaining args
@@ -337,6 +340,127 @@ purge_docs(DbName, UUIdsIdsRevs, Options) ->
group_info(DbName, DDocId) ->
group_info(DbName, DDocId, []).
+index_info(DbName, DDocs, Types) ->
+ set_io_priority(DbName, []),
+ Avail = {search_available(), nouveau_enabled()},
+ Infos = pmap(fun(DDoc) -> ddoc_info(DbName, DDoc, Types, Avail) end, DDocs),
+ rexi:reply({ok, lists:append(Infos)}).
+
+% Return [{DDocId, Sections}] or [] for empty ddocs
+ddoc_info(ShardName, {Props} = DDocEJson, Types, Avail) ->
+ DDocId = couch_util:get_value(~"_id", Props),
+ try
+ #doc{body = {Body}} = DDoc = couch_doc:from_json_obj(DDocEJson),
+ Sections = lists:append([section(T, ShardName, DDoc, Body, Avail) || T <- Types]),
+ case Sections of
+ [] -> [];
+ [_ | _] -> [{DDocId, Sections}]
+ end
+ catch
+ _Tag:Error ->
+ [{DDocId, {error, Error}}]
+ end.
+
+section(view, ShardName, DDoc, Body, _Avail) ->
+ case couch_util:get_value(~"views", Body) of
+ {[_ | _]} -> [{view_index, view_info(ShardName, DDoc)}];
+ _ -> []
+ end;
+section(search, ShardName, DDoc, Body, {SearchAvailable, _}) ->
+ case couch_util:get_value(~"indexes", Body) of
+ {[_ | _] = Indexes} ->
+ Infos = [
+ {Name, search_info(SearchAvailable, ShardName, DDoc, Name)}
+ || {Name, _} <- Indexes
+ ],
+ [{search_indexes, Infos}];
+ _ ->
+ []
+ end;
+section(nouveau, ShardName, DDoc, Body, {_, NouveauEnabled}) ->
+ case couch_util:get_value(~"nouveau", Body) of
+ {[_ | _] = Indexes} ->
+ Infos = [
+ {Name, nouveau_info(NouveauEnabled, ShardName, DDoc, Name)}
+ || {Name, _} <- Indexes
+ ],
+ [{nouveau_indexes, Infos}];
+ _ ->
+ []
+ end.
+
+view_info(ShardName, DDoc) ->
+ try
+ norm_error(couch_mrview:get_info(ShardName, DDoc))
+ catch
+ error:{badmatch, Error} -> norm_error(Error);
+ _Tag:Error -> {error, Error}
+ end.
+
+search_info(false, _ShardName, _DDoc, _IndexName) ->
+ {error, {service_unavailable, ~"Search is not available"}};
+search_info(true, ShardName, DDoc, IndexName) ->
+ try
+ norm_error(dreyfus_index:info(ShardName, DDoc, IndexName))
+ catch
+ _Tag:Error -> {error, Error}
+ end.
+
+nouveau_info(false, _ShardName, _DDoc, _IndexName) ->
+ {error, {service_unavailable, ~"nouveau is not enabled"}};
+nouveau_info(true, ShardName, DDoc, IndexName) ->
+ try
+ norm_error(nouveau_util:index_info(ShardName, DDoc, IndexName))
+ catch
+ _Tag:Error -> {error, Error}
+ end.
+
+norm_error({ok, Info}) ->
+ {ok, Info};
+norm_error({error, Error}) ->
+ {error, Error};
+norm_error(Else) ->
+ {error, Else}.
+
+search_available() ->
+ try
+ dreyfus:available()
+ catch
+ _:_ -> false
+ end.
+
+nouveau_enabled() ->
+ try
+ nouveau:enabled()
+ catch
+ _:_ -> false
+ end.
+
+% Limit concurrency and use isolated processes
+%
+pmap(Fun, Items) ->
+ lists:append([pmap_chunk(Fun, Chunk) || Chunk <- chunk(Items, ?INDEX_INFO_CONCURRENCY)]).
+
+pmap_chunk(Fun, Items) ->
+ PidRefs = [spawn_monitor(fun() -> exit({pmap_res, Fun(Item)}) end) || Item <- Items],
+ lists:map(
+ fun({Pid, Ref}) ->
+ receive
+ {'DOWN', Ref, process, Pid, {pmap_res, Result}} -> Result;
+ {'DOWN', Ref, process, Pid, Error} -> exit(Error)
+ end
+ end,
+ PidRefs
+ ).
+
+chunk([], _N) ->
+ [];
+chunk(Items, N) when length(Items) =< N ->
+ [Items];
+chunk(Items, N) ->
+ {Chunk, Rest} = lists:split(N, Items),
+ [Chunk | chunk(Rest, N)].
+
group_info(DbName, DDocId, DbOptions) ->
with_db(DbName, DbOptions, {couch_mrview, get_info, [DDocId]}).
diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl
index 1da214f7f..e8b998a19 100644
--- a/src/fabric/src/fabric_util.erl
+++ b/src/fabric/src/fabric_util.erl
@@ -38,6 +38,7 @@
-export([isolate/1, isolate/2]).
-export([get_design_doc_records/1]).
-export([w_from_opts/2, r_from_opts/2]).
+-export([aggregate_pending/2]).
-compile({inline, [{doc_id_and_rev, 1}]}).
@@ -435,6 +436,48 @@ do_isolate(Fun) ->
{'$isolerr', Tag, Reason, Stack}
end.
+% Aggregate per-shard index pending data. PerRange has one entry per shard
+% range, listing its copies as {true|false = IsPreferred, Pending} pairs, the
+% flag indicating whether that's a preferred copy.
+% Pending stats returned:
+% minimum: best copy, 0 => at least one complete copy of every range exists
+% preferred: pending count on preferred copies
+% total: all copies added together
+% maximum: worst copy, 0 => every copy returning a result is completely built
+% copies: number of shard which returned a response
+% copies_expected: the number of shard workers we expected to return a result
+aggregate_pending(PerRange, Expected) ->
+ case lists:foldl(fun aggregate_pending_range/2, undefined, PerRange) of
+ undefined ->
+ undefined;
+ {Min, Pref, Total, Max, Copies} ->
+ [
+ {minimum, Min},
+ {preferred, Pref},
+ {total, Total},
+ {maximum, Max},
+ {copies, Copies},
+ {copies_expected, Expected}
+ ]
+ end.
+
+aggregate_pending_range(Copies, Acc) ->
+ case lists:sort([P || {_, P} <- Copies, is_integer(P)]) of
+ [] ->
+ Acc;
+ [_ | _] = Ps ->
+ {Min0, Pref0, Tot0, Max0, Copies0} =
+ case Acc of
+ undefined -> {0, 0, 0, 0, 0};
+ _ -> Acc
+ end,
+ Pref = lists:sum([P || {true, P} <- Copies, is_integer(P)]),
+ Min = Min0 + hd(Ps),
+ Tot = Tot0 + lists:sum(Ps),
+ Max = Max0 + lists:last(Ps),
+ {Min, Pref0 + Pref, Tot, Max, Copies0 + length(Ps)}
+ end.
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
@@ -541,4 +584,31 @@ t_r_opts_get(_) ->
?assertEqual(3, r_from_opts(any_db, [{r, "five"}])),
?assertEqual(3, r_from_opts(any_db, [])).
+aggregate_pending_test() ->
+ ?assertEqual(undefined, aggregate_pending([], 6)),
+ ?assertEqual(undefined, aggregate_pending([[]], 6)),
+ ?assertEqual(undefined, aggregate_pending([[{true, undefined}]], 6)),
+ % a single copy per range
+ ?assertEqual(
+ [{minimum, 5}, {preferred, 0}, {total, 5}, {maximum, 5}, {copies, 1}, {copies_expected, 3}],
+ aggregate_pending([[{false, 5}]], 3)
+ ),
+ % several copies per range: best and worst copy of each range, summed
+ ?assertEqual(
+ [
+ {minimum, 3},
+ {preferred, 7},
+ {total, 18},
+ {maximum, 11},
+ {copies, 5},
+ {copies_expected, 6}
+ ],
+ aggregate_pending([[{true, 4}, {false, 0}, {false, 8}], [{true, 3}, {false, 3}]], 6)
+ ),
+ % copies without a reported backlog are skipped and not counted
+ ?assertEqual(
+ [{minimum, 2}, {preferred, 0}, {total, 2}, {maximum, 2}, {copies, 1}, {copies_expected, 6}],
+ aggregate_pending([[{true, undefined}, {false, 2}], [{false, undefined}]], 6)
+ ).
+
-endif.
diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl
index 38c43e1d5..e254309c2 100644
--- a/src/fabric/test/eunit/fabric_tests.erl
+++ b/src/fabric/test/eunit/fabric_tests.erl
@@ -402,10 +402,13 @@ design_docs_test_() ->
fun(_) -> meck:unload() end,
[
?TDEF_FE(t_design_docs_configuration),
- ?TDEF_FE(t_design_docs_configuration_io_priority)
+ ?TDEF_FE(t_design_docs_configuration_io_priority),
+ ?TDEF_FE(t_design_docs_meta)
]
}.
+-define(DDOC_RES, {ok, {[{total, 2}, {offset, 1}], [{[{~"_id", ~"_design/d"}]}]}}).
+
t_design_docs_configuration(_) ->
DbName = <<"db">>,
AdminCtx = [?ADMIN_CTX],
@@ -415,9 +418,9 @@ t_design_docs_configuration(_) ->
extra = [{namespace, <<"_design">>}, {view_row_map, true}]
},
meck:expect(
- fabric, all_docs, [DbName, AdminCtx, '_', [], QueryArgs], meck:val(all_docs_result)
+ fabric, all_docs, [DbName, AdminCtx, '_', '_', QueryArgs], meck:val(?DDOC_RES)
),
- ?assertEqual(all_docs_result, fabric:design_docs(DbName)).
+ ?assertEqual({ok, [{[{~"_id", ~"_design/d"}]}]}, fabric:design_docs(DbName)).
t_design_docs_configuration_io_priority(_) ->
DbName = <<"db">>,
@@ -428,10 +431,21 @@ t_design_docs_configuration_io_priority(_) ->
extra = [{namespace, <<"_design">>}, {io_priority, io_priority}, {view_row_map, true}]
},
meck:expect(
- fabric, all_docs, [DbName, AdminCtx, '_', [], QueryArgs], meck:val(all_docs_result)
+ fabric, all_docs, [DbName, AdminCtx, '_', '_', QueryArgs], meck:val(?DDOC_RES)
),
put(io_priority, io_priority),
- ?assertEqual(all_docs_result, fabric:design_docs(DbName)).
+ ?assertEqual({ok, [{[{~"_id", ~"_design/d"}]}]}, fabric:design_docs(DbName)).
+
+t_design_docs_meta(_) ->
+ DbName = ~"db",
+ meck:expect(fabric, all_docs, 5, meck:val(?DDOC_RES)),
+ ?assertEqual(
+ {ok, [{total, 2}, {offset, 1}], [{[{~"_id", ~"_design/d"}]}]},
+ fabric:design_docs(DbName, #mrargs{})
+ ),
+ meck:expect(fabric, all_docs, 5, meck:val({error, timeout})),
+ ?assertEqual({error, timeout}, fabric:design_docs(DbName, #mrargs{})),
+ ?assertEqual({error, timeout}, fabric:design_docs(DbName)).
query_view_test_() ->
{
diff --git a/src/nouveau/src/nouveau_fabric_info.erl b/src/nouveau/src/nouveau_fabric_info.erl
index 0fe41af5a..39dc0608f 100644
--- a/src/nouveau/src/nouveau_fabric_info.erl
+++ b/src/nouveau/src/nouveau_fabric_info.erl
@@ -16,6 +16,8 @@
-module(nouveau_fabric_info).
-export([go/3]).
+% Used by fabric_index_info to merge its per copy results the same way
+-export([build_final_response/3]).
-include_lib("mem3/include/mem3.hrl").
@@ -32,6 +34,8 @@ go(DbName, DDoc, IndexName) ->
go(DbName, _DDoc, _IndexName, Index) ->
Shards = mem3:shards(DbName),
+ Ushards = mem3:ushards(DbName),
+ USet = couch_util:set_from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
Counters0 = lists:map(
fun(#shard{} = Shard) ->
Ref = rexi:cast(
@@ -46,63 +50,137 @@ go(DbName, _DDoc, _IndexName, Index) ->
Workers = fabric_dict:fetch_keys(Counters),
RexiMon = fabric_util:create_monitors(Workers),
- Acc0 = {fabric_dict:init(Workers, nil), #{}},
- try
- fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0)
+ Acc0 = {Counters, [], USet, length(Workers), undefined},
+ try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of
+ {timeout, {WorkersDict, Resps, _, Expected, _}} ->
+ DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil),
+ fabric_util:log_timeout(DefunctWorkers, "nouveau_info"),
+ finish(Resps, USet, Expected, timeout);
+ Else ->
+ Else
after
rexi_monitor:stop(RexiMon),
fabric_util:cleanup(Workers)
end.
-handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, {Counters, Acc}) ->
- case fabric_util:remove_down_workers(Counters, NodeRef) of
- {ok, NewCounters} ->
- {ok, {NewCounters, Acc}};
- error ->
- {error, {nodedown, <<"progress not possible">>}}
- end;
-handle_message({rexi_EXIT, Reason}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, Reason}
- end;
-handle_message({ok, Info}, Worker, {Counters, Acc0}) ->
- case fabric_dict:lookup_element(Worker, Counters) of
- undefined ->
- % already heard from someone else in this range
- {ok, {Counters, Acc0}};
- nil ->
- C1 = fabric_dict:store(Worker, ok, Counters),
- C2 = fabric_view:remove_overlapping_shards(Worker, C1),
- Acc1 = maps:merge_with(fun merge_info/3, Info, Acc0),
- case fabric_dict:any(nil, C2) of
- true ->
- {ok, {C2, Acc1}};
- false ->
- {stop, Acc1}
- end
- end;
-handle_message({error, Reason}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, Reason}
- end;
-handle_message({'EXIT', _}, Worker, {Counters, Acc}) ->
- NewCounters = fabric_dict:erase(Worker, Counters),
- case fabric_ring:is_progress_possible(NewCounters) of
- true ->
- {ok, {NewCounters, Acc}};
- false ->
- {error, {nodedown, <<"progress not possible">>}}
+handle_message({ok, Info}, Worker, {Counters, Resps, USet, Expected, LastErr}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, [{Worker, Info} | Resps], USet, Expected, LastErr);
+handle_message(
+ {rexi_DOWN, _, {_, NodeRef}, _}, _Worker, {Counters, Resps, USet, Expected, LastErr}
+) ->
+ Counters1 = fabric_dict:filter(fun(#shard{node = N}, _) -> N =/= NodeRef end, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, LastErr);
+handle_message({rexi_EXIT, Reason}, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason);
+handle_message({error, Reason}, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason);
+handle_message(Reason, Worker, {Counters, Resps, USet, Expected, _}) ->
+ Counters1 = fabric_dict:erase(Worker, Counters),
+ maybe_stop(Counters1, Resps, USet, Expected, Reason).
+
+maybe_stop(Counters, Resps, USet, Expected, FinalErr) ->
+ case fabric_dict:size(Counters) of
+ 0 ->
+ Error =
+ case FinalErr of
+ undefined -> {nodedown, <<"progress not possible">>};
+ _ -> FinalErr
+ end,
+ case finish(Resps, USet, Expected, Error) of
+ {ok, Result} -> {stop, Result};
+ {error, _} = E -> E
+ end;
+ _ ->
+ {ok, {Counters, Resps, USet, Expected, FinalErr}}
+ end.
+
+% Need at least one response per range for succcess
+finish(Resps, USet, Expected, Error) ->
+ Ranges = [{B, E} || {#shard{range = [B, E]}, _} <- Resps],
+ case mem3_util:get_ring(Ranges) of
+ [] -> {error, Error};
+ _ -> {ok, build_final_response(USet, Expected, Resps)}
+ end.
+
+build_final_response(USet, Expected, Resps) ->
+ ByRange = lists:foldl(
+ fun({#shard{range = Range, name = Id, node = Node}, Info}, Acc) ->
+ orddict:append(Range, {sets:is_element({Id, Node}, USet), Info}, Acc)
+ end,
+ orddict:new(),
+ Resps
+ ),
+ RangeCopies = [Copies || {_Range, Copies} <- ByRange],
+ Reps = [maps:remove(pending_updates, representative(Copies)) || Copies <- RangeCopies],
+ PerRange = [
+ [{Pref, maps:get(pending_updates, Info, undefined)} || {Pref, Info} <- Copies]
+ || Copies <- RangeCopies
+ ],
+ Merged = lists:foldl(
+ fun(Info, Acc) -> maps:merge_with(fun merge_info/3, Info, Acc) end,
+ #{},
+ Reps
+ ),
+ case fabric_util:aggregate_pending(PerRange, Expected) of
+ undefined -> Merged;
+ Pending -> Merged#{updates_pending => maps:from_list(Pending)}
+ end.
+
+% Prefer ushards if available first
+representative(Copies) ->
+ case [Info || {true, Info} <- Copies] of
+ [Info | _] ->
+ Info;
+ [] ->
+ [{_Pref, Info} | _] = Copies,
+ Info
end.
merge_info(signature, Val, Val) ->
Val;
merge_info(_Key, Val1, Val2) ->
Val1 + Val2.
+
+-ifdef(TEST).
+-include_lib("couch/include/couch_eunit.hrl").
+
+shard(Range, Node) ->
+ #shard{name = list_to_binary(io_lib:format("~p", [Range])), node = Node, range = Range}.
+
+info(Docs, Pending) ->
+ #{~"num_docs" => Docs, pending_updates => Pending, signature => ~"s"}.
+
+build_final_response_test() ->
+ R1 = [0, 10],
+ R2 = [11, 20],
+ R1N1 = shard(R1, n1),
+ R2N2 = shard(R2, n2),
+ USet = couch_util:set_from_list([
+ {R1N1#shard.name, n1}, {R2N2#shard.name, n2}
+ ]),
+ Resps = [
+ {R1N1, info(100, 0)},
+ {shard(R1, n2), info(94, 6)},
+ {R2N2, info(98, 2)},
+ {shard(R2, n3), info(96, 4)}
+ ],
+ Result = build_final_response(USet, 6, Resps),
+ ?assertEqual(198, maps:get(~"num_docs", Result)),
+ ?assertEqual(~"s", maps:get(signature, Result)),
+ ?assertNot(maps:is_key(pending_updates, Result)),
+ ?assertEqual(
+ #{
+ minimum => 2,
+ preferred => 2,
+ total => 12,
+ maximum => 10,
+ copies => 4,
+ copies_expected => 6
+ },
+ maps:get(updates_pending, Result)
+ ).
+
+-endif.
diff --git a/src/nouveau/src/nouveau_rpc.erl b/src/nouveau/src/nouveau_rpc.erl
index 2037c7e7e..08d808a12 100644
--- a/src/nouveau/src/nouveau_rpc.erl
+++ b/src/nouveau/src/nouveau_rpc.erl
@@ -77,13 +77,5 @@ update_and_retry(DbName, Index, QueryArgs, UpdateLatency) ->
rexi:reply(Else)
end.
-info(DbName, #index{} = Index0) ->
- %% Incorporate the shard name into the record.
- Index1 = Index0#index{dbname = DbName},
- case nouveau_api:index_info(Index1) of
- {ok, Info0} ->
- Info1 = Info0#{signature => Index0#index.sig},
- rexi:reply({ok, Info1});
- {error, Reason} ->
- rexi:reply({error, Reason})
- end.
+info(DbName, #index{} = Index) ->
+ rexi:reply(nouveau_util:index_info(DbName, Index)).
diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl
index 567b77f49..2374b6e74 100644
--- a/src/nouveau/src/nouveau_util.erl
+++ b/src/nouveau/src/nouveau_util.erl
@@ -22,6 +22,8 @@
index_name/1,
design_doc_to_indexes/2,
design_doc_to_index/3,
+ index_info/2,
+ index_info/3,
verify_index_exists/2,
ensure_local_purge_docs/2,
maybe_create_local_purge_doc/2,
@@ -40,6 +42,35 @@ index_name(#index{} = Index) ->
node_prefix() ->
atom_to_binary(node(), utf8).
+index_info(DbName, #index{} = Index0) ->
+ %% Incorporate the shard name into the record.
+ Index1 = Index0#index{dbname = DbName},
+ case nouveau_api:index_info(Index1) of
+ {ok, Info} ->
+ {ok, Info#{
+ signature => Index1#index.sig,
+ pending_updates => pending_updates(DbName, Info)
+ }};
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+index_info(DbName, DDoc, IndexName) ->
+ case design_doc_to_index(DbName, DDoc, IndexName) of
+ {ok, Index} ->
+ index_info(DbName, Index);
+ {error, Reason} ->
+ {error, Reason}
+ end.
+
+% Same idea as pending_updates in couch_index - how far ahead is committed seq
+% over the index seq.
+pending_updates(DbName, #{} = Info) ->
+ GetCommSeq = fun(Db) -> couch_db:get_committed_update_seq(Db) end,
+ CommittedSeq = couch_util:with_db(DbName, GetCommSeq),
+ IndexSeq = maps:get(<<"update_seq">>, Info, 0),
+ max(CommittedSeq - IndexSeq, 0).
+
%% copied from dreyfus_index.erl
design_doc_to_indexes(DbName, #doc{body = {Fields}} = Doc) ->
RawIndexes = couch_util:get_value(<<"nouveau">>, Fields, {[]}),
diff --git a/test/elixir/lib/couch/dbtest.ex b/test/elixir/lib/couch/dbtest.ex
index d221151a3..d3b309e8f 100644
--- a/test/elixir/lib/couch/dbtest.ex
+++ b/test/elixir/lib/couch/dbtest.ex
@@ -213,6 +213,13 @@ defmodule Couch.DBTest do
resp.body
end
+ # Get Q*N
+ def shard_copies(db_name) do
+ resp = Couch.get("/#{db_name}/_shards")
+ assert resp.status_code == 200
+ Enum.reduce(resp.body["shards"], 0, fn {_range, nodes}, acc -> acc + length(nodes) end)
+ end
+
def save(db_name, document) do
resp = Couch.put("/#{db_name}/#{document["_id"]}", body: document)
assert resp.status_code in [201, 202]
diff --git a/test/elixir/test/config/nouveau.elixir b/test/elixir/test/config/nouveau.elixir
index 7fd6ff195..ec3a21634 100644
--- a/test/elixir/test/config/nouveau.elixir
+++ b/test/elixir/test/config/nouveau.elixir
@@ -35,6 +35,7 @@
"index not found",
"index function throws",
"meta",
- "stale search"
+ "stale search",
+ "index info matches nouveau info"
]
}
diff --git a/test/elixir/test/config/search.elixir b/test/elixir/test/config/search.elixir
index eeeb7d7ae..6f4a53e79 100644
--- a/test/elixir/test/config/search.elixir
+++ b/test/elixir/test/config/search.elixir
@@ -26,6 +26,7 @@
"drilldown single key single value for GET",
"drilldown single key single value for POST",
"drilldown three keys single values for POST",
+ "search info",
"search returns all items for GET",
"search returns all items for POST",
"facet counts, non-empty",
@@ -33,6 +34,7 @@
"facet ranges, empty",
"facet ranges, non-empty",
"group search",
+ "index info matches search info",
"timeouts do not expose internal state"
],
"BasicTextTest": [
diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir
index 9d4ae53da..37ed47e15 100644
--- a/test/elixir/test/config/suite.elixir
+++ b/test/elixir/test/config/suite.elixir
@@ -193,6 +193,7 @@
"commonjs require",
"consistent _rev for design docs",
"design doc deletion",
+ "design doc info reports updates_pending across all shard copies",
"language not specified, Javascript is implied",
"module id values are as expected",
"startkey and endkey",
@@ -239,6 +240,12 @@
"location header should include X-Forwarded-Host",
"location header should include custom header"
],
+ "IndexInfoTest": [
+ "index info lists every index with its per index info object",
+ "index info pages design docs like _design_docs",
+ "index info reports mango indexes by the ddoc and name _index uses",
+ "index info takes keys like _design_docs and caps the design docs"
+ ],
"InvalidDocIDsTest": [
"_local-prefixed ids are illegal",
"a PUT request with absent _id is forbidden",
diff --git a/test/elixir/test/design_docs_test.exs b/test/elixir/test/design_docs_test.exs
index 8ef744bdf..5301d8cee 100644
--- a/test/elixir/test/design_docs_test.exs
+++ b/test/elixir/test/design_docs_test.exs
@@ -282,6 +282,46 @@ defmodule DesignDocsTest do
end
end
+ @tag :with_db
+ test "design doc info reports updates_pending across all shard copies", context do
+ db_name = context[:db_name]
+ {:ok, _} = create_doc(db_name, @design_doc)
+ bulk_save(db_name, make_docs(1..50))
+ copies = shard_copies(db_name)
+
+ # with autoupdate:false we don't query teh view for all copies are behind
+ resp = Couch.get("/#{db_name}/_design/test/_info")
+ assert resp.status_code == 200
+ pending = resp.body["view_index"]["updates_pending"]
+
+ assert Enum.sort(Map.keys(pending)) ==
+ ["copies", "copies_expected", "maximum", "minimum", "preferred", "total"]
+
+ assert pending["copies"] == copies
+ assert pending["copies_expected"] == copies
+ assert pending["minimum"] > 0
+ assert pending["minimum"] <= pending["preferred"]
+ assert pending["preferred"] <= pending["maximum"]
+ assert pending["maximum"] <= pending["total"]
+
+ # one query mean it will build on all copies which server the query
+ resp = Couch.get("/#{db_name}/_design/test/_view/summate")
+ assert resp.status_code == 200
+
+ retry_until(fn ->
+ resp = Couch.get("/#{db_name}/_design/test/_info")
+
+ resp.body["view_index"]["updates_pending"] == %{
+ "minimum" => 0,
+ "preferred" => 0,
+ "total" => 0,
+ "maximum" => 0,
+ "copies" => copies,
+ "copies_expected" => copies
+ }
+ end)
+ end
+
test "commonjs in map functions", context do
db_name = context[:db_name]
diff --git a/test/elixir/test/index_info_test.exs b/test/elixir/test/index_info_test.exs
new file mode 100644
index 000000000..e5ecac039
--- /dev/null
+++ b/test/elixir/test/index_info_test.exs
@@ -0,0 +1,391 @@
+defmodule IndexInfoTest do
+ use CouchTestCase
+
+ @moduletag :index_info
+
+ @moduledoc """
+ Test GET and POST /{db}/_index_info
+ """
+
+ @pending_keys ["copies", "copies_expected", "maximum", "minimum", "preferred", "total"]
+ @lists ["nouveau_indexes", "search_indexes", "view_indexes"]
+ # expected keys in every response
+ @keys ["bookmark", "nouveau_indexes", "search_indexes", "total_rows", "view_indexes"]
+
+ # A design doc with a view group, a lib, a search index and a nouveau
+ # index.
+ @cookbook %{
+ _id: "_design/cookbook",
+ language: "javascript",
+ autoupdate: false,
+ views: %{
+ by_integer: %{map: "function(doc) { if (doc.integer) emit(doc.integer, 1); }"},
+ by_string: %{
+ map: "function(doc) { if (doc.string) emit(doc.string, 1); }",
+ reduce: "_count"
+ },
+ lib: %{helper: "exports.x = 1;"}
+ },
+ indexes: %{
+ strings: %{index: "function(doc) { if (doc.string) index('string', doc.string); }"}
+ },
+ nouveau: %{
+ strings: %{
+ index: "function(doc) { if (doc.string) index('string', 'string', doc.string); }"
+ }
+ }
+ }
+
+ defp index_info(db_name, query \\ []) do
+ resp = Couch.get("/#{db_name}/_index_info", query: query)
+
+ assert resp.status_code == 200,
+ "Expected 200, got #{resp.status_code}: #{inspect(resp.body)}"
+
+ resp.body
+ end
+
+ defp view_group_ddocs(body), do: Enum.map(body["view_indexes"], & &1["ddoc"])
+
+ defp view_group(body, ddoc) do
+ [entry] = Enum.filter(body["view_indexes"], &(&1["ddoc"] == ddoc))
+ entry
+ end
+
+ defp ddoc_with_view(id) do
+ %{_id: id, views: %{all: %{map: "function(doc) { emit(doc._id, null); }"}}}
+ end
+
+ defp assert_meta_matches_design_docs(db_name, query \\ []) do
+ resp = Couch.get("/#{db_name}/_design_docs", query: query)
+ assert resp.status_code == 200
+ body = index_info(db_name, query)
+ assert body["total_rows"] == resp.body["total_rows"]
+ body
+ end
+
+ defp create_paging_ddocs(db_name) do
+ for id <- ["_design/a", "_design/b", "_design/c"] do
+ {:ok, _} = create_doc(db_name, ddoc_with_view(id))
+ end
+
+ {:ok, _} =
+ create_doc(db_name, %{
+ _id: "_design/b2",
+ filters: %{all: "function(doc) { return true; }"}
+ })
+ end
+
+ @tag :with_db
+ test "index info lists every index with its per index info object", context do
+ db_name = context[:db_name]
+ {:ok, _} = create_doc(db_name, @cookbook)
+ {:ok, _} = create_doc(db_name, ddoc_with_view("_design/zzz"))
+ bulk_save(db_name, make_docs(1..20))
+ copies = shard_copies(db_name)
+
+ # These are not available, we want to test that scenario
+ unavailable = [
+ %{section: "nouveau", key: "enable", value: "false"},
+ %{section: "dreyfus", key: "name", value: "clouseau_none@127.0.0.1"}
+ ]
+
+ run_on_modified_server(unavailable, fn ->
+ body = index_info(db_name)
+ assert Enum.sort(Map.keys(body)) == @keys
+ # two design docs, all returned, no next page => bookmark is null
+ assert body["total_rows"] == 2
+ assert body["bookmark"] == nil
+ assert_meta_matches_design_docs(db_name)
+
+ # one view group entry per design doc sorted by ddoc id
+ assert view_group_ddocs(body) == ["_design/cookbook", "_design/zzz"]
+ cookbook = view_group(body, "_design/cookbook")
+ assert Enum.sort(Map.keys(cookbook)) == ["ddoc", "info", "ok", "views"]
+ assert cookbook["ok"] == true
+ # the lib is not a view
+ assert cookbook["views"] == ["by_integer", "by_string"]
+
+ pending = cookbook["info"]["updates_pending"]
+ assert Enum.sort(Map.keys(pending)) == @pending_keys
+ assert pending["copies"] == copies
+ assert pending["copies_expected"] == copies
+ # autoupdate = false this should not have built yet
+ assert pending["minimum"] > 0
+
+ [search] = body["search_indexes"]
+ assert Enum.sort(Map.keys(search)) == ["ddoc", "error", "name", "ok", "reason"]
+ assert search["ddoc"] == "_design/cookbook"
+ assert search["name"] == "strings"
+ assert search["ok"] == false
+ assert search["error"] == "service unavailable"
+ assert search["reason"] == "Search is not available"
+
+ [nouveau] = body["nouveau_indexes"]
+ assert Enum.sort(Map.keys(nouveau)) == ["ddoc", "error", "name", "ok", "reason"]
+ assert nouveau["ddoc"] == "_design/cookbook"
+ assert nouveau["name"] == "strings"
+ assert nouveau["ok"] == false
+ assert nouveau["error"] == "service unavailable"
+ assert nouveau["reason"] == "nouveau is not enabled"
+
+ # wehn built, info is exactly _info produces
+ resp = Couch.get("/#{db_name}/_design/cookbook/_view/by_integer")
+ assert resp.status_code == 200
+
+ retry_until(fn ->
+ info = view_group(index_info(db_name), "_design/cookbook")["info"]
+ resp = Couch.get("/#{db_name}/_design/cookbook/_info")
+ assert resp.status_code == 200
+ info["updates_pending"]["maximum"] == 0 and info == resp.body["view_index"]
+ end)
+ end)
+ end
+
+ @tag :with_db
+ test "index info reports mango indexes by the ddoc and name _index uses", context do
+ db_name = context[:db_name]
+
+ resp =
+ Couch.post("/#{db_name}/_index",
+ body: %{index: %{fields: ["integer"]}, name: "by-integer", type: "json"}
+ )
+
+ assert resp.status_code == 200
+ assert resp.body["result"] == "created"
+ bulk_save(db_name, make_docs(1..5))
+
+ resp = Couch.get("/#{db_name}/_index")
+ assert resp.status_code == 200
+ [mango] = for idx <- resp.body["indexes"], idx["type"] == "json", do: idx
+
+ body = index_info(db_name)
+ assert body["search_indexes"] == []
+ assert body["nouveau_indexes"] == []
+ entry = view_group(body, mango["ddoc"])
+ assert entry["ok"] == true
+ assert entry["views"] == [mango["name"]]
+ assert entry["info"]["language"] == "query"
+ end
+
+ @tag :with_db
+ test "index info pages design docs like _design_docs", context do
+ db_name = context[:db_name]
+ create_paging_ddocs(db_name)
+
+ assert view_group_ddocs(index_info(db_name)) == [
+ "_design/a",
+ "_design/b",
+ "_design/c"
+ ]
+
+ assert view_group_ddocs(index_info(db_name, limit: 1)) == ["_design/a"]
+ assert view_group_ddocs(index_info(db_name, limit: 3)) == ["_design/a", "_design/b"]
+ assert view_group_ddocs(index_info(db_name, skip: 1, limit: 1)) == ["_design/b"]
+
+ b = :jiffy.encode("_design/b")
+
+ assert view_group_ddocs(index_info(db_name, startkey: b)) == [
+ "_design/b",
+ "_design/c"
+ ]
+
+ assert view_group_ddocs(index_info(db_name, start_key: b)) == [
+ "_design/b",
+ "_design/c"
+ ]
+
+ # continuing from the last ddoc seen, as with _all_docs
+ assert view_group_ddocs(index_info(db_name, startkey: b, skip: 1, limit: 1)) == []
+ assert view_group_ddocs(index_info(db_name, startkey: b, skip: 2)) == ["_design/c"]
+ assert view_group_ddocs(index_info(db_name, endkey: b)) == ["_design/a", "_design/b"]
+ assert view_group_ddocs(index_info(db_name, end_key: b)) == ["_design/a", "_design/b"]
+
+ assert view_group_ddocs(index_info(db_name, endkey: b, inclusive_end: false)) == [
+ "_design/a"
+ ]
+
+ assert view_group_ddocs(index_info(db_name, descending: true)) ==
+ ["_design/c", "_design/b", "_design/a"]
+
+ # the second design doc in reverse order is _design/b2, which has no indexes
+ assert view_group_ddocs(index_info(db_name, descending: true, limit: 2)) == [
+ "_design/c"
+ ]
+
+ assert view_group_ddocs(index_info(db_name, descending: true, limit: 3)) ==
+ ["_design/c", "_design/b"]
+
+ assert view_group_ddocs(index_info(db_name, descending: true, startkey: b)) ==
+ ["_design/b", "_design/a"]
+
+ # total_rows counts design docs, not index entries, and is the number
+ # _design_docs reports for the same parameters
+ body = assert_meta_matches_design_docs(db_name)
+ assert body["total_rows"] == 4
+ assert length(body["view_indexes"]) == 3
+
+ for query <- [[limit: 1], [skip: 1, limit: 1], [startkey: b], [descending: true]] do
+ assert_meta_matches_design_docs(db_name, query)
+ end
+
+ # bookmark is the design doc after the page. Even if we have an empty ddoc
+ # with no indexes we still get its ID as the bookmark so we can continue
+ # paginating.
+ page1 = index_info(db_name, limit: 2)
+ assert view_group_ddocs(page1) == ["_design/a", "_design/b"]
+ assert page1["bookmark"] == "_design/b2"
+ page2 = index_info(db_name, limit: 2, startkey: :jiffy.encode(page1["bookmark"]))
+ assert view_group_ddocs(page2) == ["_design/c"]
+ assert page2["bookmark"] == nil
+
+ # the documented paging loop: pass each bookmark back as the startkey
+ # until it is null
+ seen =
+ Enum.reduce_while(Stream.cycle([nil]), {[], nil}, fn _, {acc, bookmark} ->
+ query =
+ if bookmark, do: [limit: 1, startkey: :jiffy.encode(bookmark)], else: [limit: 1]
+
+ page = index_info(db_name, query)
+ acc = acc ++ view_group_ddocs(page)
+
+ case page["bookmark"] do
+ nil -> {:halt, acc}
+ next -> {:cont, {acc, next}}
+ end
+ end)
+
+ assert seen == ["_design/a", "_design/b", "_design/c"]
+
+ # limit 0 bookmarks the first design doc, and the bookmark follows
+ # descending order too
+ assert index_info(db_name, limit: 0)["bookmark"] == "_design/a"
+ desc = index_info(db_name, descending: true, limit: 2)
+ assert view_group_ddocs(desc) == ["_design/c"]
+ assert desc["bookmark"] == "_design/b"
+
+ desc_startkey = index_info(db_name, descending: true, limit: 2, startkey: b)
+ assert view_group_ddocs(desc_startkey) == ["_design/b", "_design/a"]
+
+ # all three lists are always present, only the requested types are gathered
+ body = index_info(db_name, type: "view")
+ assert Enum.sort(Map.keys(body)) == @keys
+ assert length(body["view_indexes"]) == 3
+ assert body["search_indexes"] == []
+ assert body["nouveau_indexes"] == []
+ body = index_info(db_name, type: "search,nouveau")
+ assert Enum.sort(Map.keys(body)) == @keys
+ assert body["view_indexes"] == []
+
+ resp = Couch.get("/#{db_name}/_index_info", query: [type: "foo"])
+ assert resp.status_code == 400
+ assert resp.body["error"] == "query_parse_error"
+
+ resp = Couch.get("/#{db_name}/_index_info", query: [limit: "abc"])
+ assert resp.status_code == 400
+ assert resp.body["error"] == "query_parse_error"
+
+ assert Couch.put("/#{db_name}/_index_info", body: %{}).status_code == 405
+ assert Couch.get("/#{db_name}/_index_info/extra").status_code == 404
+ assert Couch.get("/#{db_name}_missing/_index_info").status_code == 404
+ end
+
+ @tag :with_db
+ test "index info takes keys like _design_docs and caps the design docs", context do
+ db_name = context[:db_name]
+ create_paging_ddocs(db_name)
+
+ resp =
+ Couch.post("/#{db_name}/_index_info",
+ body: %{keys: ["_design/c", "_design/a", "_design/nope"]}
+ )
+
+ assert resp.status_code == 200
+ assert Enum.sort(Map.keys(resp.body)) == @keys
+ assert view_group_ddocs(resp.body) == ["_design/a", "_design/c"]
+ assert resp.body["total_rows"] == 4
+ assert resp.body["bookmark"] == nil
+
+ keys = %{keys: ["_design/c", "_design/a", "_design/nope"]}
+ ref = Couch.post("/#{db_name}/_design_docs", body: keys)
+ assert ref.status_code == 200
+ assert resp.body["total_rows"] == ref.body["total_rows"]
+
+ # keys and a startkey is a 400 error
+ query = [startkey: :jiffy.encode("_design/a")]
+ body = %{keys: ["_design/b"]}
+ resp = Couch.post("/#{db_name}/_index_info", query: query, body: body)
+ ref = Couch.post("/#{db_name}/_design_docs", query: query, body: body)
+ assert resp.status_code == 400
+ assert {resp.status_code, resp.body} == {ref.status_code, ref.body}
+
+ # the query parameters apply to POST too
+ resp =
+ Couch.post("/#{db_name}/_index_info",
+ query: [type: "search"],
+ body: %{keys: ["_design/a"]}
+ )
+
+ assert resp.status_code == 200
+ assert resp.body["view_indexes"] == []
+ assert resp.body["search_indexes"] == []
+ assert resp.body["nouveau_indexes"] == []
+ assert resp.body["total_rows"] == 4
+
+ # and keys work as a query parameter, as for _design_docs
+ assert view_group_ddocs(index_info(db_name, keys: :jiffy.encode(["_design/b"]))) == [
+ "_design/b"
+ ]
+
+ resp = Couch.post("/#{db_name}/_index_info", body: %{keys: ["_design/b"]})
+ assert resp.status_code == 200
+ assert view_group_ddocs(resp.body) == ["_design/b"]
+ assert resp.body["bookmark"] == nil
+
+ # other _design_docs params are ok, too
+ resp = Couch.post("/#{db_name}/_index_info", body: %{})
+ assert resp.status_code == 200
+ assert resp.body == index_info(db_name)
+ resp = Couch.post("/#{db_name}/_index_info", body: %{limit: 1, descending: true})
+ assert resp.status_code == 200
+ assert resp.body == index_info(db_name, limit: 1, descending: true)
+
+ resp = Couch.post("/#{db_name}/_index_info", body: %{keys: "_design/a"})
+ assert resp.status_code == 400
+ assert resp.body["reason"] == "`keys` member must be an array."
+
+ # a request may cover at most max_ddoc_number_for_index_info_req design
+ # docs, ones without indexes included; page or send fewer keys
+ cap = [%{section: "chttpd", key: "max_ddoc_number_for_index_info_req", value: "2"}]
+
+ run_on_modified_server(cap, fn ->
+ resp = Couch.get("/#{db_name}/_index_info")
+ assert resp.status_code == 400
+ assert resp.body["error"] == "bad_request"
+ assert resp.body["reason"] == "too_many_design_docs"
+
+ page = index_info(db_name, limit: 2)
+ assert view_group_ddocs(page) == ["_design/a", "_design/b"]
+ assert page["bookmark"] == "_design/b2"
+ next = :jiffy.encode(page["bookmark"])
+
+ assert view_group_ddocs(index_info(db_name, limit: 2, startkey: next)) == [
+ "_design/c"
+ ]
+
+ resp =
+ Couch.post("/#{db_name}/_index_info",
+ body: %{keys: ["_design/a", "_design/b", "_design/c"]}
+ )
+
+ assert resp.status_code == 400
+ assert resp.body["reason"] == "too_many_keys"
+
+ resp =
+ Couch.post("/#{db_name}/_index_info", body: %{keys: ["_design/a", "_design/c"]})
+
+ assert resp.status_code == 200
+ assert view_group_ddocs(resp.body) == ["_design/a", "_design/c"]
+ end)
+ end
+end
diff --git a/test/elixir/test/nouveau_test.exs b/test/elixir/test/nouveau_test.exs
index 609877b7b..8aba9f1d5 100644
--- a/test/elixir/test/nouveau_test.exs
+++ b/test/elixir/test/nouveau_test.exs
@@ -177,6 +177,53 @@ defmodule NouveauTest do
assert Map.get(info, "num_docs") > 0
assert Map.get(info, "update_seq") > 0
assert String.length(Map.get(info, "signature")) > 0
+
+ pending = Map.get(info, "updates_pending")
+
+ assert Enum.sort(Map.keys(pending)) ==
+ ["copies", "copies_expected", "maximum", "minimum", "preferred", "total"]
+
+ copies = shard_copies(db_name)
+ assert pending["copies"] == copies
+ assert pending["copies_expected"] == copies
+
+ # after the query builds it we should have 0 for total and maximum
+ retry_until(fn ->
+ resp = Couch.get(url)
+ assert_status_code(resp, 200)
+ pending = resp.body["search_index"]["updates_pending"]
+ pending["maximum"] == 0 and pending["total"] == 0
+ end)
+ end
+
+ @tag :with_db
+ test "index info matches nouveau info", context do
+ db_name = context[:db_name]
+ create_search_docs(db_name)
+ create_ddoc(db_name)
+
+ # this builds the index
+ url = "/#{db_name}/_design/foo/_nouveau/bar"
+ resp = Couch.get(url, query: %{q: "*:*"})
+ assert_status_code(resp, 200)
+
+ # consistency assert that _index_info is the same as _nouveau_info
+ retry_until(fn ->
+ resp = Couch.get("/#{db_name}/_index_info", query: %{type: "nouveau"})
+ assert_status_code(resp, 200)
+ # we filter by typo, other types should be []
+ assert resp.body["view_indexes"] == []
+ assert resp.body["search_indexes"] == []
+ [entry] = resp.body["nouveau_indexes"]
+ assert entry["ddoc"] == "_design/foo"
+ assert entry["name"] == "bar"
+ assert entry["ok"] == true
+
+ resp = Couch.get("/#{db_name}/_design/foo/_nouveau_info/bar")
+ assert_status_code(resp, 200)
+ info = entry["info"]
+ info["updates_pending"]["maximum"] == 0 and info == resp.body["search_index"]
+ end)
end
@tag :with_db
diff --git a/test/elixir/test/search_test.exs b/test/elixir/test/search_test.exs
index 526cfda0d..f945d0754 100644
--- a/test/elixir/test/search_test.exs
+++ b/test/elixir/test/search_test.exs
@@ -80,6 +80,69 @@ defmodule SearchTest do
assert Enum.sort(ids) == Enum.sort(["apple", "banana", "carrot", "date"])
end
+ @tag :with_db
+ test "search info", context do
+ db_name = context[:db_name]
+ create_search_docs(db_name)
+ create_ddoc(db_name)
+
+ # this builds the index
+ url = "/#{db_name}/_design/inventory/_search/fruits"
+ resp = Couch.get(url, query: %{q: "*:*"})
+ assert_on_status(resp, 200, "Fail to do search.")
+
+ url = "/#{db_name}/_design/inventory/_search_info/fruits"
+ resp = Couch.get(url)
+ assert_on_status(resp, 200, "Fail to get search info.")
+ info = resp.body["search_index"]
+ assert info["doc_count"] == 4
+ assert String.length(info["signature"]) > 0
+
+ pending = info["updates_pending"]
+
+ assert Enum.sort(Map.keys(pending)) ==
+ ["copies", "copies_expected", "maximum", "minimum", "preferred", "total"]
+
+ copies = shard_copies(db_name)
+ assert pending["copies"] == copies
+ assert pending["copies_expected"] == copies
+
+ # after we build the index max and total are both 0
+ retry_until(fn ->
+ pending = Couch.get(url).body["search_index"]["updates_pending"]
+ pending["maximum"] == 0 and pending["total"] == 0
+ end)
+ end
+
+ @tag :with_db
+ test "index info matches search info", context do
+ db_name = context[:db_name]
+ create_search_docs(db_name)
+ create_ddoc(db_name)
+
+ url = "/#{db_name}/_design/inventory/_search/fruits"
+ resp = Couch.get(url, query: %{q: "*:*"})
+ assert_on_status(resp, 200, "Fail to do search.")
+
+ # assert _index_info "info" is the same as _search_info
+ retry_until(fn ->
+ resp = Couch.get("/#{db_name}/_index_info", query: %{type: "search"})
+ assert_on_status(resp, 200, "Fail to get index info.")
+ # only the requested type is gathered, the other lists are empty
+ assert resp.body["view_indexes"] == []
+ assert resp.body["nouveau_indexes"] == []
+ [entry] = resp.body["search_indexes"]
+ assert entry["ddoc"] == "_design/inventory"
+ assert entry["name"] == "fruits"
+ assert entry["ok"] == true
+
+ resp = Couch.get("/#{db_name}/_design/inventory/_search_info/fruits")
+ assert_on_status(resp, 200, "Fail to get search info.")
+ info = entry["info"]
+ info["updates_pending"]["maximum"] == 0 and info == resp.body["search_index"]
+ end)
+ end
+
@tag :with_db
test "drilldown single key single value for GET", context do
db_name = context[:db_name]
(couchdb) 05/05: remove unused field
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 38b3db862b626929075fd8354f7ee0ab6a00f9f8
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Sep 29 19:51:59 2026 +0100
remove unused field
---
src/nouveau/src/nouveau_index_updater.erl | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl
index 32bf6ab0e..8d3c0a909 100644
--- a/src/nouveau/src/nouveau_index_updater.erl
+++ b/src/nouveau/src/nouveau_index_updater.erl
@@ -41,7 +41,6 @@
-record(purge_acc, {
index,
exclude_list = [],
- index_update_seq,
index_purge_seq,
batch_size,
batch
@@ -89,7 +88,6 @@ update(#index{} = Index) ->
PurgeAcc0 = #purge_acc{
index = Index,
- index_update_seq = IndexUpdateSeq,
index_purge_seq = IndexPurgeSeq,
batch_size = config:get_integer("nouveau", "batch_size", 20),
batch = []
@@ -108,7 +106,7 @@ update(#index{} = Index) ->
changes_done = 0,
total_changes = TotalChanges,
exclude_idrevs = PurgeAcc1#purge_acc.exclude_list,
- update_seq = PurgeAcc1#purge_acc.index_update_seq,
+ update_seq = IndexUpdateSeq,
batch_size = config:get_integer("nouveau", "batch_size", 20),
batch = []
},
(couchdb) 03/05: if purged doc is recreated, just add it to the exclusion list
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-purge-fix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f3be3a0e89284a82bf9229a777d2b6ec07ba39d5
Author: Robert Newson <rn...@apache.org>
AuthorDate: Tue Sep 29 19:38:48 2026 +0100
if purged doc is recreated, just add it to the exclusion list
---
src/nouveau/src/nouveau_index_updater.erl | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl
index 38e284a95..290ce9e7f 100644
--- a/src/nouveau/src/nouveau_index_updater.erl
+++ b/src/nouveau/src/nouveau_index_updater.erl
@@ -266,18 +266,13 @@ purge_index(Db, Index, #purge_acc{} = PurgeAcc0) ->
PurgeAcc2#purge_acc{index_purge_seq = PurgeSeq};
FDI ->
DI = couch_doc:to_doc_info(FDI),
- #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{rev = Rev} | _]} = DI,
+ #doc_info{id = Id, revs = [#rev_info{rev = Rev} | _]} = DI,
case lists:member({Id, Rev}, PurgeAcc1#purge_acc.exclude_list) of
true ->
PurgeAcc1;
false ->
- Item = update_or_delete_index(
- Db, PurgeAcc1#purge_acc.index_update_seq, DI, Proc
- ),
PurgeAcc1#purge_acc{
- batch = [Item | PurgeAcc1#purge_acc.batch],
- exclude_list = [{Id, Rev} | PurgeAcc1#purge_acc.exclude_list],
- index_update_seq = Seq
+ exclude_list = [{Id, Rev} | PurgeAcc1#purge_acc.exclude_list]
}
end
end,