You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2019/07/23 20:13:33 UTC
[couchdb] 24/25: Add total row count support
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/views
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 81521e1e621dd5ca17a27d9c7d3893e9eb4224e7
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Tue Jul 23 15:12:31 2019 -0500
Add total row count support
---
src/couch_views/include/couch_views.hrl | 16 ++++----
src/couch_views/src/couch_views_fdb.erl | 50 ++++++++++++++++++++---
src/couch_views/src/couch_views_reader.erl | 9 ++--
src/couch_views/test/couch_views_indexer_test.erl | 4 +-
4 files changed, 59 insertions(+), 20 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 4fcc57e..525f62f 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -11,15 +11,15 @@
% the License.
% indexing
--define(VIEW_UPDATE_SEQ, 1).
--define(VIEW_ID_RANGE, 2).
--define(VIEW_MAP_RANGE, 3).
--define(VIEW_BUILDS, 4).
--define(VIEW_STATUS, 5).
--define(VIEW_WATCH, 6).
+-define(VIEW_UPDATE_SEQ, 0).
+-define(VIEW_ID_INFO, 1).
+-define(VIEW_ID_RANGE, 2).
+-define(VIEW_MAP_RANGE, 3).
--define(VIEW_ROW_KEY, 0).
--define(VIEW_ROW_VALUE, 1).
+-define(VIEW_ROW_COUNT, 0).
+
+-define(VIEW_ROW_KEY, 0).
+-define(VIEW_ROW_VALUE, 1).
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 09a9802..a0e4bd1 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -16,6 +16,8 @@
get_update_seq/2,
set_update_seq/3,
+ get_row_count/3,
+
fold_map_idx/6,
write_doc/4
@@ -59,6 +61,18 @@ set_update_seq(TxDb, Sig, Seq) ->
ok = erlfdb:set(Tx, seq_key(DbPrefix, Sig), Seq).
+get_row_count(TxDb, #mrst{sig = Sig}, ViewId) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+
+ case erlfdb:wait(erlfdb:get(Tx, row_count_key(DbPrefix, Sig, ViewId))) of
+ not_found -> 0; % Can this happen?
+ CountBin -> ?bin2uint(CountBin)
+ end.
+
+
fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
#{
db_prefix := DbPrefix
@@ -107,8 +121,9 @@ write_doc(TxDb, Sig, _ViewIds, #{deleted := true} = Doc) ->
ExistingViewKeys = get_view_keys(TxDb, Sig, DocId),
clear_id_idx(TxDb, Sig, DocId),
- lists:foreach(fun({ViewId, ViewKeys}) ->
- clear_map_idx(TxDb, Sig, ViewId, DocId, ViewKeys)
+ lists:foreach(fun({ViewId, TotalKeys, UniqueKeys}) ->
+ clear_map_idx(TxDb, Sig, ViewId, DocId, UniqueKeys),
+ update_row_count(TxDb, Sig, ViewId, -TotalKeys)
end, ExistingViewKeys);
write_doc(TxDb, Sig, ViewIds, Doc) ->
@@ -122,8 +137,17 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
clear_id_idx(TxDb, Sig, DocId),
lists:foreach(fun({ViewId, NewRows}) ->
- ExistingKeys = fabric2_util:get_value(ViewId, ExistingViewKeys, []),
update_id_idx(TxDb, Sig, ViewId, DocId, NewRows),
+
+ ExistingKeys = case lists:keyfind(ViewId, 1, ExistingViewKeys) of
+ {ViewId, TotalRows, EKeys} ->
+ Change = length(NewRows) - TotalRows,
+ update_row_count(TxDb, Sig, ViewId, Change),
+ EKeys;
+ false ->
+ update_row_count(TxDb, Sig, ViewId, length(NewRows)),
+ []
+ end,
update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows)
end, lists:zip(ViewIds, Results)).
@@ -251,7 +275,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows) ->
couch_log:error("Updating ID index: ~p ~p ~p ~p", [ViewId, DocId, NewRows, Unique]),
Key = id_idx_key(DbPrefix, Sig, DocId, ViewId),
- Val = couch_views_encoding:encode(Unique),
+ Val = couch_views_encoding:encode([length(NewRows), Unique]),
ok = erlfdb:set(Tx, Key, Val).
@@ -289,16 +313,30 @@ get_view_keys(TxDb, Sig, DocId) ->
lists:map(fun({K, V}) ->
{?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId} =
erlfdb_tuple:unpack(K, DbPrefix),
- ViewKeys = couch_views_encoding:decode(V),
- {ViewId, ViewKeys}
+ [TotalKeys, UniqueKeys] = couch_views_encoding:decode(V),
+ {ViewId, TotalKeys, UniqueKeys}
end, erlfdb:get_range(Tx, Start, End, [])).
+update_row_count(TxDb, Sig, ViewId, Increment) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ Key = row_count_key(DbPrefix, Sig, ViewId),
+ erlfdb:add(Tx, Key, Increment).
+
+
seq_key(DbPrefix, Sig) ->
Key = {?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ},
erlfdb_tuple:pack(Key, DbPrefix).
+row_count_key(DbPrefix, Sig, ViewId) ->
+ Key = {?DB_VIEWS, Sig, ?VIEW_ID_INFO, ViewId, ?VIEW_ROW_COUNT},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
Key = {?DB_VIEWS, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index ce5097ba..8bbe1fc 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -34,11 +34,12 @@ read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) ->
Fun = fun handle_row/4,
try
- % Need to add total_rows support
- Meta = {meta, [{total_rows, null}, {offset, null}]},
- UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
-
fabric2_fdb:transactional(Db, fun(TxDb) ->
+ TotalRows = couch_views_fdb:get_row_count(TxDb, Mrst, ViewId),
+
+ Meta = {meta, [{total, TotalRows}, {offset, null}]},
+ UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
+
Acc0 = #{
db => TxDb,
skip => Args#mrargs.skip,
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index e6dfdc4..fa0d99e 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -157,7 +157,7 @@ updated_docs_are_reindexed(Db) ->
DbName = fabric2_db:name(Db),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
Sig = Mrst#mrst.sig,
- Expect = [{0, [1]}, {1, []}],
+ Expect = [{0, 1, [1]}, {1, 0, []}],
fabric2_fdb:transactional(Db, fun(TxDb) ->
?assertEqual(
Expect,
@@ -214,7 +214,7 @@ updated_docs_without_changes_are_reindexed(Db) ->
DbName = fabric2_db:name(Db),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
Sig = Mrst#mrst.sig,
- Expect = [{0, [0]}, {1, []}],
+ Expect = [{0, 1, [0]}, {1, 0, []}],
fabric2_fdb:transactional(Db, fun(TxDb) ->
?assertEqual(
Expect,