You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/09/24 15:59:14 UTC
[couchdb] branch feature-ebtree-views-2 created (now 7701f2c)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 7701f2c Add test suite for reduce views
This branch includes the following new commits:
new fa3c3c8 Calculate external JSON size of a view row
new 3b508e0 Export fabric2_fdb:chunkify_binary/1,2
new 1f773fb Views on ebtree
new ba2a426 Reimplement db wide view size tracking
new 38ccbe1 Update legacy views
new 89c175d Use ebtree for reduce functions
new 7701f2c Add test suite for reduce views
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 05/07: Update legacy views
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 38ccbe1976557bccdf5aac684eebb497e4136b87
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Sep 18 11:05:00 2020 -0500
Update legacy views
---
src/couch_views/include/couch_views.hrl | 4 +
src/couch_views/src/couch_views_fdb.erl | 180 ++++++++--
src/couch_views/src/couch_views_indexer.erl | 8 +-
src/couch_views/test/couch_views_upgrade_test.erl | 400 ++++++++++++++++++++++
4 files changed, 551 insertions(+), 41 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 3882191..92b8f46 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -10,6 +10,9 @@
% License for the specific language governing permissions and limitations under
% the License.
+% Current implementation version
+-define(CURRENT_VIEW_IMPL_VERSION, 1).
+
% Index info/data subspaces
-define(VIEW_INFO, 0).
-define(VIEW_DATA, 1).
@@ -21,6 +24,7 @@
-define(VIEW_KV_SIZE, 2).
-define(VIEW_BUILD_STATUS, 3).
-define(VIEW_CREATION_VS, 4).
+-define(VIEW_IMPL_VERSION, 5).
% Data keys
-define(VIEW_ID_RANGE, 0).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index a9dbe3e..27ef386 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -13,6 +13,8 @@
-module(couch_views_fdb).
-export([
+ get_view_state/2,
+
new_interactive_index/3,
new_creation_vs/3,
get_creation_vs/2,
@@ -39,52 +41,89 @@
-include_lib("fabric/include/fabric2.hrl").
-new_interactive_index(Db, Mrst, VS) ->
- couch_views_fdb:new_creation_vs(Db, Mrst, VS),
- couch_views_fdb:set_build_status(Db, Mrst, ?INDEX_BUILDING).
+get_view_state(Db, #mrst{} = Mrst) ->
+ get_view_state(Db, Mrst#mrst.sig);
+
+get_view_state(Db, Sig) when is_binary(Sig) ->
+ #{
+ tx := Tx
+ } = Db,
+
+ VersionF = erlfdb:get(Tx, version_key(Db, Sig)),
+ ViewSeqF = erlfdb:get(Tx, seq_key(Db, Sig)),
+ ViewVSF = erlfdb:get(Tx, creation_vs_key(Db, Sig)),
+ BuildStatusF = erlfdb:get(Tx, build_status_key(Db, Sig)),
+
+ Version = case erlfdb:wait(VersionF) of
+ not_found -> not_found;
+ VsnVal -> element(1, erlfdb_tuple:unpack(VsnVal))
+ end,
+
+ ViewSeq = case erlfdb:wait(ViewSeqF) of
+ not_found -> <<>>;
+ SeqVal -> SeqVal
+ end,
+
+ ViewVS = case erlfdb:wait(ViewVSF) of
+ not_found -> not_found;
+ VSVal -> element(1, erlfdb_tuple:unpack(VSVal))
+ end,
+
+ State = #{
+ version => Version,
+ view_seq => ViewSeq,
+ view_vs => ViewVS,
+ build_status => erlfdb:wait(BuildStatusF)
+ },
+
+ maybe_upgrade_view(Db, Sig, State).
+
+
+new_interactive_index(Db, #mrst{} = Mrst, VS) ->
+ new_interactive_index(Db, Mrst#mrst.sig, VS);
+
+new_interactive_index(Db, Sig, VS) ->
+ set_version(Db, Sig),
+ new_creation_vs(Db, Sig, VS),
+ set_build_status(Db, Sig, ?INDEX_BUILDING).
%Interactive View Creation Versionstamp
%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig) = VS
new_creation_vs(TxDb, #mrst{} = Mrst, VS) ->
+ new_creation_vs(TxDb, Mrst#mrst.sig, VS);
+
+new_creation_vs(TxDb, Sig, VS) ->
#{
tx := Tx
} = TxDb,
- Key = creation_vs_key(TxDb, Mrst#mrst.sig),
+ Key = creation_vs_key(TxDb, Sig),
Value = erlfdb_tuple:pack_vs({VS}),
ok = erlfdb:set_versionstamped_value(Tx, Key, Value).
-get_creation_vs(TxDb, #mrst{} = Mrst) ->
- get_creation_vs(TxDb, Mrst#mrst.sig);
-
-get_creation_vs(TxDb, Sig) ->
+get_creation_vs(TxDb, MrstOrSig) ->
#{
- tx := Tx
- } = TxDb,
- Key = creation_vs_key(TxDb, Sig),
- case erlfdb:wait(erlfdb:get(Tx, Key)) of
- not_found ->
- not_found;
- EK ->
- {VS} = erlfdb_tuple:unpack(EK),
- VS
- end.
+ view_vs := ViewVS
+ } = get_view_state(TxDb, MrstOrSig),
+ ViewVS.
%Interactive View Build Status
%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig) = INDEX_BUILDING | INDEX_READY
-get_build_status(TxDb, #mrst{sig = Sig}) ->
+get_build_status(TxDb, MrstOrSig) ->
#{
- tx := Tx
- } = TxDb,
- Key = build_status_key(TxDb, Sig),
- erlfdb:wait(erlfdb:get(Tx, Key)).
+ build_status := BuildStatus
+ } = get_view_state(TxDb, MrstOrSig),
+ BuildStatus.
-set_build_status(TxDb, #mrst{sig = Sig}, State) ->
+set_build_status(TxDb, #mrst{} = Mrst, State) ->
+ set_build_status(TxDb, Mrst#mrst.sig, State);
+
+set_build_status(TxDb, Sig, State) ->
#{
tx := Tx
} = TxDb,
@@ -97,24 +136,18 @@ set_build_status(TxDb, #mrst{sig = Sig}, State) ->
% (<db>, ?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ) = Sequence
-get_update_seq(TxDb, #mrst{sig = Sig}) ->
+get_update_seq(TxDb, MrstOrSig) ->
#{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- case erlfdb:wait(erlfdb:get(Tx, seq_key(DbPrefix, Sig))) of
- not_found -> <<>>;
- UpdateSeq -> UpdateSeq
- end.
+ view_seq := ViewSeq
+ } = get_view_state(TxDb, MrstOrSig),
+ ViewSeq.
set_update_seq(TxDb, Sig, Seq) ->
#{
- tx := Tx,
- db_prefix := DbPrefix
+ tx := Tx
} = TxDb,
- ok = erlfdb:set(Tx, seq_key(DbPrefix, Sig), Seq).
+ ok = erlfdb:set(Tx, seq_key(TxDb, Sig), Seq).
list_signatures(Db) ->
@@ -138,7 +171,10 @@ clear_index(Db, Signature) ->
% Get view size to remove from global counter
SizeTuple = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Signature},
SizeKey = erlfdb_tuple:pack(SizeTuple, DbPrefix),
- ViewSize = ?bin2uint(erlfdb:wait(erlfdb:get(Tx, SizeKey))),
+ ViewSize = case erlfdb:wait(erlfdb:get(Tx, SizeKey)) of
+ not_found -> 0;
+ SizeVal -> ?bin2uint(SizeVal)
+ end,
% Clear index info keys
Keys = [
@@ -206,11 +242,79 @@ update_kv_size(TxDb, Sig, OldSize, NewSize) ->
erlfdb:add(Tx, DbKey, NewSize - OldSize).
-seq_key(DbPrefix, Sig) ->
+maybe_upgrade_view(_Db, _Sig, #{version := ?CURRENT_VIEW_IMPL_VERSION} = St) ->
+ St;
+maybe_upgrade_view(Db, Sig, #{version := not_found, view_seq := <<>>} = St) ->
+ % If we haven't started building the view yet
+ % then we don't change view_vs and build_status
+ % as they're still correct.
+ set_version(Db, Sig),
+ St#{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>
+ };
+maybe_upgrade_view(Db, Sig, #{version := not_found} = St) ->
+ clear_index(Db, Sig),
+ set_version(Db, Sig),
+ {ViewVS, BuildStatus} = reset_interactive_index(Db, Sig, St),
+ #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>,
+ view_vs => ViewVS,
+ build_status => BuildStatus
+ }.
+
+
+set_version(Db, Sig) ->
+ #{
+ tx := Tx
+ } = Db,
+ Key = version_key(Db, Sig),
+ Val = erlfdb_tuple:pack({?CURRENT_VIEW_IMPL_VERSION}),
+ erlfdb:set(Tx, Key, Val).
+
+
+reset_interactive_index(_Db, _Sig, #{view_vs := not_found}) ->
+ % Not an interactive index
+ {not_found, not_found};
+reset_interactive_index(Db, Sig, _St) ->
+ % We have to reset the creation versionstamp
+ % to the current update seq of the database
+ % or else we'll not have indexed any documents
+ % inserted since the creation of the interactive
+ % index.
+ #{
+ tx := Tx
+ } = Db,
+
+ DbSeq = fabric2_db:get_update_seq(Db),
+ VS = fabric2_fdb:seq_to_vs(DbSeq),
+ Key = creation_vs_key(Db, Sig),
+ Val = erlfdb_tuple:pack({VS}),
+ ok = erlfdb:set(Tx, Key, Val),
+
+ set_build_status(Db, Sig, ?INDEX_BUILDING),
+
+ {VS, ?INDEX_BUILDING}.
+
+
+
+seq_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Sig},
erlfdb_tuple:pack(Key, DbPrefix).
+version_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_IMPL_VERSION, Sig},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
creation_vs_key(Db, Sig) ->
#{
db_prefix := DbPrefix
diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index c523c4d..f03f84e 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -202,8 +202,8 @@ do_update(Db, Mrst0, State0) ->
tx := Tx
} = TxDb,
- Mrst1 = couch_views_trees:open(TxDb, Mrst0),
State1 = get_update_start_state(TxDb, Mrst0, State0),
+ Mrst1 = couch_views_trees:open(TxDb, Mrst0),
{ok, State2} = fold_changes(State1),
@@ -261,8 +261,10 @@ maybe_set_build_status(TxDb, Mrst1, _ViewVS, State) ->
% In the first iteration of update we need
% to populate our db and view sequences
get_update_start_state(TxDb, Mrst, #{db_seq := undefined} = State) ->
- ViewVS = couch_views_fdb:get_creation_vs(TxDb, Mrst),
- ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
+ #{
+ view_vs := ViewVS,
+ view_seq := ViewSeq
+ } = couch_views_fdb:get_view_state(TxDb, Mrst),
State#{
tx_db := TxDb,
diff --git a/src/couch_views/test/couch_views_upgrade_test.erl b/src/couch_views/test/couch_views_upgrade_test.erl
new file mode 100644
index 0000000..0766f53
--- /dev/null
+++ b/src/couch_views/test/couch_views_upgrade_test.erl
@@ -0,0 +1,400 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_upgrade_test).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+-define(MAP_FUN1, <<"map_fun1">>).
+-define(MAP_FUN2, <<"map_fun2">>).
+
+
+indexer_test_() ->
+ {
+ "Test view indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [
+ ?TDEF_FE(empty_state),
+ ?TDEF_FE(indexed_state),
+ ?TDEF_FE(upgrade_non_interactive),
+ ?TDEF_FE(upgrade_unbuilt_interactive),
+ ?TDEF_FE(upgrade_partially_built_interactive),
+ ?TDEF_FE(upgrade_built_interactive)
+ ]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ Db.
+
+
+foreach_teardown(Db) ->
+ meck:unload(),
+ config:delete("couch_views", "change_limit"),
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+empty_state(Db) ->
+ DDoc = create_ddoc(),
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ State = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ couch_views_fdb:get_view_state(TxDb, Mrst)
+ end),
+
+ Expect = #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>,
+ view_vs => not_found,
+ build_status => not_found
+ },
+ ?assertEqual(Expect, State),
+ assert_fdb_state(Db, Mrst, Expect).
+
+
+indexed_state(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_doc(Db, Doc1, []),
+
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([row(<<"0">>, 0, 0)], Out),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => fabric2_db:get_update_seq(Db),
+ view_vs => not_found,
+ build_status => not_found
+ }).
+
+
+upgrade_non_interactive(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_docs(Db, [DDoc, Doc1], []),
+ DbSeq = fabric2_db:get_update_seq(Db),
+
+ init_fdb_state(Db, DDoc, #{view_seq => DbSeq}),
+
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([row(<<"0">>, 0, 0)], Out),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => DbSeq,
+ view_vs => not_found,
+ build_status => not_found
+ }).
+
+
+upgrade_unbuilt_interactive(Db) ->
+ DDoc = create_ddoc(),
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_docs(Db, [DDoc, Doc1], []),
+ DbSeq = fabric2_db:get_update_seq(Db),
+
+ init_fdb_state(Db, DDoc, #{
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_BUILDING
+ }),
+
+ % Trigger an upgrade
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ couch_views_fdb:get_view_state(TxDb, Mrst)
+ end),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_BUILDING
+ }),
+
+ % Build the view
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([row(<<"0">>, 0, 0)], Out),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => DbSeq,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_READY
+ }).
+
+
+upgrade_partially_built_interactive(Db) ->
+ DDoc = create_ddoc(),
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+
+ MidSeq = fabric2_db:get_update_seq(Db),
+
+ Doc1 = doc(0),
+ {ok, _} = fabric2_db:update_doc(Db, Doc1, []),
+
+ DbSeq = fabric2_db:get_update_seq(Db),
+
+ init_fdb_state(Db, DDoc, #{
+ view_seq => MidSeq,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_BUILDING
+ }),
+
+ % Trigger an upgrade
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ couch_views_fdb:get_view_state(TxDb, Mrst)
+ end),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_BUILDING
+ }),
+
+ % Build the view
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([row(<<"0">>, 0, 0)], Out),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => DbSeq,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_READY
+ }).
+
+
+upgrade_built_interactive(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_doc(Db, Doc1, []),
+
+ DbSeq = fabric2_db:get_update_seq(Db),
+
+ init_fdb_state(Db, DDoc, #{
+ view_seq => DbSeq,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_READY
+ }),
+
+ % Trigger an upgrade
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ couch_views_fdb:get_view_state(TxDb, Mrst)
+ end),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => <<>>,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_BUILDING
+ }),
+
+ % Build the view
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([row(<<"0">>, 0, 0)], Out),
+
+ assert_fdb_state(Db, DDoc, #{
+ version => ?CURRENT_VIEW_IMPL_VERSION,
+ view_seq => DbSeq,
+ view_vs => fabric2_fdb:seq_to_vs(DbSeq),
+ build_status => ?INDEX_READY
+ }).
+
+
+init_fdb_state(Db, #doc{} = DDoc, Values) ->
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ init_fdb_state(Db, Mrst, Values);
+init_fdb_state(Db, #mrst{sig = Sig}, Values) ->
+ init_fdb_state(Db, Sig, Values);
+init_fdb_state(Db, Sig, Values) ->
+ VersionRow = case maps:get(version, Values, undefined) of
+ undefined -> [];
+ Version -> [{pack(Db, key(version, Sig)), pack({Version})}]
+ end,
+
+ SeqRow = case maps:get(view_seq, Values, undefined) of
+ undefined -> [];
+ Seq -> [{pack(Db, key(seq, Sig)), Seq}]
+ end,
+
+ VSRow = case maps:get(view_vs, Values, undefined) of
+ undefined -> [];
+ VS -> [{pack(Db, key(vs, Sig)), pack({VS})}]
+ end,
+
+ BSRow = case maps:get(build_status, Values, undefined) of
+ undefined -> [];
+ BS -> [{pack(Db, key(bs, Sig)), BS}]
+ end,
+
+ Rows = VersionRow ++ SeqRow ++ VSRow ++ BSRow,
+
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ lists:foreach(fun({K, V}) ->
+ erlfdb:set(Tx, K, V)
+ end, Rows)
+ end).
+
+
+assert_fdb_state(Db, #doc{} = DDoc, Expect) ->
+ {ok, Mrst} = couch_views_util:ddoc_to_mrst(fabric2_db:name(Db), DDoc),
+ assert_fdb_state(Db, Mrst, Expect);
+assert_fdb_state(Db, #mrst{sig = Sig}, Expect) ->
+ assert_fdb_state(Db, Sig, Expect);
+assert_fdb_state(Db, Sig, Expect) ->
+ #{
+ version := Version,
+ view_seq := ViewSeq,
+ view_vs := ViewVS,
+ build_status := BuildStatus
+ } = Expect,
+
+ VersionRow = case Version of
+ not_found -> [];
+ _ -> [{pack(Db, key(version, Sig)), pack({Version})}]
+ end,
+
+ SeqRow = case ViewSeq of
+ <<>> -> [];
+ _ -> [{pack(Db, key(seq, Sig)), ViewSeq}]
+ end,
+
+ VSRow = case ViewVS of
+ not_found -> [];
+ _ -> [{pack(Db, key(vs, Sig)), pack({ViewVS})}]
+ end,
+
+ BSRow = case BuildStatus of
+ not_found -> [];
+ _ -> [{pack(Db, key(bs, Sig)), BuildStatus}]
+ end,
+
+ ExpectRows = lists:sort(VersionRow ++ SeqRow ++ VSRow ++ BSRow),
+
+ RawExistingRows = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ RangePrefix = erlfdb_tuple:pack({?DB_VIEWS, ?VIEW_INFO}, DbPrefix),
+ erlfdb:wait(erlfdb:get_range_startswith(Tx, RangePrefix))
+ end),
+
+ % Ignore the KV size key in the view info rows
+ KVSizeKey = pack(Db, key(kv_size, Sig)),
+ ExistingRows = lists:keydelete(KVSizeKey, 1, RawExistingRows),
+
+ ?assertEqual(ExpectRows, ExistingRows).
+
+
+key(version, Sig) -> {?DB_VIEWS, ?VIEW_INFO, ?VIEW_IMPL_VERSION, Sig};
+key(seq, Sig) -> {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Sig};
+key(kv_size, Sig) -> {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Sig};
+key(vs, Sig) -> {?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig};
+key(bs, Sig) -> {?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig}.
+
+
+pack(Db, Key) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+pack(Value) ->
+ erlfdb_tuple:pack(Value).
+
+
+row(Id, Key, Value) ->
+ {row, [
+ {id, Id},
+ {key, Key},
+ {value, Value}
+ ]}.
+
+
+fold_fun({meta, _Meta}, Acc) ->
+ {ok, Acc};
+fold_fun({row, _} = Row, Acc) ->
+ {ok, [Row | Acc]};
+fold_fun(complete, Acc) ->
+ {ok, lists:reverse(Acc)}.
+
+
+create_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/bar">>},
+ {<<"views">>, {[
+ {?MAP_FUN1, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>}
+ ]}},
+ {?MAP_FUN2, {[
+ {<<"map">>, <<"function(doc) {}">>}
+ ]}}
+ ]}}
+ ]}).
+
+
+doc(Id) ->
+ doc(Id, Id).
+
+
+doc(Id, Val) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"val">>, Val}
+ ]}).
+
+
+run_query(#{} = Db, DDoc, <<_/binary>> = View) ->
+ couch_views:query(Db, DDoc, View, fun fold_fun/2, [], #mrargs{}).
\ No newline at end of file
[couchdb] 01/07: Calculate external JSON size of a view row
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fa3c3c8dd3d5b77371cd74dfbe149623797160ad
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Jul 24 10:58:53 2020 -0500
Calculate external JSON size of a view row
---
src/couch/src/couch_ejson_size.erl | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/couch/src/couch_ejson_size.erl b/src/couch/src/couch_ejson_size.erl
index f550568..76e3924 100644
--- a/src/couch/src/couch_ejson_size.erl
+++ b/src/couch/src/couch_ejson_size.erl
@@ -15,6 +15,11 @@
-export([encoded_size/1]).
+%% View rows
+
+encoded_size({EJson, DocId}) when is_binary(DocId) ->
+ encoded_size(EJson) + encoded_size(DocId);
+
%% Compound objects
encoded_size({[]}) ->
[couchdb] 02/07: Export fabric2_fdb:chunkify_binary/1,2
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3b508e0c5c20d12d0e9007327edd68da23158458
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Aug 6 12:34:29 2020 -0500
Export fabric2_fdb:chunkify_binary/1,2
---
src/fabric/src/fabric2_fdb.erl | 33 ++++++++++++++++++---------------
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 52303ce..36fa451 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -77,6 +77,9 @@
get_approximate_tx_size/1,
+ chunkify_binary/1,
+ chunkify_binary/2,
+
debug_cluster/0,
debug_cluster/2
]).
@@ -1176,6 +1179,21 @@ get_approximate_tx_size(#{} = TxDb) ->
erlfdb:wait(erlfdb:get_approximate_size(Tx)).
+chunkify_binary(Data) ->
+ chunkify_binary(Data, binary_chunk_size()).
+
+
+chunkify_binary(Data, Size) ->
+ case Data of
+ <<>> ->
+ [];
+ <<Head:Size/binary, Rest/binary>> ->
+ [Head | chunkify_binary(Rest, Size)];
+ <<_/binary>> when size(Data) < Size ->
+ [Data]
+ end.
+
+
debug_cluster() ->
debug_cluster(<<>>, <<16#FE, 16#FF, 16#FF>>).
@@ -1677,21 +1695,6 @@ sum_rem_rev_sizes(RevInfos) ->
end, 0, RevInfos).
-chunkify_binary(Data) ->
- chunkify_data(Data, binary_chunk_size()).
-
-
-chunkify_data(Data, Size) ->
- case Data of
- <<>> ->
- [];
- <<Head:Size/binary, Rest/binary>> ->
- [Head | chunkify_data(Rest, Size)];
- <<_/binary>> when size(Data) < Size ->
- [Data]
- end.
-
-
get_fold_acc(Db, RangePrefix, UserCallback, UserAcc, Options)
when is_map(Db) orelse Db =:= undefined ->
[couchdb] 06/07: Use ebtree for reduce functions
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 89c175d97be413815ed5b69c803a92aa9d80c3c5
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Jul 29 10:34:48 2020 -0500
Use ebtree for reduce functions
---
src/couch_views/src/couch_views.erl | 6 --
src/couch_views/src/couch_views_fdb.erl | 1 -
src/couch_views/src/couch_views_reader.erl | 153 ++++++++++++++++++++++++++++-
3 files changed, 150 insertions(+), 10 deletions(-)
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index da8a142..2d91631 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -161,12 +161,6 @@ maybe_update_view(TxDb, Mrst, false, _Args) ->
end.
-is_reduce_view(#mrargs{view_type = ViewType}) ->
- ViewType =:= red;
-is_reduce_view({Reduce, _, _}) ->
- Reduce =:= red.
-
-
to_mrargs(#mrargs{} = Args) ->
Args;
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 27ef386..fc05af2 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -298,7 +298,6 @@ reset_interactive_index(Db, Sig, _St) ->
{VS, ?INDEX_BUILDING}.
-
seq_key(Db, Sig) ->
#{
db_prefix := DbPrefix
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index a785c7b..e2422f7 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -23,7 +23,15 @@
-include_lib("fabric/include/fabric2.hrl").
-read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
+read(Db, Mrst, ViewName, UserCallback, UserAcc, Args) ->
+ ReadFun = case Args of
+ #mrargs{view_type = map} -> fun read_map_view/6;
+ #mrargs{view_type = red} -> fun read_red_view/6
+ end,
+ ReadFun(Db, Mrst, ViewName, UserCallback, UserAcc, Args).
+
+
+read_map_view(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
#mrst{
@@ -68,6 +76,73 @@ read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
end.
+read_red_view(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
+ try
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ #mrst{
+ language = Lang,
+ views = Views
+ } = Mrst = couch_views_fdb:set_trees(TxDb, Mrst0),
+
+ #mrargs{
+ extra = Extra
+ } = Args,
+
+ {Idx, Lang, View} = get_red_view(Lang, Args, ViewName, Views),
+ Fun = fun handle_red_row/3,
+
+ Meta = get_red_meta(TxDb, Mrst, View, Args),
+ UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
+
+ Finalizer = case couch_util:get_value(finalizer, Extra) of
+ undefined ->
+ {_, FunSrc} = lists:nth(Idx, View#mrview.reduce_funs),
+ FunSrc;
+ CustomFun->
+ CustomFun
+ end,
+
+ Acc0 = #{
+ db => TxDb,
+ skip => Args#mrargs.skip,
+ limit => Args#mrargs.limit,
+ mrargs => undefined,
+ finalizer => Finalizer,
+ red_idx => Idx,
+ language => Lang,
+ callback => UserCallback,
+ acc => UserAcc1
+ },
+
+ Acc1 = lists:foldl(fun(KeyArgs, KeyAcc0) ->
+ Opts = mrargs_to_fdb_options(KeyArgs),
+ KeyAcc1 = KeyAcc0#{
+ mrargs := KeyArgs
+ },
+ couch_views_fdb:fold_red_idx(
+ TxDb,
+ View,
+ Idx,
+ Opts,
+ Fun,
+ KeyAcc1
+ )
+ end, Acc0, expand_keys_args(Args)),
+
+ #{
+ acc := UserAcc2
+ } = Acc1,
+ {ok, maybe_stop(UserCallback(complete, UserAcc2))}
+ end)
+ catch
+ throw:{complete, Out} ->
+ {_, Final} = UserCallback(complete, Out),
+ {ok, Final};
+ throw:{done, Out} ->
+ {ok, Out}
+ end.
+
+
get_map_meta(TxDb, Mrst, View, #mrargs{update_seq = true}) ->
TotalRows = couch_views_trees:get_row_count(TxDb, View),
ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
@@ -78,6 +153,14 @@ get_map_meta(TxDb, _Mrst, View, #mrargs{}) ->
{meta, [{total, TotalRows}, {offset, null}]}.
+get_red_meta(TxDb, Mrst, _View, #mrargs{update_seq = true}) ->
+ ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
+ {meta, [{update_seq, ViewSeq}]};
+
+get_red_meta(_TxDb, _Mrst, _View, #mrargs{}) ->
+ {meta, []}.
+
+
handle_map_row(_DocId, _Key, _Value, #{skip := Skip} = Acc) when Skip > 0 ->
Acc#{skip := Skip - 1};
@@ -115,6 +198,38 @@ handle_map_row(DocId, Key, Value, Acc) ->
Acc#{limit := Limit - 1, acc := UserAcc1}.
+handle_red_row(_Key, _Red, #{skip := Skip} = Acc) when Skip > 0 ->
+ Acc#{skip := Skip - 1};
+
+handle_red_row(_Key, _Value, #{limit := 0, acc := UserAcc}) ->
+ throw({complete, UserAcc});
+
+handle_red_row(Key0, Value0, Acc) ->
+ #{
+ limit := Limit,
+ finalizer := Finalizer,
+ callback := UserCallback,
+ acc := UserAcc0
+ } = Acc,
+
+ Key1 = case Key0 of
+ undefined -> null;
+ _ -> Key0
+ end,
+ Value1 = maybe_finalize(Finalizer, Value0),
+ Row = [{key, Key1}, {value, Value1}],
+
+ UserAcc1 = maybe_stop(UserCallback({row, Row}, UserAcc0)),
+ Acc#{limit := Limit - 1, acc := UserAcc1}.
+
+
+maybe_finalize(null, Red) ->
+ Red;
+maybe_finalize(Finalizer, Red) ->
+ {ok, Finalized} = couch_query_servers:finalize(Finalizer, Red),
+ Finalized.
+
+
get_map_view(Lang, Args, ViewName, Views) ->
case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
{map, View, _Args} -> View;
@@ -122,6 +237,13 @@ get_map_view(Lang, Args, ViewName, Views) ->
end.
+get_red_view(Lang, Args, ViewName, Views) ->
+ case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
+ {red, {Idx, Lang, View}, _} -> {Idx, Lang, View};
+ _ -> throw({not_found, missing_named_view})
+ end.
+
+
expand_keys_args(#mrargs{keys = undefined} = Args) ->
[Args];
@@ -136,12 +258,14 @@ expand_keys_args(#mrargs{keys = Keys} = Args) ->
mrargs_to_fdb_options(Args) ->
#mrargs{
+ view_type = ViewType,
start_key = StartKey,
start_key_docid = StartKeyDocId,
end_key = EndKey,
end_key_docid = EndKeyDocId0,
direction = Direction,
- inclusive_end = InclusiveEnd
+ inclusive_end = InclusiveEnd,
+ group_level = GroupLevel
} = Args,
StartKeyOpts = if StartKey == undefined -> []; true ->
@@ -160,10 +284,33 @@ mrargs_to_fdb_options(Args) ->
[{end_key, {EndKey, EndKeyDocId}}]
end,
+ GroupFunOpt = make_group_key_fun(ViewType, GroupLevel),
+
[
{dir, Direction},
{inclusive_end, InclusiveEnd}
- ] ++ StartKeyOpts ++ EndKeyOpts.
+ ] ++ StartKeyOpts ++ EndKeyOpts ++ GroupFunOpt.
+
+
+make_group_key_fun(map, _) ->
+ [];
+
+make_group_key_fun(red, exact) ->
+ [
+ {group_key_fun, fun({Key, _DocId}) -> Key end}
+ ];
+
+make_group_key_fun(red, 0) ->
+ [
+ {group_key_fun, group_all}
+ ];
+
+make_group_key_fun(red, N) when is_integer(N), N > 0 ->
+ GKFun = fun
+ ({Key, _DocId}) when is_list(Key) -> lists:sublist(Key, N);
+ ({Key, _DocId}) -> Key
+ end,
+ [{group_key_fun, GKFun}].
maybe_stop({ok, Acc}) -> Acc;
[couchdb] 03/07: Views on ebtree
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1f773fba879c3374cdab30de4b067a0e6efe6748
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Jul 24 10:59:05 2020 -0500
Views on ebtree
---
rel/overlay/etc/default.ini | 6 +
src/couch_views/include/couch_views.hrl | 5 +
src/couch_views/src/couch_views.erl | 55 +--
src/couch_views/src/couch_views_fdb.erl | 331 ++---------------
src/couch_views/src/couch_views_indexer.erl | 56 +--
src/couch_views/src/couch_views_reader.erl | 115 +++---
src/couch_views/src/couch_views_trees.erl | 429 ++++++++++++++++++++++
src/couch_views/src/couch_views_updater.erl | 13 +-
src/couch_views/src/couch_views_util.erl | 35 ++
src/couch_views/test/couch_views_cleanup_test.erl | 2 +-
src/couch_views/test/couch_views_indexer_test.erl | 64 ++--
src/couch_views/test/couch_views_size_test.erl | 25 +-
src/couch_views/test/couch_views_updater_test.erl | 4 +-
src/mango/src/mango_cursor_view.erl | 14 +-
src/mango/src/mango_idx_view.erl | 7 +-
src/mango/src/mango_idx_view.hrl | 13 +
16 files changed, 689 insertions(+), 485 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index abcf0bd..3a377c7 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -337,6 +337,12 @@ iterations = 10 ; iterations for password hashing
; The maximum allowed value size emitted from a view for a document (in bytes)
;value_size_limit = 64000
;
+; The maximum size of B+Tree nodes used by the id btree
+;id_btree_node_size = 100
+;
+; The maximum size of B+Tree nodes used by view btrees
+;view_btree_node_size = 100
+;
; Batch size sensing parameters
; batch_initial_size = 100 ; Initial batch size in number of documents
; batch_search_increment = 500 ; Size change when searching for the threshold
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index 3d0110f..3882191 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -13,6 +13,7 @@
% Index info/data subspaces
-define(VIEW_INFO, 0).
-define(VIEW_DATA, 1).
+-define(VIEW_TREES, 3).
% Index info keys
-define(VIEW_UPDATE_SEQ, 0).
@@ -25,6 +26,10 @@
-define(VIEW_ID_RANGE, 0).
-define(VIEW_MAP_RANGE, 1).
+% Tree keys
+-define(VIEW_ID_TREE, 0).
+-define(VIEW_ROW_TREES, 1).
+
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index d9ba0c1..da8a142 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -48,11 +48,7 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
Args1 = to_mrargs(Args0),
Args2 = couch_mrview_util:set_view_type(Args1, ViewName, Views),
Args3 = couch_mrview_util:validate_args(Args2),
- ok = check_range(Args3),
- case is_reduce_view(Args3) of
- true -> throw(not_implemented);
- false -> ok
- end,
+ ok = check_range(Mrst, ViewName, Args3),
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
@@ -100,9 +96,10 @@ get_info(Db, DDoc) ->
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
Sig = fabric2_util:to_hex(Mrst#mrst.sig),
{UpdateSeq, DataSize, Status} = fabric2_fdb:transactional(Db, fun(TxDb) ->
- Seq = couch_views_fdb:get_update_seq(TxDb, Mrst),
- DataSize = get_total_view_size(TxDb, Mrst),
- JobStatus = case couch_views_jobs:job_state(TxDb, Mrst) of
+ Mrst1 = couch_views_trees:open(TxDb, Mrst),
+ Seq = couch_views_fdb:get_update_seq(TxDb, Mrst1),
+ DataSize = get_total_view_size(TxDb, Mrst1),
+ JobStatus = case couch_views_jobs:job_state(TxDb, Mrst1) of
{ok, pending} -> true;
{ok, running} -> true;
{ok, finished} -> false;
@@ -124,10 +121,9 @@ get_info(Db, DDoc) ->
get_total_view_size(TxDb, Mrst) ->
- ViewIds = [View#mrview.id_num || View <- Mrst#mrst.views],
- lists:foldl(fun (ViewId, Total) ->
- Total + couch_views_fdb:get_kv_size(TxDb, Mrst, ViewId)
- end, 0, ViewIds).
+ lists:foldl(fun(View, Total) ->
+ Total + couch_views_trees:get_kv_size(TxDb, View)
+ end, 0, Mrst#mrst.views).
read_view(Db, Mrst, ViewName, Callback, Acc0, Args) ->
@@ -185,16 +181,29 @@ to_mrargs(#{} = Args) ->
end, #mrargs{}, Args).
-check_range(#mrargs{start_key = undefined}) ->
+check_range(Mrst, ViewName, Args) ->
+ #mrst{
+ language = Lang,
+ views = Views
+ } = Mrst,
+ View = case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
+ {map, V, _} -> V;
+ {red, {_, _, V}, _} -> V
+ end,
+ Cmp = couch_views_util:collate_fun(View),
+ check_range(Args, Cmp).
+
+
+check_range(#mrargs{start_key = undefined}, _Cmp) ->
ok;
-check_range(#mrargs{end_key = undefined}) ->
+check_range(#mrargs{end_key = undefined}, _Cmp) ->
ok;
-check_range(#mrargs{start_key = K, end_key = K}) ->
+check_range(#mrargs{start_key = K, end_key = K}, _Cmp) ->
ok;
-check_range(Args) ->
+check_range(Args, Cmp) ->
#mrargs{
direction = Dir,
start_key = SK,
@@ -203,10 +212,10 @@ check_range(Args) ->
end_key_docid = EKD
} = Args,
- case {Dir, view_cmp(SK, SKD, EK, EKD)} of
- {fwd, false} ->
+ case {Dir, Cmp({SK, SKD}, {EK, EKD})} of
+ {fwd, gt} ->
throw(check_range_error(<<"true">>));
- {rev, true} ->
+ {rev, lt} ->
throw(check_range_error(<<"false">>));
_ ->
ok
@@ -220,14 +229,6 @@ check_range_error(Descending) ->
Descending/binary>>}.
-view_cmp(SK, SKD, EK, EKD) ->
- BinSK = couch_views_encoding:encode(SK, key),
- BinEK = couch_views_encoding:encode(EK, key),
- PackedSK = erlfdb_tuple:pack({BinSK, SKD}),
- PackedEK = erlfdb_tuple:pack({BinEK, EKD}),
- PackedSK =< PackedEK.
-
-
get_update_options(#mrst{design_opts = Opts}) ->
IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index c957222..e813f2b 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -22,15 +22,10 @@
get_update_seq/2,
set_update_seq/3,
- get_row_count/3,
- get_kv_size/3,
-
- fold_map_idx/6,
-
- write_doc/4,
-
list_signatures/1,
- clear_index/2
+ clear_index/2,
+
+ persist_chunks/3
]).
-ifdef(TEST).
@@ -38,10 +33,6 @@
-compile(nowarn_export_all).
-endif.
--define(LIST_VALUE, 0).
--define(JSON_VALUE, 1).
--define(VALUE, 2).
-
-include("couch_views.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
@@ -126,94 +117,6 @@ set_update_seq(TxDb, Sig, Seq) ->
ok = erlfdb:set(Tx, seq_key(DbPrefix, Sig), Seq).
-get_row_count(TxDb, #mrst{sig = Sig}, ViewId) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- case erlfdb:wait(erlfdb:get(Tx, row_count_key(DbPrefix, Sig, ViewId))) of
- not_found -> 0; % Can this happen?
- CountBin -> ?bin2uint(CountBin)
- end.
-
-
-get_kv_size(TxDb, #mrst{sig = Sig}, ViewId) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- case erlfdb:wait(erlfdb:get(Tx, kv_size_key(DbPrefix, Sig, ViewId))) of
- not_found -> 0; % Can this happen?
- SizeBin -> ?bin2uint(SizeBin)
- end.
-
-
-fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) ->
- #{
- db_prefix := DbPrefix
- } = TxDb,
-
- MapIdxPrefix = map_idx_prefix(DbPrefix, Sig, ViewId),
- FoldAcc = #{
- prefix => MapIdxPrefix,
- callback => Callback,
- acc => Acc0
- },
- Fun = aegis:wrap_fold_fun(TxDb, fun fold_fwd/2),
-
- #{
- acc := Acc1
- } = fabric2_fdb:fold_range(TxDb, MapIdxPrefix, Fun, FoldAcc, Options),
-
- Acc1.
-
-
-write_doc(TxDb, Sig, _ViewIds, #{deleted := true} = Doc) ->
- #{
- id := DocId
- } = Doc,
-
- ExistingViewKeys = get_view_keys(TxDb, Sig, DocId),
-
- clear_id_idx(TxDb, Sig, DocId),
- lists:foreach(fun({ViewId, TotalKeys, TotalSize, UniqueKeys}) ->
- clear_map_idx(TxDb, Sig, ViewId, DocId, UniqueKeys),
- update_row_count(TxDb, Sig, ViewId, -TotalKeys),
- update_kv_size(TxDb, Sig, ViewId, -TotalSize)
- end, ExistingViewKeys);
-
-write_doc(TxDb, Sig, ViewIds, Doc) ->
- #{
- id := DocId,
- results := Results,
- kv_sizes := KVSizes
- } = Doc,
-
- ExistingViewKeys = get_view_keys(TxDb, Sig, DocId),
-
- clear_id_idx(TxDb, Sig, DocId),
-
- lists:foreach(fun({ViewId, NewRows, KVSize}) ->
- update_id_idx(TxDb, Sig, ViewId, DocId, NewRows, KVSize),
-
- ExistingKeys = case lists:keyfind(ViewId, 1, ExistingViewKeys) of
- {ViewId, TotalRows, TotalSize, EKeys} ->
- RowChange = length(NewRows) - TotalRows,
- update_row_count(TxDb, Sig, ViewId, RowChange),
- update_kv_size(TxDb, Sig, ViewId, KVSize - TotalSize),
- EKeys;
- false ->
- RowChange = length(NewRows),
- update_row_count(TxDb, Sig, ViewId, RowChange),
- update_kv_size(TxDb, Sig, ViewId, KVSize),
- []
- end,
- update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows)
- end, lists:zip3(ViewIds, Results, KVSizes)).
-
-
list_signatures(Db) ->
#{
db_prefix := DbPrefix
@@ -244,145 +147,38 @@ clear_index(Db, Signature) ->
end, Keys),
% Clear index data
- RangeTuple = {?DB_VIEWS, ?VIEW_DATA, Signature},
- RangePrefix = erlfdb_tuple:pack(RangeTuple, DbPrefix),
- erlfdb:clear_range_startswith(Tx, RangePrefix).
-
-
-% For each row in a map view we store the the key/value
-% in FoundationDB:
-%
-% `(EncodedSortKey, (EncodedKey, EncodedValue))`
-%
-% The difference between `EncodedSortKey` and `EndcodedKey` is
-% the use of `couch_util:get_sort_key/1` which turns UTF-8
-% strings into binaries that are byte comparable. Given a sort
-% key binary we cannot recover the input so to return unmodified
-% user data we are forced to store the original.
-
-fold_fwd({RowKey, PackedKeyValue}, Acc) ->
- #{
- prefix := Prefix,
- callback := UserCallback,
- acc := UserAcc0
- } = Acc,
-
- {{_SortKey, DocId}, _DupeId} =
- erlfdb_tuple:unpack(RowKey, Prefix),
-
- {EncodedOriginalKey, EncodedValue} = erlfdb_tuple:unpack(PackedKeyValue),
- Value = couch_views_encoding:decode(EncodedValue),
- Key = couch_views_encoding:decode(EncodedOriginalKey),
-
- UserAcc1 = UserCallback(DocId, Key, Value, UserAcc0),
-
- Acc#{
- acc := UserAcc1
- }.
-
-
-clear_id_idx(TxDb, Sig, DocId) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- {Start, End} = id_idx_range(DbPrefix, Sig, DocId),
- ok = erlfdb:clear_range(Tx, Start, End).
-
-
-clear_map_idx(TxDb, Sig, ViewId, DocId, ViewKeys) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
+ DataTuple = {?DB_VIEWS, ?VIEW_DATA, Signature},
+ DataPrefix = erlfdb_tuple:pack(DataTuple, DbPrefix),
+ erlfdb:clear_range_startswith(Tx, DataPrefix),
- lists:foreach(fun(ViewKey) ->
- {Start, End} = map_idx_range(DbPrefix, Sig, ViewId, ViewKey, DocId),
- ok = erlfdb:clear_range(Tx, Start, End)
- end, ViewKeys).
+ % Clear tree data
+ TreeTuple = {?DB_VIEWS, ?VIEW_TREES, Signature},
+ TreePrefix = erlfdb_tuple:pack(TreeTuple, DbPrefix),
+ erlfdb:clear_range_startswith(Tx, TreePrefix).
-update_id_idx(TxDb, Sig, ViewId, DocId, [], _KVSize) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
- Key = id_idx_key(DbPrefix, Sig, DocId, ViewId),
- ok = erlfdb:clear(Tx, Key);
-
-update_id_idx(TxDb, Sig, ViewId, DocId, NewRows, KVSize) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- Unique = lists:usort([K || {K, _V} <- NewRows]),
-
- Key = id_idx_key(DbPrefix, Sig, DocId, ViewId),
- Val = couch_views_encoding:encode([length(NewRows), KVSize, Unique]),
- ok = erlfdb:set(Tx, Key, aegis:encrypt(TxDb, Key, Val)).
-
-
-update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- lists:foreach(fun(RemKey) ->
- {Start, End} = map_idx_range(DbPrefix, Sig, ViewId, RemKey, DocId),
- ok = erlfdb:clear_range(Tx, Start, End)
- end, ExistingKeys),
-
- KVsToAdd = process_rows(NewRows),
- MapIdxPrefix = map_idx_prefix(DbPrefix, Sig, ViewId),
-
- lists:foreach(fun({DupeId, Key1, Key2, EV}) ->
- KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId),
- Val = erlfdb_tuple:pack({Key2, EV}),
- ok = erlfdb:set(Tx, KK, aegis:encrypt(TxDb, KK, Val))
- end, KVsToAdd).
-
-
-get_view_keys(TxDb, Sig, DocId) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
- {Start, End} = id_idx_range(DbPrefix, Sig, DocId),
- lists:map(fun({K, V}) ->
- {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_ID_RANGE, DocId, ViewId} =
- erlfdb_tuple:unpack(K, DbPrefix),
- [TotalKeys, TotalSize, UniqueKeys] = couch_views_encoding:decode(V),
- {ViewId, TotalKeys, TotalSize, UniqueKeys}
- end, aegis:decrypt(TxDb, erlfdb:get_range(Tx, Start, End, []))).
-
-
-update_row_count(TxDb, Sig, ViewId, Increment) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
- Key = row_count_key(DbPrefix, Sig, ViewId),
- erlfdb:add(Tx, Key, Increment).
+persist_chunks(Tx, set, [Key, Value]) ->
+ Chunks = fabric2_fdb:chunkify_binary(Value),
+ LastId = lists:foldl(fun(Chunk, Id) ->
+ ChunkKey = erlfdb_tuple:pack({Id}, Key),
+ erlfdb:set(Tx, ChunkKey, Chunk),
+ Id + 1
+ end, 0, Chunks),
+ % We update nodes in place, so its possible that
+ % a node shrank. This clears any keys that we haven't
+ % just overwritten for the provided key.
+ LastIdKey = erlfdb_tuple:pack({LastId}, Key),
+ EndRange = <<Key/binary, 16#FF>>,
+ erlfdb:clear_range(Tx, LastIdKey, EndRange);
-update_kv_size(TxDb, Sig, ViewId, Increment) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
-
- % Track a view specific size for calls to
- % GET /dbname/_design/doc/_info`
- IdxKey = kv_size_key(DbPrefix, Sig, ViewId),
- erlfdb:add(Tx, IdxKey, Increment),
+persist_chunks(Tx, get, Key) ->
+ Rows = erlfdb:get_range_startswith(Tx, Key),
+ Values = [V || {_K, V} <- Rows],
+ iolist_to_binary(Values);
- % Track a database level rollup for calls to
- % GET /dbname
- DbKey = db_kv_size_key(DbPrefix),
- erlfdb:add(Tx, DbKey, Increment).
+persist_chunks(Tx, clear, Key) ->
+ erlfdb:clear_range_startswith(Tx, Key).
seq_key(DbPrefix, Sig) ->
@@ -390,54 +186,6 @@ seq_key(DbPrefix, Sig) ->
erlfdb_tuple:pack(Key, DbPrefix).
-row_count_key(DbPrefix, Sig, ViewId) ->
- Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_ROW_COUNT, Sig, ViewId},
- erlfdb_tuple:pack(Key, DbPrefix).
-
-
-kv_size_key(DbPrefix, Sig, ViewId) ->
- Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Sig, ViewId},
- erlfdb_tuple:pack(Key, DbPrefix).
-
-
-db_kv_size_key(DbPrefix) ->
- Key = {?DB_STATS, <<"sizes">>, <<"views">>},
- erlfdb_tuple:pack(Key, DbPrefix).
-
-
-id_idx_key(DbPrefix, Sig, DocId, ViewId) ->
- Key = {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_ID_RANGE, DocId, ViewId},
- erlfdb_tuple:pack(Key, DbPrefix).
-
-
-id_idx_range(DbPrefix, Sig, DocId) ->
- Key = {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_ID_RANGE, DocId},
- erlfdb_tuple:range(Key, DbPrefix).
-
-
-map_idx_prefix(DbPrefix, Sig, ViewId) ->
- Key = {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_MAP_RANGE, ViewId},
- erlfdb_tuple:pack(Key, DbPrefix).
-
-
-map_idx_key(MapIdxPrefix, MapKey, DupeId) ->
- Key = {MapKey, DupeId},
- erlfdb_tuple:pack(Key, MapIdxPrefix).
-
-
-map_idx_range(DbPrefix, Sig, ViewId, MapKey, DocId) ->
- Encoded = couch_views_encoding:encode(MapKey, key),
- Key = {
- ?DB_VIEWS,
- ?VIEW_DATA,
- Sig,
- ?VIEW_MAP_RANGE,
- ViewId,
- {Encoded, DocId}
- },
- erlfdb_tuple:range(Key, DbPrefix).
-
-
creation_vs_key(Db, Sig) ->
#{
db_prefix := DbPrefix
@@ -452,24 +200,3 @@ build_status_key(Db, Sig) ->
} = Db,
Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig},
erlfdb_tuple:pack(Key, DbPrefix).
-
-
-process_rows(Rows) ->
- Encoded = lists:map(fun({K, V}) ->
- EK1 = couch_views_encoding:encode(K, key),
- EK2 = couch_views_encoding:encode(K, value),
- EV = couch_views_encoding:encode(V, value),
- {EK1, EK2, EV}
- end, Rows),
-
- Grouped = lists:foldl(fun({K1, K2, V}, Acc) ->
- dict:append(K1, {K2, V}, Acc)
- end, dict:new(), Encoded),
-
- dict:fold(fun(K1, Vals, DAcc) ->
- Vals1 = lists:keysort(2, Vals),
- {_, Labeled} = lists:foldl(fun({K2, V}, {Count, Acc}) ->
- {Count + 1, [{Count, K1, K2, V} | Acc]}
- end, {0, []}, Vals1),
- Labeled ++ DAcc
- end, [], Grouped).
diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index 17b0daa..c523c4d 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -102,6 +102,8 @@ init() ->
update_stats => #{}
},
+ process_flag(sensitive, false),
+
try
update(Db, Mrst, State)
catch
@@ -110,6 +112,10 @@ init() ->
error:database_does_not_exist ->
fail_job(Job, Data, db_deleted, "Database was deleted");
Error:Reason ->
+ Stack = erlang:get_stacktrace(),
+ Fmt = "Error building view for ddoc ~s in ~s: ~p:~p ~p",
+ couch_log:error(Fmt, [DbName, DDocId, Error, Reason, Stack]),
+
NewRetry = Retries + 1,
RetryLimit = retry_limit(),
@@ -196,6 +202,7 @@ do_update(Db, Mrst0, State0) ->
tx := Tx
} = TxDb,
+ Mrst1 = couch_views_trees:open(TxDb, Mrst0),
State1 = get_update_start_state(TxDb, Mrst0, State0),
{ok, State2} = fold_changes(State1),
@@ -212,7 +219,7 @@ do_update(Db, Mrst0, State0) ->
DocAcc1 = fetch_docs(TxDb, DesignOpts, DocAcc),
- {Mrst1, MappedDocs} = map_docs(Mrst0, DocAcc1),
+ {Mrst2, MappedDocs} = map_docs(Mrst0, DocAcc1),
TotalKVs = write_docs(TxDb, Mrst1, MappedDocs, State2),
ChangesDone = ChangesDone0 + length(DocAcc),
@@ -225,14 +232,14 @@ do_update(Db, Mrst0, State0) ->
case Count < Limit of
true ->
- maybe_set_build_status(TxDb, Mrst1, ViewVS,
+ maybe_set_build_status(TxDb, Mrst2, ViewVS,
?INDEX_READY),
report_progress(State2#{changes_done := ChangesDone},
finished),
- {Mrst1, finished};
+ {Mrst2, finished};
false ->
State3 = report_progress(State2, update),
- {Mrst1, State3#{
+ {Mrst2, State3#{
tx_db := undefined,
count := 0,
doc_acc := [],
@@ -339,7 +346,7 @@ map_docs(Mrst, Docs) ->
end, Docs),
Deleted1 = lists:map(fun(Doc) ->
- Doc#{results => []}
+ Doc#{results => [[] || _ <- Mrst1#mrst.views]}
end, Deleted0),
DocsToMap = lists:map(fun(Doc) ->
@@ -370,9 +377,8 @@ map_docs(Mrst, Docs) ->
{Mrst1, MappedDocs}.
-write_docs(TxDb, Mrst, Docs, State) ->
+write_docs(TxDb, Mrst, Docs0, State) ->
#mrst{
- views = Views,
sig = Sig
} = Mrst,
@@ -380,15 +386,15 @@ write_docs(TxDb, Mrst, Docs, State) ->
last_seq := LastSeq
} = State,
- ViewIds = [View#mrview.id_num || View <- Views],
KeyLimit = key_size_limit(),
ValLimit = value_size_limit(),
- TotalKVCount = lists:foldl(fun(Doc0, KVCount) ->
- Doc1 = calculate_kv_sizes(Mrst, Doc0, KeyLimit, ValLimit),
- couch_views_fdb:write_doc(TxDb, Sig, ViewIds, Doc1),
- KVCount + count_kvs(Doc1)
- end, 0, Docs),
+ {Docs1, TotalKVCount} = lists:mapfoldl(fun(Doc0, KVCount) ->
+ Doc1 = check_kv_size_limit(Mrst, Doc0, KeyLimit, ValLimit),
+ {Doc1, KVCount + count_kvs(Doc1)}
+ end, 0, Docs0),
+
+ couch_views_trees:update_views(TxDb, Mrst, Docs1),
if LastSeq == false -> ok; true ->
couch_views_fdb:set_update_seq(TxDb, Sig, LastSeq)
@@ -479,7 +485,7 @@ start_query_server(#mrst{} = Mrst) ->
Mrst.
-calculate_kv_sizes(Mrst, Doc, KeyLimit, ValLimit) ->
+check_kv_size_limit(Mrst, Doc, KeyLimit, ValLimit) ->
#mrst{
db_name = DbName,
idx_name = IdxName
@@ -488,10 +494,10 @@ calculate_kv_sizes(Mrst, Doc, KeyLimit, ValLimit) ->
results := Results
} = Doc,
try
- KVSizes = lists:map(fun(ViewRows) ->
- lists:foldl(fun({K, V}, Acc) ->
- KeySize = erlang:external_size(K),
- ValSize = erlang:external_size(V),
+ lists:foreach(fun(ViewRows) ->
+ lists:foreach(fun({K, V}) ->
+ KeySize = couch_ejson_size:encoded_size(K),
+ ValSize = couch_ejson_size:encoded_size(V),
if KeySize =< KeyLimit -> ok; true ->
throw({size_error, key})
@@ -499,18 +505,20 @@ calculate_kv_sizes(Mrst, Doc, KeyLimit, ValLimit) ->
if ValSize =< ValLimit -> ok; true ->
throw({size_error, value})
- end,
-
- Acc + KeySize + ValSize
- end, 0, ViewRows)
+ end
+ end, ViewRows)
end, Results),
- Doc#{kv_sizes => KVSizes}
+ Doc
catch throw:{size_error, Type} ->
#{id := DocId} = Doc,
Fmt = "View ~s size error for docid `~s`, excluded from indexing "
"in db `~s` for design doc `~s`",
couch_log:error(Fmt, [Type, DocId, DbName, IdxName]),
- Doc#{deleted := true, results := [], kv_sizes => []}
+ Doc#{
+ deleted := true,
+ results := [[] || _ <- Mrst#mrst.views],
+ kv_sizes => []
+ }
end.
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 61a78d7..a785c7b 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -23,24 +23,24 @@
-include_lib("fabric/include/fabric2.hrl").
-read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) ->
- #mrst{
- language = Lang,
- sig = Sig,
- views = Views
- } = Mrst,
-
- ViewId = get_view_id(Lang, Args, ViewName, Views),
- Fun = fun handle_row/4,
-
+read(Db, Mrst0, ViewName, UserCallback, UserAcc0, Args) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
- Meta = get_meta(TxDb, Mrst, ViewId, Args),
+ #mrst{
+ language = Lang,
+ views = Views
+ } = Mrst = couch_views_trees:open(TxDb, Mrst0),
+
+ View = get_map_view(Lang, Args, ViewName, Views),
+ Fun = fun handle_map_row/4,
+
+ Meta = get_map_meta(TxDb, Mrst, View, Args),
UserAcc1 = maybe_stop(UserCallback(Meta, UserAcc0)),
Acc0 = #{
db => TxDb,
skip => Args#mrargs.skip,
+ limit => Args#mrargs.limit,
mrargs => undefined,
callback => UserCallback,
acc => UserAcc1
@@ -51,14 +51,7 @@ read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) ->
KeyAcc1 = KeyAcc0#{
mrargs := KeyArgs
},
- couch_views_fdb:fold_map_idx(
- TxDb,
- Sig,
- ViewId,
- Opts,
- Fun,
- KeyAcc1
- )
+ couch_views_trees:fold_map_idx(TxDb, View, Opts, Fun, KeyAcc1)
end, Acc0, expand_keys_args(Args)),
#{
@@ -66,27 +59,35 @@ read(Db, Mrst, ViewName, UserCallback, UserAcc0, Args) ->
} = Acc1,
{ok, maybe_stop(UserCallback(complete, UserAcc2))}
end)
- catch throw:{done, Out} ->
- {ok, Out}
+ catch
+ throw:{complete, Out} ->
+ {_, Final} = UserCallback(complete, Out),
+ {ok, Final};
+ throw:{done, Out} ->
+ {ok, Out}
end.
-get_meta(TxDb, Mrst, ViewId, #mrargs{update_seq = true}) ->
- TotalRows = couch_views_fdb:get_row_count(TxDb, Mrst, ViewId),
+get_map_meta(TxDb, Mrst, View, #mrargs{update_seq = true}) ->
+ TotalRows = couch_views_trees:get_row_count(TxDb, View),
ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
{meta, [{update_seq, ViewSeq}, {total, TotalRows}, {offset, null}]};
-get_meta(TxDb, Mrst, ViewId, #mrargs{}) ->
- TotalRows = couch_views_fdb:get_row_count(TxDb, Mrst, ViewId),
+get_map_meta(TxDb, _Mrst, View, #mrargs{}) ->
+ TotalRows = couch_views_trees:get_row_count(TxDb, View),
{meta, [{total, TotalRows}, {offset, null}]}.
-handle_row(_DocId, _Key, _Value, #{skip := Skip} = Acc) when Skip > 0 ->
+handle_map_row(_DocId, _Key, _Value, #{skip := Skip} = Acc) when Skip > 0 ->
Acc#{skip := Skip - 1};
-handle_row(DocId, Key, Value, Acc) ->
+handle_map_row(_DocID, _Key, _Value, #{limit := 0, acc := UserAcc}) ->
+ throw({complete, UserAcc});
+
+handle_map_row(DocId, Key, Value, Acc) ->
#{
db := TxDb,
+ limit := Limit,
mrargs := Args,
callback := UserCallback,
acc := UserAcc0
@@ -111,13 +112,13 @@ handle_row(DocId, Key, Value, Acc) ->
end,
UserAcc1 = maybe_stop(UserCallback({row, Row}, UserAcc0)),
- Acc#{acc := UserAcc1}.
+ Acc#{limit := Limit - 1, acc := UserAcc1}.
-get_view_id(Lang, Args, ViewName, Views) ->
+get_map_view(Lang, Args, ViewName, Views) ->
case couch_mrview_util:extract_view(Lang, Args, ViewName, Views) of
- {map, View, _Args} -> View#mrview.id_num;
- {red, {_Idx, _Lang, View}} -> View#mrview.id_num
+ {map, View, _Args} -> View;
+ {red, {_Idx, _Lang, View}, _} -> View
end.
@@ -135,57 +136,33 @@ expand_keys_args(#mrargs{keys = Keys} = Args) ->
mrargs_to_fdb_options(Args) ->
#mrargs{
- start_key = StartKey0,
+ start_key = StartKey,
start_key_docid = StartKeyDocId,
- end_key = EndKey0,
- end_key_docid = EndKeyDocId,
+ end_key = EndKey,
+ end_key_docid = EndKeyDocId0,
direction = Direction,
- limit = Limit,
- skip = Skip,
inclusive_end = InclusiveEnd
} = Args,
- StartKey1 = if StartKey0 == undefined -> undefined; true ->
- couch_views_encoding:encode(StartKey0, key)
- end,
-
- StartKeyOpts = case {StartKey1, StartKeyDocId} of
- {undefined, _} ->
- [];
- {StartKey1, StartKeyDocId} ->
- [{start_key, {StartKey1, StartKeyDocId}}]
+ StartKeyOpts = if StartKey == undefined -> []; true ->
+ [{start_key, {StartKey, StartKeyDocId}}]
end,
- EndKey1 = if EndKey0 == undefined -> undefined; true ->
- couch_views_encoding:encode(EndKey0, key)
+ EndKeyDocId = case {Direction, EndKeyDocId0} of
+ {fwd, <<255>>} when InclusiveEnd -> <<255>>;
+ {fwd, <<255>>} when not InclusiveEnd -> <<>>;
+ {rev, <<>>} when InclusiveEnd -> <<>>;
+ {rev, <<>>} when not InclusiveEnd -> <<255>>;
+ _ -> EndKeyDocId0
end,
- EndKeyOpts = case {EndKey1, EndKeyDocId, Direction} of
- {undefined, _, _} ->
- [];
- {EndKey1, <<>>, rev} when not InclusiveEnd ->
- % When we iterate in reverse with
- % inclusive_end=false we have to set the
- % EndKeyDocId to <<255>> so that we don't
- % include matching rows.
- [{end_key_gt, {EndKey1, <<255>>}}];
- {EndKey1, <<255>>, _} when not InclusiveEnd ->
- % When inclusive_end=false we need to
- % elide the default end_key_docid so as
- % to not sort past the docids with the
- % given end key.
- [{end_key_gt, {EndKey1}}];
- {EndKey1, EndKeyDocId, _} when not InclusiveEnd ->
- [{end_key_gt, {EndKey1, EndKeyDocId}}];
- {EndKey1, EndKeyDocId, _} when InclusiveEnd ->
- [{end_key, {EndKey1, EndKeyDocId}}]
+ EndKeyOpts = if EndKey == undefined -> []; true ->
+ [{end_key, {EndKey, EndKeyDocId}}]
end,
[
{dir, Direction},
- {limit, Limit + Skip},
- {streaming_mode, want_all},
- {restart_tx, true}
+ {inclusive_end, InclusiveEnd}
] ++ StartKeyOpts ++ EndKeyOpts.
diff --git a/src/couch_views/src/couch_views_trees.erl b/src/couch_views/src/couch_views_trees.erl
new file mode 100644
index 0000000..d67ccd5
--- /dev/null
+++ b/src/couch_views/src/couch_views_trees.erl
@@ -0,0 +1,429 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_trees).
+
+-export([
+ open/2,
+
+ get_row_count/2,
+ get_kv_size/2,
+
+ fold_map_idx/5,
+
+ update_views/3
+]).
+
+-ifdef(TEST).
+-compile(export_all).
+-compile(nowarn_export_all).
+-endif.
+
+
+-include("couch_views.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+
+
+open(TxDb, Mrst) ->
+ #mrst{
+ sig = Sig,
+ language = Lang,
+ views = Views
+ } = Mrst,
+ Mrst#mrst{
+ id_btree = open_id_tree(TxDb, Sig),
+ views = [open_view_tree(TxDb, Sig, Lang, V) || V <- Views]
+ }.
+
+
+get_row_count(TxDb, View) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ {Count, _} = ebtree:full_reduce(Tx, View#mrview.btree),
+ Count.
+
+
+get_kv_size(TxDb, View) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ {_, TotalSize} = ebtree:full_reduce(Tx, View#mrview.btree),
+ TotalSize.
+
+
+fold_map_idx(TxDb, View, Options, Callback, Acc0) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ #mrview{
+ btree = Btree
+ } = View,
+
+ CollateFun = couch_views_util:collate_fun(View),
+
+ {Dir, StartKey, EndKey, InclusiveEnd} = to_map_opts(Options),
+
+ Wrapper = fun(KVs0, WAcc) ->
+ % Remove any keys that match Start or End key
+ % depending on direction
+ KVs1 = case InclusiveEnd of
+ true ->
+ KVs0;
+ false when Dir == fwd ->
+ lists:filter(fun({K, _V}) ->
+ case CollateFun(K, EndKey) of
+ lt -> true;
+ eq -> false;
+ gt -> false
+ end
+ end, KVs0);
+ false when Dir == rev ->
+ lists:filter(fun({K, _V}) ->
+ case CollateFun(K, EndKey) of
+ lt -> false;
+ eq -> false;
+ gt -> true
+ end
+ end, KVs0)
+ end,
+ % Expand dups
+ KVs2 = lists:flatmap(fun({K, V}) ->
+ case V of
+ {dups, Dups} when Dir == fwd ->
+ [{K, D} || D <- Dups];
+ {dups, Dups} when Dir == rev ->
+ [{K, D} || D <- lists:reverse(Dups)];
+ _ ->
+ [{K, V}]
+ end
+ end, KVs1),
+ lists:foldl(fun({{Key, DocId}, Value}, WAccInner) ->
+ Callback(DocId, Key, Value, WAccInner)
+ end, WAcc, KVs2)
+ end,
+
+ case Dir of
+ fwd ->
+ ebtree:range(Tx, Btree, StartKey, EndKey, Wrapper, Acc0);
+ rev ->
+ % Start/End keys swapped on purpose because ebtree
+ ebtree:reverse_range(Tx, Btree, EndKey, StartKey, Wrapper, Acc0)
+ end.
+
+
+update_views(TxDb, Mrst, Docs) ->
+ #{
+ tx := Tx
+ } = TxDb,
+
+ % Collect update information
+ #{
+ ids := IdMap,
+ views := ViewMaps,
+ delete_ref := DeleteRef
+ } = gather_update_info(Tx, Mrst, Docs),
+
+ % Update the IdBtree
+ update_btree(Tx, Mrst#mrst.id_btree, IdMap, DeleteRef),
+
+ % Update each view's BTree
+ lists:foreach(fun(View) ->
+ #mrview{
+ id_num = ViewId,
+ btree = BTree
+ } = View,
+
+ ViewMap = maps:get(ViewId, ViewMaps, #{}),
+ update_btree(Tx, BTree, ViewMap, DeleteRef)
+ end, Mrst#mrst.views).
+
+
+open_id_tree(TxDb, Sig) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ Prefix = id_tree_prefix(DbPrefix, Sig),
+ TreeOpts = [
+ {persist_fun, fun couch_views_fdb:persist_chunks/3},
+ {cache_fun, create_cache_fun(id_tree)}
+ ],
+ ebtree:open(Tx, Prefix, get_order(id_btree), TreeOpts).
+
+
+open_view_tree(TxDb, Sig, Lang, View) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ #mrview{
+ id_num = ViewId
+ } = View,
+ Prefix = view_tree_prefix(DbPrefix, Sig, ViewId),
+ TreeOpts = [
+ {collate_fun, couch_views_util:collate_fun(View)},
+ {reduce_fun, make_reduce_fun(Lang, View)},
+ {persist_fun, fun couch_views_fdb:persist_chunks/3},
+ {cache_fun, create_cache_fun({view, ViewId})}
+ ],
+ View#mrview{
+ btree = ebtree:open(Tx, Prefix, get_order(view_btree), TreeOpts)
+ }.
+
+
+get_order(id_btree) ->
+ min_order(config:get_integer("couch_views", "id_btree_node_size", 100));
+get_order(view_btree) ->
+ min_order(config:get_integer("couch_views", "view_btree_node_size", 100)).
+
+
+min_order(V) when is_integer(V), V < 2 ->
+ 2;
+min_order(V) when is_integer(V), V rem 2 == 0 ->
+ V;
+min_order(V) ->
+ V + 1.
+
+
+make_reduce_fun(_Lang, #mrview{}) ->
+ fun
+ (KVs, _ReReduce = false) ->
+ TotalSize = lists:foldl(fun({K, V}, Acc) ->
+ KSize = couch_ejson_size:encoded_size(K),
+ VSize = case V of
+ {dups, Dups} ->
+ lists:foldl(fun(D, DAcc) ->
+ DAcc + couch_ejson_size:encoded_size(D)
+ end, 0, Dups);
+ _ ->
+ couch_ejson_size:encoded_size(V)
+ end,
+ KSize + VSize + Acc
+ end, 0, KVs),
+ {length(KVs), TotalSize};
+ (KRs, _ReReduce = true) ->
+ lists:foldl(fun({Count, Size}, {CountAcc, SizeAcc}) ->
+ {Count + CountAcc, Size + SizeAcc}
+ end, {0, 0}, KRs)
+ end.
+
+
+create_cache_fun(TreeId) ->
+ CacheTid = case get(TreeId) of
+ undefined ->
+ Tid = ets:new(?MODULE, [protected, set]),
+ put(TreeId, {ebtree_cache, Tid}),
+ Tid;
+ {ebtree_cache, Tid} ->
+ Tid
+ end,
+ fun
+ (set, [Id, Node]) ->
+ true = ets:insert_new(CacheTid, {Id, Node}),
+ ok;
+ (clear, Id) ->
+ ets:delete(CacheTid, Id),
+ ok;
+ (get, Id) ->
+ case ets:lookup(CacheTid, Id) of
+ [{Id, Node}] -> Node;
+ [] -> undefined
+ end
+ end.
+
+
+to_map_opts(Options) ->
+ Dir = case lists:keyfind(dir, 1, Options) of
+ {dir, D} -> D;
+ _ -> fwd
+ end,
+
+ InclusiveEnd = case lists:keyfind(inclusive_end, 1, Options) of
+ {inclusive_end, IE} -> IE;
+ _ -> true
+ end,
+
+ StartKey = case lists:keyfind(start_key, 1, Options) of
+ {start_key, SK} -> SK;
+ false when Dir == fwd -> ebtree:min();
+ false when Dir == rev -> ebtree:max()
+ end,
+
+ EndKey = case lists:keyfind(end_key, 1, Options) of
+ {end_key, EK} -> EK;
+ false when Dir == fwd -> ebtree:max();
+ false when Dir == rev -> ebtree:min()
+ end,
+
+ {Dir, StartKey, EndKey, InclusiveEnd}.
+
+
+gather_update_info(Tx, Mrst, Docs) ->
+ % A special token used to indicate that the row should be deleted
+ DeleteRef = erlang:make_ref(),
+
+ AllDocIds = [DocId || #{id := DocId} <- Docs],
+
+ BaseIdMap = lists:foldl(fun(DocId, Acc) ->
+ maps:put(DocId, DeleteRef, Acc)
+ end, #{}, AllDocIds),
+
+ % Build the initial set of rows to delete
+ % ExistingViewKeys is a list of {DocId, [{ViewId, [Key | _]} | _]}
+ ExistingViewKeys = ebtree:lookup_multi(Tx, Mrst#mrst.id_btree, AllDocIds),
+
+ % For each view, create an initial map that contains the
+ % list of keys to delete. The final result is a map of
+ % maps:
+ % #{ViewId => #{Key => DeleteRef}}
+ BaseViewMaps = lists:foldl(fun({DocId, ViewIdKeys}, ViewIdAcc1) ->
+ lists:foldl(fun({ViewId, Keys}, ViewIdAcc2) ->
+ OldViewMap = maps:get(ViewId, ViewIdAcc2, #{}),
+ NewViewMap = lists:foldl(fun(Key, ViewMapAcc) ->
+ maps:put({Key, DocId}, DeleteRef, ViewMapAcc)
+ end, OldViewMap, Keys),
+ maps:put(ViewId, NewViewMap, ViewIdAcc2)
+ end, ViewIdAcc1, ViewIdKeys)
+ end, #{}, ExistingViewKeys),
+
+ % Build our base accumulator
+ InfoAcc1 = #{
+ ids => BaseIdMap,
+ views => BaseViewMaps,
+ delete_ref => DeleteRef
+ },
+
+ % Insert results from each document into the map of
+ % maps which leaves us with a final shape of:
+ % #{ViewId => #{Key => Value}}
+ % where Value may be a copy of `DeleteRef` which flags
+ % that the Key should be deleted from the view.
+ lists:foldl(fun(Doc, InfoAcc2) ->
+ insert_doc(Mrst, Doc, InfoAcc2)
+ end, InfoAcc1, Docs).
+
+
+insert_doc(_Mrst, #{deleted := true} = _Doc, InfoAcc) ->
+ InfoAcc;
+insert_doc(Mrst, Doc, InfoAcc0) ->
+ #{
+ id := DocId,
+ results := Results
+ } = Doc,
+
+ FinalAcc = lists:foldl(fun({View, RawNewRows}, {IdKeyAcc, InfoAcc1}) ->
+ #mrview{
+ id_num = ViewId
+ } = View,
+ #{
+ views := ViewMaps
+ } = InfoAcc1,
+
+ DedupedRows = dedupe_rows(View, RawNewRows),
+ IdKeys = lists:usort([K || {K, _V} <- DedupedRows]),
+
+ OldViewMap = maps:get(ViewId, ViewMaps, #{}),
+ NewViewMap = lists:foldl(fun({K, V}, ViewMapAcc) ->
+ maps:put({K, DocId}, V, ViewMapAcc)
+ end, OldViewMap, DedupedRows),
+
+ {[{ViewId, IdKeys} | IdKeyAcc], InfoAcc1#{
+ views := maps:put(ViewId, NewViewMap, ViewMaps)
+ }}
+ end, {[], InfoAcc0}, lists:zip(Mrst#mrst.views, Results)),
+
+ {IdRows, #{ids := IdMap} = InfoAcc2} = FinalAcc,
+
+ % Don't store a row in the id_btree if it hasn't got any
+ % keys that will need to be deleted.
+ NonEmptyRows = [1 || {_ViewId, Rows} <- IdRows, Rows /= []],
+ if length(NonEmptyRows) == 0 -> InfoAcc2; true ->
+ InfoAcc2#{ids := maps:put(DocId, IdRows, IdMap)}
+ end.
+
+
+update_btree(Tx, BTree, Map, DeleteRef) ->
+ {ToRemove, ToInsert} = maps:fold(fun(Key, Value, {Keys, Rows}) ->
+ case Value of
+ DeleteRef -> {[Key | Keys], Rows};
+ _ -> {Keys, [{Key, Value} | Rows]}
+ end
+ end, {[], []}, Map),
+
+ lists:foreach(fun(Key) ->
+ ebtree:delete(Tx, BTree, Key)
+ end, ToRemove),
+
+ ebtree:insert_multi(Tx, BTree, ToInsert).
+
+
+dedupe_rows(View, KVs0) ->
+ CollateFun = couch_views_util:collate_fun(View),
+ KVs1 = lists:sort(fun({KeyA, ValA}, {KeyB, ValB}) ->
+ case CollateFun({KeyA, <<>>}, {KeyB, <<>>}) of
+ lt -> true;
+ eq -> ValA =< ValB;
+ gt -> false
+ end
+ end, KVs0),
+ dedupe_rows_int(CollateFun, KVs1).
+
+
+dedupe_rows_int(_CollateFun, []) ->
+ [];
+
+dedupe_rows_int(_CollateFun, [KV]) ->
+ [KV];
+
+dedupe_rows_int(CollateFun, [{K1, V1} | RestKVs]) ->
+ RestDeduped = dedupe_rows_int(CollateFun, RestKVs),
+ case RestDeduped of
+ [{K2, V2} | RestRestDeduped] ->
+ case CollateFun({K1, <<>>}, {K2, <<>>}) of
+ eq -> [{K1, combine_vals(V1, V2)} | RestRestDeduped];
+ _ -> [{K1, V1} | RestDeduped]
+ end;
+ [] ->
+ [{K1, V1}]
+ end.
+
+
+combine_vals(V1, {dups, V2}) ->
+ {dups, [V1 | V2]};
+combine_vals(V1, V2) ->
+ {dups, [V1, V2]}.
+
+
+id_tree_prefix(DbPrefix, Sig) ->
+ Key = {?DB_VIEWS, ?VIEW_TREES, Sig, ?VIEW_ID_TREE},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+view_tree_prefix(DbPrefix, Sig, ViewId) ->
+ Key = {?DB_VIEWS, ?VIEW_TREES, Sig, ?VIEW_ROW_TREES, ViewId},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+dedupe_basic_test() ->
+ View = #mrview{},
+ ?assertEqual([{1, 1}], dedupe_rows(View, [{1, 1}])).
+
+dedupe_simple_test() ->
+ View = #mrview{},
+ ?assertEqual([{1, {dups, [1, 2]}}], dedupe_rows(View, [{1, 1}, {1, 2}])).
+
+-endif.
diff --git a/src/couch_views/src/couch_views_updater.erl b/src/couch_views/src/couch_views_updater.erl
index ba9fadb..7e5466e 100644
--- a/src/couch_views/src/couch_views_updater.erl
+++ b/src/couch_views/src/couch_views_updater.erl
@@ -87,16 +87,17 @@ write_doc(Db, #doc{deleted = Deleted} = Doc) ->
},
lists:foreach(fun(DDoc) ->
- {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+ {ok, Mrst0} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+ Mrst1 = couch_views_trees:open(Db, Mrst0),
- case should_index_doc(Doc, Mrst) of
+ case should_index_doc(Doc, Mrst1) of
true ->
- {Mrst1, Result1} = couch_views_indexer:map_docs(Mrst, Result0),
- DocNumber = couch_views_indexer:write_docs(Db, Mrst1,
+ {Mrst2, Result1} = couch_views_indexer:map_docs(Mrst1, Result0),
+ DocNumber = couch_views_indexer:write_docs(Db, Mrst2,
Result1, State),
- couch_views_plugin:after_interactive_write(Db, Mrst1,
+ couch_views_plugin:after_interactive_write(Db, Mrst2,
Result1, DocNumber),
- couch_eval:release_map_context(Mrst1#mrst.qserver);
+ couch_eval:release_map_context(Mrst2#mrst.qserver);
false ->
ok
end
diff --git a/src/couch_views/src/couch_views_util.erl b/src/couch_views/src/couch_views_util.erl
index 6298acf..1e3e4be 100644
--- a/src/couch_views/src/couch_views_util.erl
+++ b/src/couch_views/src/couch_views_util.erl
@@ -15,6 +15,7 @@
-export([
ddoc_to_mrst/2,
+ collate_fun/1,
validate_args/1,
validate_args/2,
is_paginated/1,
@@ -82,6 +83,40 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
{ok, IdxState#mrst{sig=couch_hash:md5_hash(term_to_binary(SigInfo))}}.
+collate_fun(View) ->
+ #mrview{
+ options = Options
+ } = View,
+ case couch_util:get_value(<<"collation">>, Options) of
+ <<"raw">> -> fun collate_raw/2;
+ _ -> fun collate_rows/2
+ end.
+
+
+collate_raw(A, A) -> eq;
+collate_raw(A, B) when A < B -> lt;
+collate_raw(A, B) when A > B -> gt.
+
+
+collate_rows({KeyA, DocIdA}, {KeyB, DocIdB}) ->
+ case couch_ejson_compare:less(KeyA, KeyB) of
+ N when N < 0 -> lt;
+ 0 when DocIdA < DocIdB -> lt;
+ 0 when DocIdA == DocIdB -> eq;
+ 0 -> gt; % when DocIdA > DocIdB
+ N when N > 0 -> gt
+ end;
+
+collate_rows(KeyA, KeyB) ->
+ % When collating reduce group keys they don't
+ % come with a docid.
+ case couch_ejson_compare:less(KeyA, KeyB) of
+ N when N < 0 -> lt;
+ 0 -> eq;
+ N when N > 0 -> gt
+ end.
+
+
validate_args(Args) ->
validate_args(Args, []).
diff --git a/src/couch_views/test/couch_views_cleanup_test.erl b/src/couch_views/test/couch_views_cleanup_test.erl
index e4dcdce..54048c9 100644
--- a/src/couch_views/test/couch_views_cleanup_test.erl
+++ b/src/couch_views/test/couch_views_cleanup_test.erl
@@ -302,7 +302,7 @@ view_has_data(Db, DDoc) ->
SigKey = erlfdb_tuple:pack(SigKeyTuple, DbPrefix),
SigVal = erlfdb:wait(erlfdb:get(Tx, SigKey)),
- RangeKeyTuple = {?DB_VIEWS, ?VIEW_DATA, Sig},
+ RangeKeyTuple = {?DB_VIEWS, ?VIEW_TREES, Sig},
RangeKey = erlfdb_tuple:pack(RangeKeyTuple, DbPrefix),
Range = erlfdb:wait(erlfdb:get_range_startswith(Tx, RangeKey)),
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index 86c0a81..75be245 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -126,13 +126,12 @@ updated_docs_are_reindexed(Db) ->
% Check that our id index is updated properly
% as well.
DbName = fabric2_db:name(Db),
- {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
- Sig = Mrst#mrst.sig,
+ {ok, Mrst0} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
fabric2_fdb:transactional(Db, fun(TxDb) ->
- ?assertMatch(
- [{0, 1, _, [1]}],
- couch_views_fdb:get_view_keys(TxDb, Sig, <<"0">>)
- )
+ #{tx := Tx} = TxDb,
+ Mrst1 = couch_views_trees:open(TxDb, Mrst0),
+ IdRow = ebtree:lookup(Tx, Mrst1#mrst.id_btree, <<"0">>),
+ ?assertEqual({<<"0">>, [{1, []}, {0, [1]}]}, IdRow)
end).
@@ -160,13 +159,12 @@ updated_docs_without_changes_are_reindexed(Db) ->
% Check fdb directly to make sure we've also
% removed the id idx keys properly.
DbName = fabric2_db:name(Db),
- {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
- Sig = Mrst#mrst.sig,
+ {ok, Mrst0} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
fabric2_fdb:transactional(Db, fun(TxDb) ->
- ?assertMatch(
- [{0, 1, _, [0]}],
- couch_views_fdb:get_view_keys(TxDb, Sig, <<"0">>)
- )
+ #{tx := Tx} = TxDb,
+ Mrst1 = couch_views_trees:open(TxDb, Mrst0),
+ IdRow = ebtree:lookup(Tx, Mrst1#mrst.id_btree, <<"0">>),
+ ?assertEqual({<<"0">>, [{1, []}, {0, [0]}]}, IdRow)
end).
@@ -208,10 +206,12 @@ deleted_docs_are_unindexed(Db) ->
% Check fdb directly to make sure we've also
% removed the id idx keys properly.
DbName = fabric2_db:name(Db),
- {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
- Sig = Mrst#mrst.sig,
+ {ok, Mrst0} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
fabric2_fdb:transactional(Db, fun(TxDb) ->
- ?assertEqual([], couch_views_fdb:get_view_keys(TxDb, Sig, <<"0">>))
+ #{tx := Tx} = TxDb,
+ Mrst1 = couch_views_trees:open(TxDb, Mrst0),
+ IdRow = ebtree:lookup(Tx, Mrst1#mrst.id_btree, <<"0">>),
+ ?assertEqual(false, IdRow)
end).
@@ -296,11 +296,9 @@ fewer_multipe_identical_keys_from_same_doc(Db) ->
handle_size_key_limits(Db) ->
ok = meck:new(config, [passthrough]),
- ok = meck:expect(config, get_integer, fun(Section, Key, Default) ->
- case Section == "couch_views" andalso Key == "key_size_limit" of
- true -> 15;
- _ -> Default
- end
+ ok = meck:expect(config, get_integer, fun
+ ("couch_views", "key_size_limit", _Default) -> 15;
+ (_Section, _Key, Default) -> Default
end),
DDoc = create_ddoc(multi_emit_key_limit),
@@ -328,11 +326,9 @@ handle_size_key_limits(Db) ->
handle_size_value_limits(Db) ->
ok = meck:new(config, [passthrough]),
- ok = meck:expect(config, get_integer, fun(Section, _, Default) ->
- case Section of
- "couch_views" -> 15;
- _ -> Default
- end
+ ok = meck:expect(config, get_integer, fun
+ ("couch_views", "value_size_limit", _Default) -> 15;
+ (_Section, _Key, Default) -> Default
end),
DDoc = create_ddoc(multi_emit_key_limit),
@@ -386,12 +382,6 @@ multiple_design_docs(Db) ->
end)
end,
- % This is how we check that no index updates took place
- meck:new(couch_views_fdb, [passthrough]),
- meck:expect(couch_views_fdb, write_doc, fun(TxDb, Sig, ViewIds, Doc) ->
- meck:passthrough([TxDb, Sig, ViewIds, Doc])
- end),
-
DDoc1 = create_ddoc(simple, <<"_design/bar1">>),
DDoc2 = create_ddoc(simple, <<"_design/bar2">>),
@@ -399,7 +389,7 @@ multiple_design_docs(Db) ->
{ok, {Pos1, Rev1}} = fabric2_db:update_doc(Db, DDoc1, []),
?assertEqual({ok, [row(<<"0">>, 0, 0)]}, run_query(Db, DDoc1, ?MAP_FUN1)),
- % Because run_query/3 can return, and unsurbscribe from the job,
+ % Because run_query/3 can return, and unsubscribe from the job,
% before it actually finishes, ensure we wait for the job to
% finish so we get a deterministic setup every time.
JobId = get_job_id(Db, DDoc1),
@@ -413,10 +403,16 @@ multiple_design_docs(Db) ->
Cleanup(),
- meck:reset(couch_views_fdb),
+ % Assert that no updates are applied
+ meck:new(couch_views_fdb, [passthrough]),
+ meck:expect(couch_views_trees, update_views, fun(TxDb, Mrst, Docs) ->
+ case Docs of
+ [] -> meck:passthrough([TxDb, Mrst, Docs]);
+ [_ | _] -> erlang:error(update_triggered)
+ end
+ end),
?assertEqual({ok, [row(<<"0">>, 0, 0)]}, run_query(Db, DDoc2, ?MAP_FUN1)),
?assertEqual(ok, wait_job_finished(JobId, 5000)),
- ?assertEqual(0, meck:num_calls(couch_views_fdb, write_doc, 4)),
DDoc2Del = DDoc2#doc{revs = {Pos2, [Rev2]}, deleted = true},
{ok, _} = fabric2_db:update_doc(Db, DDoc2Del, []),
diff --git a/src/couch_views/test/couch_views_size_test.erl b/src/couch_views/test/couch_views_size_test.erl
index 18fa9e6..cc2fe39 100644
--- a/src/couch_views/test/couch_views_size_test.erl
+++ b/src/couch_views/test/couch_views_size_test.erl
@@ -193,16 +193,21 @@ cleanup({Ctx, Db}) ->
create_transition_tests({_Ctx, Db}) ->
- Transitions = generate_transitions(),
- Single = lists:flatmap(fun(T) ->
- Name = lists:flatten(io_lib:format("single ~s", [tname(T)])),
- [{Name, fun() -> check_single_transition(Db, T) end}]
- end, lists:sort(Transitions)),
- Multi = lists:flatmap(fun(T) ->
- Name = lists:flatten(io_lib:format("multi ~s", [tname(T)])),
- [{Name, fun() -> check_multi_transition(Db, T) end}]
- end, lists:sort(group(shuffle(Transitions)))),
- subset(?NUM_SINGLE_TESTS, Single) ++ subset(?NUM_MULTI_TESTS, Multi).
+ try
+ throw(disabled),
+ Transitions = generate_transitions(),
+ Single = lists:flatmap(fun(T) ->
+ Name = lists:flatten(io_lib:format("single ~s", [tname(T)])),
+ [{Name, fun() -> check_single_transition(Db, T) end}]
+ end, lists:sort(Transitions)),
+ Multi = lists:flatmap(fun(T) ->
+ Name = lists:flatten(io_lib:format("multi ~s", [tname(T)])),
+ [{Name, fun() -> check_multi_transition(Db, T) end}]
+ end, lists:sort(group(shuffle(Transitions)))),
+ subset(?NUM_SINGLE_TESTS, Single) ++ subset(?NUM_MULTI_TESTS, Multi)
+ catch throw:disabled ->
+ [{"Disabled", fun() -> ok end}]
+ end.
check_single_transition(Db, {Set1, Set2, Transition}) ->
diff --git a/src/couch_views/test/couch_views_updater_test.erl b/src/couch_views/test/couch_views_updater_test.erl
index 89c341a..aadbe94 100644
--- a/src/couch_views/test/couch_views_updater_test.erl
+++ b/src/couch_views/test/couch_views_updater_test.erl
@@ -69,7 +69,7 @@ foreach_setup() ->
Docs = make_docs(3),
fabric2_db:update_docs(Db, Docs),
- meck:new(couch_views_fdb, [passthrough]),
+ meck:new(couch_views_trees, [passthrough]),
{Db, DDoc}.
@@ -135,7 +135,7 @@ includes_design_docs({Db, _}) ->
handle_erlfdb_errors({Db, _}) ->
- meck:expect(couch_views_fdb, write_doc, fun(_, _, _, _) ->
+ meck:expect(couch_views_trees, update_views, fun(_, _, _) ->
error({erlfdb_error, 1009})
end),
?assertError({erlfdb_error, 1009}, fabric2_db:update_docs(Db, [doc(4)])).
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 44ae220..411f4af 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -31,6 +31,7 @@
-include_lib("fabric/include/fabric.hrl").
-include("mango_cursor.hrl").
+-include("mango_idx_view.hrl").
create(Db, Indexes, Selector, Opts) ->
@@ -85,16 +86,15 @@ explain(Cursor) ->
maybe_replace_max_json([]) ->
[];
+maybe_replace_max_json([?MAX_JSON_OBJ | T]) ->
+ [<<"<MAX>">> | maybe_replace_max_json(T)];
+
+maybe_replace_max_json([H | T]) ->
+ [H | maybe_replace_max_json(T)];
+
maybe_replace_max_json(?MAX_STR) ->
<<"<MAX>">>;
-maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
- MAX_VAL = couch_views_encoding:max(),
- H1 = if H == MAX_VAL -> <<"<MAX>">>;
- true -> H
- end,
- [H1 | maybe_replace_max_json(T)];
-
maybe_replace_max_json(EndKey) ->
EndKey.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index f80cc21..a73d82a 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,6 +34,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include("mango_idx_view.hrl").
validate_new(#idx{}=Idx, _Db) ->
@@ -131,7 +132,7 @@ is_usable(Idx, Selector, SortFields) ->
% and the selector is not a text search (so requires a text index)
RequiredFields = columns(Idx),
- % sort fields are required to exist in the results so
+ % sort fields are required to exist in the results so
% we don't need to check the selector for these
RequiredFields1 = ordsets:subtract(lists:usort(RequiredFields), lists:usort(SortFields)),
@@ -182,11 +183,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [couch_views_encoding:max()];
+ [?MAX_JSON_OBJ];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
- [couch_views_encoding:max()];
+ [?MAX_JSON_OBJ];
false ->
[Key | end_key(Rest)]
end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
new file mode 100644
index 0000000..d0f4674
--- /dev/null
+++ b/src/mango/src/mango_idx_view.hrl
@@ -0,0 +1,13 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-define(MAX_JSON_OBJ, {[{<<255, 255, 255, 255>>, <<>>}]}).
\ No newline at end of file
[couchdb] 04/07: Reimplement db wide view size tracking
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ba2a42633616bd435e47d3445dac0b7892ec9176
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Sep 3 12:05:45 2020 -0500
Reimplement db wide view size tracking
---
src/couch_views/src/couch_views_fdb.erl | 27 +-
src/couch_views/src/couch_views_trees.erl | 30 +-
src/couch_views/test/couch_views_size_test.erl | 829 +++++++++----------------
3 files changed, 357 insertions(+), 529 deletions(-)
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index e813f2b..a9dbe3e 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -135,6 +135,11 @@ clear_index(Db, Signature) ->
db_prefix := DbPrefix
} = Db,
+ % Get view size to remove from global counter
+ SizeTuple = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Signature},
+ SizeKey = erlfdb_tuple:pack(SizeTuple, DbPrefix),
+ ViewSize = ?bin2uint(erlfdb:wait(erlfdb:get(Tx, SizeKey))),
+
% Clear index info keys
Keys = [
{?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Signature},
@@ -154,7 +159,12 @@ clear_index(Db, Signature) ->
% Clear tree data
TreeTuple = {?DB_VIEWS, ?VIEW_TREES, Signature},
TreePrefix = erlfdb_tuple:pack(TreeTuple, DbPrefix),
- erlfdb:clear_range_startswith(Tx, TreePrefix).
+ erlfdb:clear_range_startswith(Tx, TreePrefix),
+
+ % Decrement db wide view size counter
+ DbSizeTuple = {?DB_STATS, <<"sizes">>, <<"views">>},
+ DbSizeKey = erlfdb_tuple:pack(DbSizeTuple, DbPrefix),
+ erlfdb:add(Tx, DbSizeKey, -ViewSize).
persist_chunks(Tx, set, [Key, Value]) ->
@@ -181,6 +191,21 @@ persist_chunks(Tx, clear, Key) ->
erlfdb:clear_range_startswith(Tx, Key).
+update_kv_size(TxDb, Sig, OldSize, NewSize) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+
+ ViewTuple = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Sig},
+ ViewKey = erlfdb_tuple:pack(ViewTuple, DbPrefix),
+ erlfdb:set(Tx, ViewKey, ?uint2bin(NewSize)),
+
+ DbTuple = {?DB_STATS, <<"sizes">>, <<"views">>},
+ DbKey = erlfdb_tuple:pack(DbTuple, DbPrefix),
+ erlfdb:add(Tx, DbKey, NewSize - OldSize).
+
+
seq_key(DbPrefix, Sig) ->
Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Sig},
erlfdb_tuple:pack(Key, DbPrefix).
diff --git a/src/couch_views/src/couch_views_trees.erl b/src/couch_views/src/couch_views_trees.erl
index d67ccd5..85e1b91 100644
--- a/src/couch_views/src/couch_views_trees.erl
+++ b/src/couch_views/src/couch_views_trees.erl
@@ -127,6 +127,12 @@ update_views(TxDb, Mrst, Docs) ->
tx := Tx
} = TxDb,
+ % Get initial KV size
+ OldKVSize = lists:foldl(fun(View, SizeAcc) ->
+ {_, Size} = ebtree:full_reduce(Tx, View#mrview.btree),
+ SizeAcc + Size
+ end, 0, Mrst#mrst.views),
+
% Collect update information
#{
ids := IdMap,
@@ -146,7 +152,15 @@ update_views(TxDb, Mrst, Docs) ->
ViewMap = maps:get(ViewId, ViewMaps, #{}),
update_btree(Tx, BTree, ViewMap, DeleteRef)
- end, Mrst#mrst.views).
+ end, Mrst#mrst.views),
+
+ % Get new KV size after update
+ NewKVSize = lists:foldl(fun(View, SizeAcc) ->
+ {_, Size} = ebtree:full_reduce(Tx, View#mrview.btree),
+ SizeAcc + Size
+ end, 0, Mrst#mrst.views),
+
+ couch_views_fdb:update_kv_size(TxDb, Mrst#mrst.sig, OldKVSize, NewKVSize).
open_id_tree(TxDb, Sig) ->
@@ -200,16 +214,18 @@ make_reduce_fun(_Lang, #mrview{}) ->
fun
(KVs, _ReReduce = false) ->
TotalSize = lists:foldl(fun({K, V}, Acc) ->
- KSize = couch_ejson_size:encoded_size(K),
- VSize = case V of
+ Acc + case V of
{dups, Dups} ->
lists:foldl(fun(D, DAcc) ->
- DAcc + couch_ejson_size:encoded_size(D)
+ KSize = couch_ejson_size:encoded_size(K),
+ VSize = couch_ejson_size:encoded_size(D),
+ DAcc + KSize + VSize
end, 0, Dups);
_ ->
- couch_ejson_size:encoded_size(V)
- end,
- KSize + VSize + Acc
+ KSize = couch_ejson_size:encoded_size(K),
+ VSize = couch_ejson_size:encoded_size(V),
+ KSize + VSize
+ end
end, 0, KVs),
{length(KVs), TotalSize};
(KRs, _ReReduce = true) ->
diff --git a/src/couch_views/test/couch_views_size_test.erl b/src/couch_views/test/couch_views_size_test.erl
index cc2fe39..16537a3 100644
--- a/src/couch_views/test/couch_views_size_test.erl
+++ b/src/couch_views/test/couch_views_size_test.erl
@@ -16,162 +16,38 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch_mrview/include/couch_mrview.hrl").
--include_lib("fabric/include/fabric2.hrl").
-include_lib("couch_views/include/couch_views.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
-% N.B., we should move to couch_ejson_size instead
-% of erlang:external_size
-%
-% to calculate view size:
-% total = 0
-% for (fdb_k, fdb_v) in VIEW_MAP_RANGE:
-% {EncUserKey, EncUserval} = erlfdb_tuple:unpack(fdb_v),
-% UserKey = couch_views_encoding:decode(EncUserKey),
-% UserVal = couch_views_encoding:decode(EncUserVal),
-% total += erlang:external_size(UserKey),
-% total += erlang:external_size(UserVal)
-%
-% Our goal in checking the size calculations is that we cover
-% as much of the possible key mutation space as possible while
-% not relying on fuzzing out the edge cases. Conceptually we have
-% two sets of keys E and U. E is keys as currently exist in the
-% view, and U is the new set of keys corresponding to an update.
-%
-% Both sets E and U have the same possible set of state variables:
-%
-% 1. N unique keys, where 0 =< N =< infinity
-% 2. D keys with duplicates, where 0 =< D =< N,
-% 3. R repeats for each member of D, for 2 =< R =< infinity
-%
-% Given two sets S1 and S2, we then have a set of transition variables:
-%
-% 1. deltaN - shared unique keys, where 0 =< deltaN =< N
-% 2. deltaD - shared duplicates, where 0 =< deltaD =< N
-% 3. deltaR - shared repeats for each D, where 2 =< deltaR =< infinity
-%
-% To search our state transition space, we can create two functions to
-% first define our start and end states, and for each transition we have
-% a function that defines the shared overlap between states.
-%
-% Given a list of transitions are checks then become simple in that
-% we can iterate over each transition checking that our index is valid
-% after each one. Index validation will purely look at the existing
-% state of the index in fdb and validate correctness.
-
--define(NUM_SINGLE_TESTS, 100).
--define(NUM_MULTI_TESTS, 100).
-
--define(N_DOMAIN, [0, 1, 2, 5]).
--define(D_DOMAIN, [0, 1, 2, 5]).
--define(R_DOMAIN, [2, 4]).
-
--define(DELTA_N_DOMAIN, [0, 1, 2, 5]).
--define(DELTA_D_DOMAIN, [0, 1, 2, 5]).
--define(DELTA_R_DOMAIN, [1, 2, 4]).
-
-
-generate_sets() ->
- permute(?N_DOMAIN, ?D_DOMAIN, ?R_DOMAIN, fun(N, D, R) ->
- % We can't have more duplicates than total keys
- case D > N of
- true -> throw(skip);
- false -> ok
- end,
-
- % Only include one of the repeat values
- % for our zero sets
- case D == 0 of
- true when R == 2 -> ok;
- true -> throw(skip);
- false -> ok
- end,
-
- % Replace R with a sentinel value for sanity
- % when there are no dupes to have repeats
- ActualR = if D == 0 -> 0; true -> R end,
-
- {N, D, ActualR}
- end).
-
-
-generate_transitions() ->
- Sets = generate_sets(),
- Pairs = [{Set1, Set2} || Set1 <- Sets, Set2 <- Sets],
- lists:flatmap(fun({{N1, D1, _R1} = S1, {N2, D2, _R2} = S2}) ->
- Filter = fun(DeltaN, DeltaD, DeltaR) ->
- % Can't share more keys than the smaller of the
- % two sets
- case DeltaN > min(N1, N2) of
- true -> throw(skip);
- false -> ok
- end,
-
- % For DeltaD == 0, all combinations of DeltaD and
- % DeltaR are equivalent tests
- case DeltaN == 0 of
- true when DeltaD == 0, DeltaR == 1 -> ok;
- true -> throw(skip);
- false -> ok
- end,
-
- % Can't share more dupes than exist in either set
- % or the total number of shared keys
- case DeltaD > min(D1, D2) orelse DeltaD > DeltaN of
- true -> throw(skip);
- false -> ok
- end,
-
- % For DeltaD == 0, all DeltaR correspond to the
- % same test so only include one instance
- case DeltaD == 0 of
- true when DeltaR == 1 -> ok;
- true -> throw(skip);
- false -> ok
- end,
-
- % If we have more non-repeated keys in our
- % transition than there's "room" for in the target
- % set it isn't a valid test case.
- TransitionNonRepeats = DeltaN - DeltaD,
- TargetNonRepeats = N2 - D2,
- case TransitionNonRepeats > TargetNonRepeats of
- true -> throw(skip);
- false -> ok
- end,
-
- {S1, S2, {DeltaN, DeltaD, DeltaR}}
- end,
- permute(?DELTA_N_DOMAIN, ?DELTA_D_DOMAIN, ?DELTA_R_DOMAIN, Filter)
- end, Pairs).
-
-
-permute(NList, DList, RList, Filter) ->
- % Technically we could call into Filter in each
- % outer loops to conditionally skip inner loops.
- % If someone comes along looking to speed up the
- % fixture setup time, this would likely be an
- % easy win.
- lists:foldl(fun(N, NAcc) ->
- lists:foldl(fun(D, DAcc) ->
- lists:foldl(fun(R, RAcc) ->
- try
- [Filter(N, D, R) | RAcc]
- catch throw:skip ->
- RAcc
- end
- end, DAcc, RList)
- end, NAcc, DList)
- end, [], NList).
-
-
-row_transition_test_() ->
+
+-define(MAP_FUN1, <<"map_fun1">>).
+-define(MAP_FUN2, <<"map_fun2">>).
+
+
+indexer_test_() ->
{
- "Test view size tracking",
+ "Test view indexing",
{
setup,
fun setup/0,
fun cleanup/1,
- fun create_transition_tests/1
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [
+ ?TDEF_FE(empty_view),
+ ?TDEF_FE(single_doc),
+ ?TDEF_FE(multiple_docs),
+ ?TDEF_FE(update_no_size_change),
+ ?TDEF_FE(update_increases_size),
+ ?TDEF_FE(update_decreases_size),
+ ?TDEF_FE(deleting_docs_decreases_size),
+ ?TDEF_FE(multi_identical_keys_count_twice),
+ ?TDEF_FE(multiple_design_docs),
+ ?TDEF_FE(multiple_identical_design_docs)
+ ]
+ }
}
}.
@@ -183,387 +59,298 @@ setup() ->
couch_js,
couch_views
]),
- {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
- {Ctx, Db}.
+ Ctx.
-cleanup({Ctx, Db}) ->
- ok = fabric2_db:delete(fabric2_db:name(Db), []),
+cleanup(Ctx) ->
test_util:stop_couch(Ctx).
-create_transition_tests({_Ctx, Db}) ->
- try
- throw(disabled),
- Transitions = generate_transitions(),
- Single = lists:flatmap(fun(T) ->
- Name = lists:flatten(io_lib:format("single ~s", [tname(T)])),
- [{Name, fun() -> check_single_transition(Db, T) end}]
- end, lists:sort(Transitions)),
- Multi = lists:flatmap(fun(T) ->
- Name = lists:flatten(io_lib:format("multi ~s", [tname(T)])),
- [{Name, fun() -> check_multi_transition(Db, T) end}]
- end, lists:sort(group(shuffle(Transitions)))),
- subset(?NUM_SINGLE_TESTS, Single) ++ subset(?NUM_MULTI_TESTS, Multi)
- catch throw:disabled ->
- [{"Disabled", fun() -> ok end}]
- end.
-
-
-check_single_transition(Db, {Set1, Set2, Transition}) ->
- clear_views(Db),
- InitKVs = init_set(Set1, [a, b, c, d, e]),
- CommonKVs = reduce_set(Transition, InitKVs),
- FinalKVs = fill_set(Set2, CommonKVs, [v, w, x, y, z]),
- {InitJSONKVs, Bindings} = unlabel(InitKVs, #{}),
- {FinalJSONKVs, _} = unlabel(FinalKVs, Bindings),
-
- Sig = couch_uuids:random(),
- DocId = couch_uuids:random(),
-
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- write_docs(TxDb, Sig, [make_doc(DocId, InitJSONKVs)])
- end),
-
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- write_docs(TxDb, Sig, [make_doc(DocId, FinalJSONKVs)])
- end),
-
- validate_index(Db, Sig, #{DocId => FinalJSONKVs}).
-
-
-check_multi_transition(Db, Transitions) ->
- clear_views(Db),
-
- {Docs, IdMap} = lists:mapfoldl(fun({Set1, Set2, Transition}, IdMapAcc) ->
- DocId = couch_uuids:random(),
- InitKVs = init_set(Set1, [a, b, c, d, e]),
- CommonKVs = reduce_set(Transition, InitKVs),
- FinalKVs = fill_set(Set2, CommonKVs, [v, w, x, y, z]),
- {InitJSONKVs, Bindings} = unlabel(InitKVs, #{}),
- {FinalJSONKVs, _} = unlabel(FinalKVs, Bindings),
- InitDoc = make_doc(DocId, InitJSONKVs),
- FinalDoc = make_doc(DocId, FinalJSONKVs),
- {{InitDoc, FinalDoc}, maps:put(DocId, FinalJSONKVs, IdMapAcc)}
- end, #{}, Transitions),
-
- {InitDocs, FinalDocs} = lists:unzip(Docs),
-
- Sig = couch_uuids:random(),
-
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- write_docs(TxDb, Sig, InitDocs)
- end),
-
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- write_docs(TxDb, Sig, FinalDocs)
- end),
-
- validate_index(Db, Sig, IdMap).
-
-
-clear_views(Db) ->
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
- {Start, End} = erlfdb_tuple:range({?DB_VIEWS}, DbPrefix),
- erlfdb:clear_range(Tx, Start, End),
-
- GlobalKey = {?DB_STATS, <<"sizes">>, <<"views">>},
- BinGlobalKey = erlfdb_tuple:pack(GlobalKey, DbPrefix),
- erlfdb:set(Tx, BinGlobalKey, ?uint2bin(0))
- end).
-
-
-write_docs(TxDb, Sig, Docs) ->
- Mrst = #mrst{
- sig = Sig,
- views = [#mrview{
- id_num = 1
- }]
+foreach_setup() ->
+ config:set("couch_views", "view_btree_node_size", "4", false),
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ Db.
+
+
+foreach_teardown(Db) ->
+ meck:unload(),
+ config:delete("couch_views", "change_limit"),
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+empty_view(Db) ->
+ DDoc = create_ddoc(),
+ ?assertEqual(0, view_size(Db)),
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual(0, view_size(Db)).
+
+
+single_doc(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_doc(Db, Doc1, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Row: key: 0, row: 0, docid: "0"
+ % Bytes: key: 1, row: 1, docid: 3
+ % Total: 1 + 1 + 3 = 5
+ ?assertEqual(5, view_size(Db)).
+
+
+multiple_docs(Db) ->
+ DDoc = create_ddoc(),
+ Docs = [doc(I) || I <- lists:seq(0, 49)],
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_docs(Db, Docs, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Rows 0-9: 1 + 1 + 3 = 5
+ % Rows 10->49: 2 + 2 + 4 = 8
+ % 10 * 5 + 40 * 8 = 370
+ ?assertEqual(370, view_size(Db)).
+
+
+update_no_size_change(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ ?assertEqual(5, view_size(Db)),
+
+ Doc2 = Doc1#doc{
+ revs = {Pos, [Rev]},
+ body = {[{<<"val">>, 1}]}
},
- IdxState = #{
- last_seq => <<"foo">>
+ {ok, _} = fabric2_db:update_doc(Db, Doc2, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Row became: key: 1, val: 1, docid: "0"
+ % 1 + 1 + 3 = 5 so samesies
+ ?assertEqual(5, view_size(Db)).
+
+
+update_increases_size(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ ?assertEqual(5, view_size(Db)),
+
+ Doc2 = Doc1#doc{
+ revs = {Pos, [Rev]},
+ body = {[{<<"val">>, 10}]}
},
- couch_views_indexer:write_docs(TxDb, Mrst, Docs, IdxState).
-
-
-validate_index(Db, Sig, JSONRows) ->
- #{
- db_prefix := DbPrefix
- } = Db,
- Rows = fabric2_fdb:transactional(Db, fun(TxDb) ->
- #{
- tx := Tx
- } = TxDb,
- {Start, End} = erlfdb_tuple:range({?DB_VIEWS}, DbPrefix),
- erlfdb:get_range(Tx, Start, End)
- end),
-
- InitAcc = #{
- row_count => 0,
- kv_size => 0,
- ids => #{},
- rows => []
+ {ok, _} = fabric2_db:update_doc(Db, Doc2, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Row became: key: 10, val: 10, docid: "0"
+ % 2 + 2 + 3 = 7
+ ?assertEqual(7, view_size(Db)).
+
+
+update_decreases_size(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(10),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Row: key: 10, val: 10, docid: "10"
+ % 2 + 2 + 4 = 8
+ ?assertEqual(8, view_size(Db)),
+
+ Doc2 = Doc1#doc{
+ revs = {Pos, [Rev]},
+ body = {[{<<"val">>, 0}]}
},
+ {ok, _} = fabric2_db:update_doc(Db, Doc2, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
- MapData = lists:foldl(fun({Key, Value}, Acc) ->
- case erlfdb_tuple:unpack(Key, DbPrefix) of
- {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Sig} ->
- ?assertEqual(<<"foo">>, Value),
- Acc;
- {?DB_VIEWS, ?VIEW_INFO, ?VIEW_ROW_COUNT, Sig, 1} ->
- maps:put(row_count, ?bin2uint(Value), Acc);
- {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Sig, 1} ->
- maps:put(kv_size, ?bin2uint(Value), Acc);
- {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_ID_RANGE, DocId, 1} ->
- [
- TotalKeys, TotalSize, UniqueKeys
- ] = couch_views_encoding:decode(Value),
- maps:update_with(ids, fun(Ids) ->
- false = maps:is_key(DocId, Ids),
- maps:put(DocId, {TotalKeys, TotalSize, UniqueKeys}, Ids)
- end, Acc);
- {?DB_VIEWS, ?VIEW_DATA, Sig, ?VIEW_MAP_RANGE, 1, MapKey, _DupeId} ->
- {EncKey, DocId} = MapKey,
- {UserKey, UserVal} = erlfdb_tuple:unpack(Value),
-
- UserJsonKey = couch_views_encoding:decode(UserKey),
- UserJsonVal = couch_views_encoding:decode(UserVal),
-
- ?assertEqual(
- EncKey,
- couch_views_encoding:encode(UserJsonKey, key)
- ),
-
- maps:update_with(rows, fun(RAcc) ->
- [{DocId, UserJsonKey, UserJsonVal} | RAcc]
- end, Acc)
- end
- end, InitAcc, Rows),
-
- #{
- row_count := RowCount,
- kv_size := KVSize,
- ids := MapIds,
- rows := MapRows
- } = MapData,
-
- SumFun = fun(_DocId, {TotalKVs, TotalSize, _UniqueKeys}, {KVAcc, SAcc}) ->
- {KVAcc + TotalKVs, SAcc + TotalSize}
- end,
- {SumKVCount, SumKVSize} = maps:fold(SumFun, {0, 0}, MapIds),
- ?assertEqual(RowCount, length(MapRows)),
- ?assertEqual(RowCount, SumKVCount),
- ?assertEqual(KVSize, SumKVSize),
- ?assert(KVSize >= 0),
-
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- GlobalSize = get_global_size(TxDb),
- ?assertEqual(KVSize, GlobalSize),
-
- ViewSize = couch_views_fdb:get_kv_size(TxDb, #mrst{sig = Sig}, 1),
- ?assertEqual(KVSize, ViewSize)
- end),
-
- % Compare our raw JSON rows to what was indexed
- IdsFromJSONRows = maps:fold(fun(DocId, DocRows, IdAcc) ->
- FinalAcc = lists:foldl(fun({JsonKey, JsonVal}, {CAcc, SAcc, UAcc}) ->
- KeySize = erlang:external_size(JsonKey),
- ValSize = erlang:external_size(JsonVal),
- NewUnique = lists:usort([JsonKey | UAcc]),
- {CAcc + 1, SAcc + KeySize + ValSize, NewUnique}
- end, {0, 0, []}, DocRows),
- if FinalAcc == {0, 0, []} -> IdAcc; true ->
- maps:put(DocId, FinalAcc, IdAcc)
- end
- end, #{}, JSONRows),
- ?assertEqual(MapIds, IdsFromJSONRows),
-
- % Compare the found id entries to our row data
- IdsFromMapRows = lists:foldl(fun({DocId, JsonKey, JsonVal}, Acc) ->
- KeySize = erlang:external_size(JsonKey),
- ValSize = erlang:external_size(JsonVal),
- Default = {1, KeySize + ValSize, [JsonKey]},
- maps:update_with(DocId, fun({TotalKVs, TotalSize, UniqueKeys}) ->
- NewUnique = lists:usort([JsonKey | UniqueKeys]),
- {TotalKVs + 1, TotalSize + KeySize + ValSize, NewUnique}
- end, Default, Acc)
- end, #{}, MapRows),
- ?assertEqual(MapIds, IdsFromMapRows).
-
-
-make_doc(DocId, []) ->
- case rand:uniform() < 0.5 of
- true ->
- #{
- id => DocId,
- deleted => true,
- results => [[]]
- };
- false ->
- #{
- id => DocId,
- deleted => false,
- results => [[]]
- }
- end;
-make_doc(DocId, Results) ->
- #{
- id => DocId,
- deleted => false,
- results => [Results]
- }.
+ % Row became: key: 0, val: 0, docid: "10"
+ % 1 + 1 + 4 = 6
+ ?assertEqual(6, view_size(Db)).
+
+
+deleting_docs_decreases_size(Db) ->
+ DDoc = create_ddoc(),
+ Doc1 = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc1, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ ?assertEqual(5, view_size(Db)),
+
+ Doc2 = Doc1#doc{
+ revs = {Pos, [Rev]},
+ deleted = true,
+ body = {[{<<"val">>, 1}]}
+ },
+ {ok, _} = fabric2_db:update_doc(Db, Doc2, []),
+ {ok, []} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ ?assertEqual(0, view_size(Db)).
-get_global_size(TxDb) ->
- #{
- tx := Tx,
- db_prefix := DbPrefix
- } = TxDb,
- GlobalKey = {?DB_STATS, <<"sizes">>, <<"views">>},
- BinGlobalKey = erlfdb_tuple:pack(GlobalKey, DbPrefix),
- ?bin2uint(erlfdb:wait(erlfdb:get(Tx, BinGlobalKey))).
-
-
-init_set({N, D, R}, Labels) ->
- {Dupes, RestLabels} = fill_keys(D, Labels, []),
- {Unique, _} = fill_keys(N - D, RestLabels, []),
- % Sanity assertions
- N = length(Unique) + length(Dupes),
- D = length(Dupes),
- {Unique, [{Key, R} || Key <- Dupes]}.
-
-
-reduce_set({DeltaN, DeltaD, DeltaR}, {Unique, Dupes}) ->
- NewDupes = lists:sublist(Dupes, DeltaD),
- NewUnique = lists:sublist(Unique, DeltaN - DeltaD),
- {NewUnique, [{Key, DeltaR} || {Key, _} <- NewDupes]}.
-
-
-fill_set({N, D, R}, {Unique, Dupes}, Labels) ->
- AddDupes = D - length(Dupes),
- {NewDupes, RestLabels} = fill_keys(AddDupes, Labels, Dupes),
-
- AddUnique = N - length(Unique) - length(NewDupes),
- {NewUnique, _} = fill_keys(AddUnique, RestLabels, Unique),
- % Sanity assertions
- N = length(NewUnique) + length(NewDupes),
- D = length(NewDupes),
- {NewUnique, lists:map(fun(Dupe) ->
- case Dupe of
- {_, _} -> Dupe;
- A when is_atom(A) -> {A, R}
- end
- end, NewDupes)}.
-
-
-fill_keys(0, Labels, Acc) ->
- {Acc, Labels};
-fill_keys(Count, [Label | RestLabels], Acc) when Count > 0 ->
- fill_keys(Count - 1, RestLabels, [Label | Acc]).
-
-
-unlabel({Unique, Dupes}, Bindings) ->
- lists:foldl(fun(Item, {KVAcc, BindingsAcc}) ->
- {KVs, NewBindingsAcc} = unlabel_item(Item, BindingsAcc),
- {KVs ++ KVAcc, NewBindingsAcc}
- end, {[], Bindings}, Unique ++ Dupes).
-
-
-unlabel_item(Label, Bindings) when is_atom(Label) ->
- NewBindings = maybe_bind(Label, Bindings),
- KV = maps:get(Label, NewBindings),
- {[KV], NewBindings};
-unlabel_item({Label, Count}, Bindings) when is_atom(Label), is_integer(Count) ->
- NewBindings = maybe_bind(Label, Bindings),
- {K, _} = KV = maps:get(Label, NewBindings),
- ToAdd = lists:map(fun(_) ->
- {K, gen_value()}
- end, lists:seq(1, Count - 1)),
- {[KV | ToAdd], NewBindings}.
-
-
-maybe_bind(Label, Bindings) ->
- case maps:is_key(Label, Bindings) of
- true ->
- case rand:uniform() < 0.5 of
- true ->
- rebind(Label, Bindings);
- false ->
- Bindings
- end;
- false ->
- bind(Label, Bindings)
- end.
-
-
-bind(Label, Bindings) ->
- maps:put(Label, {gen_key(), gen_value()}, Bindings).
-
-
-rebind(Label, Bindings) ->
- {Key, _} = maps:get(Label, Bindings),
- maps:put(Label, {Key, gen_value()}, Bindings).
-
-
-gen_key() ->
- Unique = couch_uuids:random(),
- case rand:uniform() of
- N when N < 0.2 ->
- [Unique, true, rand:uniform()];
- N when N < 0.4 ->
- {[{Unique, true}, {<<"foo">>, [<<"bar">>, null, 1, {[]}]}]};
- _ ->
- Unique
- end.
-
-
-gen_value() ->
- case rand:uniform() of
- N when N < 0.2 ->
- [false, rand:uniform(), {[]}];
- N when N < 0.4 ->
- {[{<<"a">>, 1}, {<<"b">>, 2}]};
- N when N < 0.6 ->
- rand:uniform(100);
- N when N < 0.8 ->
- rand:uniform();
- _ ->
- 1
- end.
-
-
-group(Items) ->
- case length(Items) > 5 of
- true ->
- {Group, Rest} = lists:split(5, Items),
- [lists:sort(Group) | group(Rest)];
- false when Items == [] ->
- [];
- false ->
- [lists:sort(Items)]
- end.
-
-
-shuffle(Items) ->
- Tagged = [{rand:uniform(), I} || I <- Items],
- Sorted = lists:sort(Tagged),
- [I || {_T, I} <- Sorted].
-
-
-subset(Count, Items) ->
- Random = shuffle(Items),
- Take = lists:sublist(Random, Count),
- lists:sort(Take).
-
-
-tname([]) ->
- [];
-tname([Transition | RestTransitions]) ->
- [tname(Transition) | tname(RestTransitions)];
-tname({{N1, D1, R1}, {N2, D2, R2}, {DN, DD, DR}}) ->
- io_lib:format("~b~b~b~b~b~b~b~b~b", [N1, D1, R1, N2, D2, R2, DN, DD, DR]).
+multi_identical_keys_count_twice(Db) ->
+ DDoc = create_ddoc(multi_emit_same),
+ Doc = doc(0),
+
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_doc(Db, Doc, []),
+ {ok, _} = run_query(Db, DDoc, ?MAP_FUN1),
+
+ % Two rows that are the same
+ ?assertEqual(10, view_size(Db)).
+
+
+multiple_design_docs(Db) ->
+ Cleanup = fun() ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ ok = couch_views:cleanup_indices(TxDb, DDocs)
+ end)
+ end,
+
+ DDoc1 = create_ddoc(simple, <<"_design/bar1">>),
+ DDoc2 = create_ddoc(multi_emit_same, <<"_design/bar2">>),
+
+ % Simple test as before
+ {ok, _} = fabric2_db:update_doc(Db, doc(0), []),
+ {ok, {Pos1, Rev1}} = fabric2_db:update_doc(Db, DDoc1, []),
+ {ok, _} = run_query(Db, DDoc1, ?MAP_FUN1),
+ ?assertEqual(5, view_size(Db)),
+
+ % Adding a second ddoc increases the size
+ {ok, {Pos2, Rev2}} = fabric2_db:update_doc(Db, DDoc2, []),
+ {ok, _} = run_query(Db, DDoc2, ?MAP_FUN1),
+ ?assertEqual(15, view_size(Db)),
+
+ % Removing the first ddoc decreases the size
+ DDoc1Del = DDoc1#doc{revs = {Pos1, [Rev1]}, deleted = true},
+ {ok, _} = fabric2_db:update_doc(Db, DDoc1Del, []),
+ Cleanup(),
+ ?assertEqual(10, view_size(Db)),
+
+ % Removing the second ddoc drops the size
+ DDoc2Del = DDoc2#doc{revs = {Pos2, [Rev2]}, deleted = true},
+ {ok, _} = fabric2_db:update_doc(Db, DDoc2Del, []),
+ Cleanup(),
+ ?assertEqual(0, view_size(Db)).
+
+
+multiple_identical_design_docs(Db) ->
+ Cleanup = fun() ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ ok = couch_views:cleanup_indices(TxDb, DDocs)
+ end)
+ end,
+ DDoc1 = create_ddoc(simple, <<"_design/bar1">>),
+ DDoc2 = create_ddoc(simple, <<"_design/bar2">>),
+
+ % Simple test as before
+ {ok, _} = fabric2_db:update_doc(Db, doc(0), []),
+ {ok, {Pos1, Rev1}} = fabric2_db:update_doc(Db, DDoc1, []),
+ {ok, _} = run_query(Db, DDoc1, ?MAP_FUN1),
+ ?assertEqual(5, view_size(Db)),
+
+ % Adding a second ddoc with the same sig does not double the size
+ {ok, {Pos2, Rev2}} = fabric2_db:update_doc(Db, DDoc2, []),
+ {ok, _} = run_query(Db, DDoc2, ?MAP_FUN1),
+ ?assertEqual(5, view_size(Db)),
+
+ % Removing the first ddoc does not decrease the size
+ DDoc1Del = DDoc1#doc{revs = {Pos1, [Rev1]}, deleted = true},
+ {ok, _} = fabric2_db:update_doc(Db, DDoc1Del, []),
+ Cleanup(),
+ ?assertEqual(5, view_size(Db)),
+
+ % Removing the second ddoc drops the size
+ DDoc2Del = DDoc2#doc{revs = {Pos2, [Rev2]}, deleted = true},
+ {ok, _} = fabric2_db:update_doc(Db, DDoc2Del, []),
+ Cleanup(),
+ ?assertEqual(0, view_size(Db)).
+
+
+view_size(Db) ->
+ {ok, Info} = fabric2_db:get_db_info(Db),
+ {sizes, {Sizes}} = lists:keyfind(sizes, 1, Info),
+ {<<"views">>, ViewSize} = lists:keyfind(<<"views">>, 1, Sizes),
+ ViewSize.
+
+
+create_ddoc() ->
+ create_ddoc(simple).
+
+
+create_ddoc(Type) ->
+ create_ddoc(Type, <<"_design/bar">>).
+
+
+create_ddoc(simple, DocId) when is_binary(DocId) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, DocId},
+ {<<"views">>, {[
+ {?MAP_FUN1, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>}
+ ]}},
+ {?MAP_FUN2, {[
+ {<<"map">>, <<"function(doc) {}">>}
+ ]}}
+ ]}}
+ ]});
+
+create_ddoc(multi_emit_same, DocId) when is_binary(DocId) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, DocId},
+ {<<"views">>, {[
+ {?MAP_FUN1, {[
+ {<<"map">>, <<"function(doc) { "
+ "emit(doc.val, doc.val * 2); "
+ "emit(doc.val, doc.val); "
+ "if(doc.extra) {"
+ " emit(doc.val, doc.extra);"
+ "}"
+ "}">>}
+ ]}},
+ {?MAP_FUN2, {[
+ {<<"map">>, <<"function(doc) {}">>}
+ ]}}
+ ]}}
+ ]}).
+
+
+doc(Id) ->
+ doc(Id, Id).
+
+
+doc(Id, Val) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"val">>, Val}
+ ]}).
+
+
+run_query(#{} = Db, DDoc, <<_/binary>> = View) ->
+ couch_views:query(Db, DDoc, View, fun fold_fun/2, [], #mrargs{}).
+
+
+fold_fun({meta, _Meta}, Acc) ->
+ {ok, Acc};
+fold_fun({row, _} = Row, Acc) ->
+ {ok, [Row | Acc]};
+fold_fun(complete, Acc) ->
+ {ok, lists:reverse(Acc)}.
\ No newline at end of file
[couchdb] 07/07: Add test suite for reduce views
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch feature-ebtree-views-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7701f2c8610b4d2bcb4fbdeef27982974e610f08
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Aug 5 12:47:37 2020 -0500
Add test suite for reduce views
---
src/couch_views/test/couch_views_red_test.erl | 764 ++++++++++++++++++++++++++
1 file changed, 764 insertions(+)
diff --git a/src/couch_views/test/couch_views_red_test.erl b/src/couch_views/test/couch_views_red_test.erl
new file mode 100644
index 0000000..875e90b
--- /dev/null
+++ b/src/couch_views/test/couch_views_red_test.erl
@@ -0,0 +1,764 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_red_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
+
+
+-define(TDEF(A), {atom_to_list(A), fun A/1}).
+-define(TDEFI(A), {atom_to_list(A), fun A/0}).
+
+
+with(Tests) ->
+ fun(ArgsTuple) ->
+ lists:map(fun({Name, Fun}) ->
+ {Name, ?_test(Fun(ArgsTuple))}
+ end, Tests)
+ end.
+
+
+-define(NUM_DOCS, 2000).
+
+
+reduce_views_shraed_db_test_() ->
+ {
+ "Reduce views",
+ {
+ setup,
+ fun setup_shared_db/0,
+ fun teardown_shared_db/1,
+ with([
+ ?TDEF(should_reduce),
+ ?TDEF(should_reduce_rev),
+ ?TDEF(should_reduce_start_key),
+ ?TDEF(should_reduce_start_key_rev),
+ ?TDEF(should_reduce_end_key),
+ ?TDEF(should_reduce_end_key_rev),
+ ?TDEF(should_reduce_inclusive_end_false),
+ ?TDEF(should_reduce_inclusive_end_false_rev),
+ ?TDEF(should_reduce_start_and_end_key),
+ ?TDEF(should_reduce_start_and_end_key_rev),
+ ?TDEF(should_reduce_empty_range),
+ ?TDEF(should_reduce_empty_range_rev),
+ ?TDEF(should_reduce_grouped),
+ ?TDEF(should_reduce_grouped_rev),
+ ?TDEF(should_reduce_grouped_start_key),
+ ?TDEF(should_reduce_grouped_start_key_rev),
+ ?TDEF(should_reduce_grouped_end_key),
+ ?TDEF(should_reduce_grouped_end_key_rev),
+ ?TDEF(should_reduce_grouped_inclusive_end_false),
+ ?TDEF(should_reduce_grouped_inclusive_end_false_rev),
+ ?TDEF(should_reduce_grouped_start_and_end_key),
+ ?TDEF(should_reduce_grouped_start_and_end_key_rev),
+ ?TDEF(should_reduce_grouped_empty_range),
+ ?TDEF(should_reduce_grouped_empty_range_rev),
+
+ ?TDEF(should_reduce_array_keys),
+ ?TDEF(should_reduce_grouped_array_keys),
+ ?TDEF(should_reduce_group_1_array_keys),
+ ?TDEF(should_reduce_group_1_array_keys_start_key),
+ ?TDEF(should_reduce_group_1_array_keys_start_key_rev),
+ ?TDEF(should_reduce_group_1_array_keys_end_key),
+ ?TDEF(should_reduce_group_1_array_keys_end_key_rev),
+ ?TDEF(should_reduce_group_1_array_keys_inclusive_end_false),
+ ?TDEF(should_reduce_group_1_array_keys_inclusive_end_false_rev),
+ ?TDEF(should_reduce_group_1_array_keys_start_and_end_key),
+ ?TDEF(should_reduce_group_1_array_keys_start_and_end_key_rev),
+ ?TDEF(should_reduce_group_1_array_keys_sub_array_select),
+ ?TDEF(should_reduce_group_1_array_keys_sub_array_select_rev),
+ ?TDEF(should_reduce_group_1_array_keys_sub_array_inclusive_end),
+ ?TDEF(should_reduce_group_1_array_keys_empty_range),
+ ?TDEF(should_reduce_group_1_array_keys_empty_range_rev)
+ ])
+ }
+ }.
+
+
+reduce_views_individual_test_() ->
+ {
+ "Reduce views",
+ {
+ setup,
+ fun setup_individual/0,
+ fun teardown_individual/1,
+ [
+ ?TDEFI(should_collate_group_keys)
+ ]
+ }
+ }.
+
+
+setup_shared_db() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]),
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ fabric2_db:update_docs(Db, [create_ddoc()]),
+ make_docs(Db, ?NUM_DOCS),
+ run_query(Db, <<"baz">>, #{limit => 0}),
+ {Db, Ctx}.
+
+
+teardown_shared_db({Db, Ctx}) ->
+ fabric2_db:delete(fabric2_db:name(Db), [{user_ctx, ?ADMIN_USER}]),
+ test_util:stop_couch(Ctx).
+
+
+setup_individual() ->
+ test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]).
+
+
+teardown_individual(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+should_reduce({Db, _}) ->
+ Result = run_query(Db, <<"baz_count">>, #{}),
+ Expect = {ok, [row(null, ?NUM_DOCS)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_rev({Db, _}) ->
+ Args = #{
+ direction => rev
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, ?NUM_DOCS)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_start_key({Db, _}) ->
+ Args = #{
+ start_key => 4
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, ?NUM_DOCS - 3)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_start_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ start_key => 4
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 4)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_end_key({Db, _}) ->
+ Args = #{
+ end_key => 6
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 6)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ end_key => 6
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, ?NUM_DOCS - 5)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_inclusive_end_false({Db, _}) ->
+ Args = #{
+ end_key => 6,
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 5)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_inclusive_end_false_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ end_key => 6,
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, ?NUM_DOCS - 6)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_start_and_end_key({Db, _}) ->
+ Args = #{
+ start_key => 3,
+ end_key => 5
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 3)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_start_and_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ start_key => 5,
+ end_key => 3
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 3)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_empty_range({Db, _}) ->
+ Args = #{
+ start_key => 100000,
+ end_key => 100001
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 0)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_empty_range_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ start_key => 100001,
+ end_key => 100000
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [row(null, 0)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped({Db, _}) ->
+ Args = #{
+ group_level => exact
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(1, ?NUM_DOCS)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(?NUM_DOCS, 1, -1)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_start_key({Db, _}) ->
+ Args = #{
+ group_level => exact,
+ start_key => 3
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(3, ?NUM_DOCS)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_start_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact,
+ start_key => 3
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(3, 1),
+ row(2, 1),
+ row(1, 1)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_end_key({Db, _}) ->
+ Args = #{
+ group_level => exact,
+ end_key => 6
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(1, 6)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact,
+ end_key => 6
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(?NUM_DOCS, 6, -1)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_inclusive_end_false({Db, _}) ->
+ Args = #{
+ group_level => exact,
+ end_key => 4,
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(1, 3)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_inclusive_end_false_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact,
+ end_key => 4,
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(?NUM_DOCS, 5, -1)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_start_and_end_key({Db, _}) ->
+ Args = #{
+ group_level => exact,
+ start_key => 2,
+ end_key => 4
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(2, 4)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_start_and_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact,
+ start_key => 4,
+ end_key => 2
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, [
+ row(I, 1) || I <- lists:seq(4, 2, -1)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_empty_range({Db, _}) ->
+ Args = #{
+ group_level => exact,
+ start_key => 100000,
+ end_key => 100001
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, []},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_empty_range_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => exact,
+ start_key => 100001,
+ end_key => 100000
+ },
+ Result = run_query(Db, <<"baz_count">>, Args),
+ Expect = {ok, []},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_array_keys({Db, _}) ->
+ Result = run_query(Db, <<"boom">>, #{}),
+ Expect = {ok, [row(null, 1.5 * ?NUM_DOCS)]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_grouped_array_keys({Db, _}) ->
+ Args = #{
+ group_level => exact
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, lists:sort([
+ row([I rem 3, I], 1.5) || I <- lists:seq(1, ?NUM_DOCS)
+ ])},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys({Db, _}) ->
+ Args = #{
+ group_level => 1
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([0], rem_count(0, ?NUM_DOCS) * 1.5),
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5),
+ row([2], rem_count(2, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_start_key({Db, _}) ->
+ Args = #{
+ group_level => 1,
+ start_key => [1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5),
+ row([2], rem_count(2, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_start_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ start_key => [1, ?NUM_DOCS + 1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5),
+ row([0], rem_count(0, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_end_key({Db, _}) ->
+ Args = #{
+ group_level => 1,
+ end_key => [1, ?NUM_DOCS + 1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([0], rem_count(0, ?NUM_DOCS) * 1.5),
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ end_key => [1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([2], rem_count(2, ?NUM_DOCS) * 1.5),
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_inclusive_end_false({Db, _}) ->
+ Args = #{
+ group_level => 1,
+ end_key => [1],
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([0], rem_count(0, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_inclusive_end_false_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ end_key => [1, ?NUM_DOCS + 1],
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([2], rem_count(2, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_start_and_end_key({Db, _}) ->
+ Args = #{
+ group_level => 1,
+ start_key => [1],
+ end_key => [1, ?NUM_DOCS + 1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_start_and_end_key_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ start_key => [1, ?NUM_DOCS + 1],
+ end_key => [1]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([1], rem_count(1, ?NUM_DOCS) * 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_sub_array_select({Db, _}) ->
+ % Test that keys are applied below the key grouping
+ Args = #{
+ group_level => 1,
+ start_key => [0, ?NUM_DOCS - 6],
+ end_key => [1, 4]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([0], 3.0),
+ row([1], 3.0)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_sub_array_select_rev({Db, _}) ->
+ % Test that keys are applied below the key grouping
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ start_key => [1, 4],
+ end_key => [0, ?NUM_DOCS - 6]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([1], 3.0),
+ row([0], 3.0)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_sub_array_inclusive_end({Db, _}) ->
+ % Test that keys are applied below the key grouping
+ Args = #{
+ group_level => 1,
+ start_key => [0, ?NUM_DOCS - 6],
+ end_key => [1, 4],
+ inclusive_end => false
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, [
+ row([0], 3.0),
+ row([1], 1.5)
+ ]},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_empty_range({Db, _}) ->
+ Args = #{
+ group_level => 1,
+ start_key => [100],
+ end_key => [101]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, []},
+ ?assertEqual(Expect, Result).
+
+
+should_reduce_group_1_array_keys_empty_range_rev({Db, _}) ->
+ Args = #{
+ direction => rev,
+ group_level => 1,
+ start_key => [101],
+ end_key => [100]
+ },
+ Result = run_query(Db, <<"boom">>, Args),
+ Expect = {ok, []},
+ ?assertEqual(Expect, Result).
+
+
+should_collate_group_keys() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/bar">>},
+ {<<"views">>, {[
+ {<<"group">>, {[
+ {<<"map">>, <<"function(doc) {emit([doc.val], 1);}">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}}
+ ]}}
+ ]}),
+ % val is "föö" without combining characters
+ Doc1 = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"a">>},
+ {<<"val">>, <<16#66, 16#C3, 16#B6, 16#C3, 16#B6>>}
+ ]}),
+ % val is "föö" without combining characters
+ Doc2 = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"b">>},
+ {<<"val">>, <<16#66, 16#6F, 16#CC, 16#88, 16#6F, 16#CC, 16#88>>}
+ ]}),
+ {ok, _} = fabric2_db:update_docs(Db, [DDoc, Doc1, Doc2]),
+
+ % An implementation detail we have is that depending on
+ % the direction of the view read we'll get the first
+ % or last key to represent a group. In this particular
+ % implementation the document ID breaks the sort tie
+ % in the map view data.
+
+ ArgsFwd = #{
+ group_level => exact
+ },
+ ResultFwd = run_query(Db, DDoc, <<"group">>, ArgsFwd),
+ ExpectFwd = {ok, [
+ row([<<16#66, 16#C3, 16#B6, 16#C3, 16#B6>>], 2)
+ ]},
+ ?assertEqual(ExpectFwd, ResultFwd),
+
+ ArgsRev = #{
+ direction => rev,
+ group_level => exact
+ },
+ ResultRev = run_query(Db, DDoc, <<"group">>, ArgsRev),
+ ExpectRev = {ok, [
+ row([<<16#66, 16#6F, 16#CC, 16#88, 16#6F, 16#CC, 16#88>>], 2)
+ ]},
+ ?assertEqual(ExpectRev, ResultRev).
+
+
+rem_count(Rem, Count) ->
+ Members = [I || I <- lists:seq(1, Count), I rem 3 == Rem],
+ length(Members).
+
+
+run_query(Db, Idx, Args) ->
+ DDoc = create_ddoc(),
+ run_query(Db, DDoc, Idx, Args).
+
+
+run_query(Db, DDoc, Idx, Args) ->
+ couch_views:query(Db, DDoc, Idx, fun default_cb/2, [], Args).
+
+
+default_cb(complete, Acc) ->
+ {ok, lists:reverse(Acc)};
+default_cb({final, Info}, []) ->
+ {ok, [Info]};
+default_cb({final, _}, Acc) ->
+ {ok, Acc};
+default_cb({meta, _}, Acc) ->
+ {ok, Acc};
+default_cb(ok, ddoc_updated) ->
+ {ok, ddoc_updated};
+default_cb(Row, Acc) ->
+ {ok, [Row | Acc]}.
+
+
+row(Key, Value) ->
+ {row, [{key, Key}, {value, Value}]}.
+
+
+create_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/bar">>},
+ {<<"views">>, {[
+ {<<"baz">>, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>}
+ ]}},
+ {<<"baz_count">>, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}},
+ {<<"baz_size">>, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>},
+ {<<"reduce">>, <<"_sum">>}
+ ]}},
+ {<<"boom">>, {[
+ {<<"map">>, <<
+ "function(doc) {\n"
+ " emit([doc.val % 3, doc.val], 1.5);\n"
+ "}"
+ >>},
+ {<<"reduce">>, <<"_sum">>}
+ ]}},
+ {<<"bing">>, {[
+ {<<"map">>, <<"function(doc) {}">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}},
+ {<<"bing_hyper">>, {[
+ {<<"map">>, <<"function(doc) {}">>},
+ {<<"reduce">>, <<"_approx_count_distinct">>}
+ ]}},
+ {<<"doc_emit">>, {[
+ {<<"map">>, <<"function(doc) {emit(doc.val, doc)}">>}
+ ]}},
+ {<<"duplicate_keys">>, {[
+ {<<"map">>, <<
+ "function(doc) {\n"
+ " emit(doc._id, doc.val);\n"
+ " emit(doc._id, doc.val + 1);\n"
+ "}">>},
+ {<<"reduce">>, <<"_count">>}
+ ]}},
+ {<<"zing">>, {[
+ {<<"map">>, <<
+ "function(doc) {\n"
+ " if(doc.foo !== undefined)\n"
+ " emit(doc.foo, 0);\n"
+ "}"
+ >>}
+ ]}}
+ ]}}
+ ]}).
+
+
+make_docs(Db, TotalDocs) when TotalDocs > 0 ->
+ make_docs(Db, TotalDocs, 0).
+
+
+make_docs(Db, TotalDocs, DocsMade) when TotalDocs > DocsMade ->
+ DocCount = min(TotalDocs - DocsMade, 500),
+ Docs = [doc(I + DocsMade) || I <- lists:seq(1, DocCount)],
+ fabric2_db:update_docs(Db, Docs),
+ make_docs(Db, TotalDocs, DocsMade + DocCount);
+
+make_docs(_Db, TotalDocs, DocsMade) when TotalDocs =< DocsMade ->
+ ok.
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"val">>, Id}
+ ]}).