You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2020/04/06 15:55:59 UTC
[couchdb] branch prototype/fdb-layer updated (d520d73 -> 5652e72)
This is an automated email from the ASF dual-hosted git repository.
garren pushed a change to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from d520d73 Merge pull request #2662 from cloudant/couch_view-rate_limit
new c22d007 fix all_docs call to return row
new cc2d232 move all_doc view options to fabric2_util
new a8b930d add include_docs option to fold_docs
new 551bd91 add fabric2 after_doc_write plugin
new 9d27c6e Add couch_views_indexer build to creation versionstamp
new 34ca5e4 All couch_view queries to run across transactions
new b856501 Add couch_views_updater interactive indexer
new 0c12654 Add couch_views_encoding max value
new aa940db remove mango native proc
new e28eb6b remove partition opts from mango
new bd1667d remove unneeded r/w parameter
new 4f8e33e Remove view_cb predicate push down
new dd44442 Remove quorum stats
new c483652 Add mango indexing
new 377b0c2 Update Mango query to work with couch_views
new 1bc7c54 Update mango test creds to same as elixir tests
new 29ef7fb update mango tests to work with Mango on FDB
new 5652e72 remove defer and load_ddocs from mango_utils
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Makefile | 5 +-
rel/apps/couch_epi.config | 1 +
rel/overlay/etc/default.ini | 1 +
src/chttpd/src/chttpd_db.erl | 33 +-
src/couch/src/couch_proc_manager.erl | 1 -
src/couch_eval/src/couch_eval.erl | 3 +
src/couch_js/src/couch_js_proc_manager.erl | 1 -
src/couch_views/include/couch_views.hrl | 6 +
src/couch_views/src/couch_views.app.src | 1 +
src/couch_views/src/couch_views.erl | 16 +-
src/couch_views/src/couch_views_ddoc.erl | 42 +++
src/couch_views/src/couch_views_encoding.erl | 12 +
.../src/couch_views_epi.erl} | 15 +-
.../src/couch_views_fabric2_plugin.erl} | 15 +-
src/couch_views/src/couch_views_fdb.erl | 76 +++++
src/couch_views/src/couch_views_indexer.erl | 77 +++--
src/couch_views/src/couch_views_jobs.erl | 21 +-
src/couch_views/src/couch_views_reader.erl | 3 +-
src/couch_views/src/couch_views_sup.erl | 2 +-
src/couch_views/src/couch_views_updater.erl | 101 ++++++
src/couch_views/test/couch_views_indexer_test.erl | 42 ++-
src/couch_views/test/couch_views_map_test.erl | 41 ++-
.../test/couch_views_trace_index_test.erl | 5 +-
src/couch_views/test/couch_views_updater_test.erl | 230 +++++++++++++
src/fabric/src/fabric2_db.erl | 27 +-
src/fabric/src/fabric2_db_plugin.erl | 5 +
src/fabric/src/fabric2_fdb.erl | 15 +-
src/fabric/src/fabric2_util.erl | 38 +++
src/mango/src/mango_crud.erl | 35 +-
src/mango/src/mango_cursor.erl | 10 +-
src/mango/src/mango_cursor_text.erl | 9 -
src/mango/src/mango_cursor_view.erl | 183 ++--------
src/mango/src/mango_error.erl | 14 -
src/mango/src/mango_eval.erl | 115 +++++++
src/mango/src/mango_execution_stats.erl | 8 -
src/mango/src/mango_execution_stats.hrl | 1 -
src/mango/src/mango_httpd.erl | 148 ++++----
src/mango/src/mango_idx.erl | 206 ++++--------
src/mango/src/mango_idx.hrl | 5 +-
src/mango/src/mango_idx_special.erl | 4 +-
src/mango/src/mango_idx_text.erl | 1 -
src/mango/src/mango_idx_view.erl | 28 +-
src/mango/src/mango_idx_view.hrl | 13 -
src/mango/src/mango_native_proc.erl | 373 ---------------------
src/mango/src/mango_opts.erl | 42 ---
src/mango/src/mango_util.erl | 58 +---
src/mango/test/01-index-crud-test.py | 15 +
src/mango/test/02-basic-find-test.py | 15 -
src/mango/test/05-index-selection-test.py | 7 +-
src/mango/test/12-use-correct-index-test.py | 38 ++-
src/mango/test/13-stable-update-test.py | 51 ---
src/mango/test/13-users-db-find-test.py | 7 +-
src/mango/test/15-execution-stats-test.py | 7 +-
src/mango/test/17-multi-type-value-test.py | 4 +-
src/mango/test/19-find-conflicts.py | 7 +-
src/mango/test/20-no-timeout-test.py | 32 --
src/mango/test/22-build-wait-selected-index.py | 50 +++
src/mango/test/README.md | 4 +-
src/mango/test/mango.py | 26 +-
src/mango/test/user_docs.py | 7 +-
60 files changed, 1167 insertions(+), 1191 deletions(-)
create mode 100644 src/couch_views/src/couch_views_ddoc.erl
copy src/{setup/src/setup_epi.erl => couch_views/src/couch_views_epi.erl} (89%)
copy src/{couch/src/couch_rand.erl => couch_views/src/couch_views_fabric2_plugin.erl} (69%)
create mode 100644 src/couch_views/src/couch_views_updater.erl
create mode 100644 src/couch_views/test/couch_views_updater_test.erl
create mode 100644 src/mango/src/mango_eval.erl
delete mode 100644 src/mango/src/mango_idx_view.hrl
delete mode 100644 src/mango/src/mango_native_proc.erl
delete mode 100644 src/mango/test/13-stable-update-test.py
delete mode 100644 src/mango/test/20-no-timeout-test.py
create mode 100644 src/mango/test/22-build-wait-selected-index.py
[couchdb] 12/18: Remove view_cb predicate push down
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4f8e33ea7fa2d4131bfe7fb5cca139f00ab17c8f
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:11:03 2020 +0200
Remove view_cb predicate push down
Removes the view callback that was performed on the nodes before
sending the results back to the co-ordinator.
---
src/mango/src/mango_cursor_view.erl | 95 +------------------------------------
1 file changed, 1 insertion(+), 94 deletions(-)
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 5187c10..b88f6ea 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -19,7 +19,6 @@
]).
-export([
- view_cb/2,
handle_message/2,
handle_all_docs_message/2,
composite_indexes/2,
@@ -114,8 +113,7 @@ base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
reduce = false,
start_key = StartKey,
end_key = EndKey,
- include_docs = true,
- extra = [{callback, {?MODULE, view_cb}}, {selector, Selector}]
+ include_docs = true
}.
@@ -226,66 +224,6 @@ choose_best_index(_DbName, IndexRanges) ->
{SelectedIndex, SelectedIndexRanges}.
-view_cb({meta, Meta}, Acc) ->
- % Map function starting
- put(mango_docs_examined, 0),
- set_mango_msg_timestamp(),
- ok = rexi:stream2({meta, Meta}),
- {ok, Acc};
-view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
- ViewRow = #view_row{
- id = couch_util:get_value(id, Row),
- key = couch_util:get_value(key, Row),
- doc = couch_util:get_value(doc, Row)
- },
- case ViewRow#view_row.doc of
- null ->
- maybe_send_mango_ping();
- undefined ->
- % include_docs=false. Use quorum fetch at coordinator
- ok = rexi:stream2(ViewRow),
- set_mango_msg_timestamp();
- Doc ->
- put(mango_docs_examined, get(mango_docs_examined) + 1),
- Selector = couch_util:get_value(selector, Options),
- couch_stats:increment_counter([mango, docs_examined]),
- case mango_selector:match(Selector, Doc) of
- true ->
- ok = rexi:stream2(ViewRow),
- set_mango_msg_timestamp();
- false ->
- maybe_send_mango_ping()
- end
- end,
- {ok, Acc};
-view_cb(complete, Acc) ->
- % Send shard-level execution stats
- ok = rexi:stream2({execution_stats, {docs_examined, get(mango_docs_examined)}}),
- % Finish view output
- ok = rexi:stream_last(complete),
- {ok, Acc};
-view_cb(ok, ddoc_updated) ->
- rexi:reply({ok, ddoc_updated}).
-
-
-maybe_send_mango_ping() ->
- Current = os:timestamp(),
- LastPing = get(mango_last_msg_timestamp),
- % Fabric will timeout if it has not heard a response from a worker node
- % after 5 seconds. Send a ping every 4 seconds so the timeout doesn't happen.
- case timer:now_diff(Current, LastPing) > ?HEARTBEAT_INTERVAL_IN_USEC of
- false ->
- ok;
- true ->
- rexi:ping(),
- set_mango_msg_timestamp()
- end.
-
-
-set_mango_msg_timestamp() ->
- put(mango_last_msg_timestamp, os:timestamp()).
-
-
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
@@ -456,34 +394,3 @@ update_bookmark_keys(#cursor{limit = Limit} = Cursor, Props) when Limit > 0 ->
};
update_bookmark_keys(Cursor, _Props) ->
Cursor.
-
-
-%%%%%%%% module tests below %%%%%%%%
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-
-does_not_refetch_doc_with_value_test() ->
- Cursor = #cursor {
- db = <<"db">>,
- opts = [],
- execution_stats = #execution_stats{},
- selector = mango_selector:normalize({[{<<"user_id">>, <<"1234">>}]})
- },
- RowProps = [
- {id,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {key,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {doc,{
- [
- {<<"_id">>,<<"b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4">>},
- {<<"_rev">>,<<"1-a954fe2308f14307756067b0e18c2968">>},
- {<<"user_id">>,11}
- ]
- }}
- ],
- {Match, _, _} = doc_member(Cursor, RowProps),
- ?assertEqual(Match, ok).
-
-
--endif.
[couchdb] 04/18: add fabric2 after_doc_write plugin
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 551bd91d1f0c195d44c5a5f660c1bdcfcc117b1a
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Thu Mar 19 17:05:54 2020 +0200
add fabric2 after_doc_write plugin
---
src/fabric/src/fabric2_db_plugin.erl | 5 +++++
src/fabric/src/fabric2_fdb.erl | 3 +++
2 files changed, 8 insertions(+)
diff --git a/src/fabric/src/fabric2_db_plugin.erl b/src/fabric/src/fabric2_db_plugin.erl
index fb83ed4..1d923dd 100644
--- a/src/fabric/src/fabric2_db_plugin.erl
+++ b/src/fabric/src/fabric2_db_plugin.erl
@@ -15,6 +15,7 @@
-export([
validate_dbname/3,
before_doc_update/3,
+ after_doc_write/6,
after_doc_read/2,
validate_docid/1,
check_is_admin/1,
@@ -49,6 +50,10 @@ before_doc_update(Db, Doc0, UpdateType) ->
end.
+after_doc_write(Db, Doc, NewWinner, OldWinner, NewRevId, Seq)->
+ with_pipe(after_doc_write, [Db, Doc, NewWinner, OldWinner, NewRevId, Seq]).
+
+
after_doc_read(Db, Doc0) ->
Fun = fabric2_db:get_after_doc_read_fun(Db),
case with_pipe(after_doc_read, [Doc0, Db]) of
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 22ccc99..912d4df 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -832,6 +832,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
ok
end,
+ fabric2_db_plugin:after_doc_write(Db, Doc, NewWinner, OldWinner,
+ NewRevId, WinnerVS),
+
% Update database size
AddSize = sum_add_rev_sizes([NewWinner | ToUpdate]),
RemSize = sum_rem_rev_sizes(ToRemove),
[couchdb] 08/18: Add couch_views_encoding max value
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0c12654ec79ba9ad4a8b1dd43cd89e8deb5797c3
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 24 15:48:14 2020 +0200
Add couch_views_encoding max value
Adds a max value to use for encoding. This is useful when getting the
max range when encoding startkey/endkeys.
---
src/couch_views/src/couch_views_encoding.erl | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/src/couch_views/src/couch_views_encoding.erl b/src/couch_views/src/couch_views_encoding.erl
index ef5fed9..2f69db3 100644
--- a/src/couch_views/src/couch_views_encoding.erl
+++ b/src/couch_views/src/couch_views_encoding.erl
@@ -14,6 +14,7 @@
-export([
+ max/0,
encode/1,
encode/2,
decode/1
@@ -27,6 +28,11 @@
-define(STRING, 4).
-define(LIST, 5).
-define(OBJECT, 6).
+-define(MAX, 255).
+
+
+max() ->
+ max_encoding_value.
encode(X) ->
@@ -51,6 +57,9 @@ encode_int(false, _Type) ->
encode_int(true, _Type) ->
{?TRUE};
+encode_int(max_encoding_value, _Type) ->
+ {?MAX};
+
encode_int(Num, key) when is_number(Num) ->
{?NUMBER, float(Num)};
@@ -87,6 +96,9 @@ decode_int({?FALSE}) ->
decode_int({?TRUE}) ->
true;
+decode_int({?MAX}) ->
+ max_encoding_value;
+
decode_int({?STRING, Bin}) ->
Bin;
[couchdb] 09/18: remove mango native proc
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit aa940dbd1cf514b2f898b5b5816454b7919fc092
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 10 18:20:27 2020 +0200
remove mango native proc
---
src/couch/src/couch_proc_manager.erl | 1 -
src/couch_js/src/couch_js_proc_manager.erl | 1 -
src/mango/src/mango_native_proc.erl | 373 -----------------------------
3 files changed, 375 deletions(-)
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index 0daef3e..376e12e 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -109,7 +109,6 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/couch_js/src/couch_js_proc_manager.erl b/src/couch_js/src/couch_js_proc_manager.erl
index 0964696..45f1736 100644
--- a/src/couch_js/src/couch_js_proc_manager.erl
+++ b/src/couch_js/src/couch_js_proc_manager.erl
@@ -108,7 +108,6 @@ init([]) ->
ets:new(?SERVERS, [public, named_table, set]),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
- ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
maybe_configure_erlang_native_servers(),
{ok, #state{
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
deleted file mode 100644
index cbf3622..0000000
--- a/src/mango/src/mango_native_proc.erl
+++ /dev/null
@@ -1,373 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(mango_native_proc).
--behavior(gen_server).
-
-
--include("mango_idx.hrl").
-
-
--export([
- start_link/0,
- set_timeout/2,
- prompt/2
-]).
-
--export([
- init/1,
- terminate/2,
- handle_call/3,
- handle_cast/2,
- handle_info/2,
- code_change/3
-]).
-
-
--record(st, {
- indexes = [],
- timeout = 5000
-}).
-
-
--record(tacc, {
- index_array_lengths = true,
- fields = all_fields,
- path = []
-}).
-
-
-start_link() ->
- gen_server:start_link(?MODULE, [], []).
-
-
-set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
- gen_server:call(Pid, {set_timeout, TimeOut}).
-
-
-prompt(Pid, Data) ->
- gen_server:call(Pid, {prompt, Data}).
-
-
-init(_) ->
- {ok, #st{}}.
-
-
-terminate(_Reason, _St) ->
- ok.
-
-
-handle_call({set_timeout, TimeOut}, _From, St) ->
- {reply, ok, St#st{timeout=TimeOut}};
-
-handle_call({prompt, [<<"reset">>]}, _From, St) ->
- {reply, true, St#st{indexes=[]}};
-
-handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) ->
- {reply, true, St#st{indexes=[]}};
-
-handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
- Indexes = case validate_index_info(IndexInfo) of
- true ->
- St#st.indexes ++ [IndexInfo];
- false ->
- couch_log:error("No Valid Indexes For: ~p", [IndexInfo]),
- St#st.indexes
- end,
- NewSt = St#st{indexes = Indexes},
- {reply, true, NewSt};
-
-handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) ->
- {reply, map_doc(St, mango_json:to_binary(Doc)), St};
-
-handle_call({prompt, [<<"reduce">>, RedSrcs, _]}, _From, St) ->
- {reply, [true, [null || _ <- RedSrcs]], St};
-
-handle_call({prompt, [<<"rereduce">>, RedSrcs, _]}, _From, St) ->
- {reply, [true, [null || _ <- RedSrcs]], St};
-
-handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
- Vals = case index_doc(St, mango_json:to_binary(Doc)) of
- [] ->
- [[]];
- Else ->
- Else
- end,
- {reply, Vals, St};
-
-
-handle_call(Msg, _From, St) ->
- {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
-
-
-handle_cast(garbage_collect, St) ->
- erlang:garbage_collect(),
- {noreply, St};
-
-handle_cast(stop, St) ->
- {stop, normal, St};
-
-handle_cast(Msg, St) ->
- {stop, {invalid_cast, Msg}, St}.
-
-
-handle_info(Msg, St) ->
- {stop, {invalid_info, Msg}, St}.
-
-
-code_change(_OldVsn, St, _Extra) ->
- {ok, St}.
-
-
-map_doc(#st{indexes=Indexes}, Doc) ->
- lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes).
-
-
-index_doc(#st{indexes=Indexes}, Doc) ->
- lists:map(fun(Idx) -> get_text_entries(Idx, Doc) end, Indexes).
-
-
-get_index_entries({IdxProps}, Doc) ->
- {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
- Selector = get_index_partial_filter_selector(IdxProps),
- case should_index(Selector, Doc) of
- false ->
- [];
- true ->
- Values = get_index_values(Fields, Doc),
- case lists:member(not_found, Values) of
- true -> [];
- false -> [[Values, null]]
- end
- end.
-
-
-get_index_values(Fields, Doc) ->
- lists:map(fun({Field, _Dir}) ->
- case mango_doc:get_field(Doc, Field) of
- not_found -> not_found;
- bad_path -> not_found;
- Value -> Value
- end
- end, Fields).
-
-
-get_text_entries({IdxProps}, Doc) ->
- Selector = get_index_partial_filter_selector(IdxProps),
- case should_index(Selector, Doc) of
- true ->
- get_text_entries0(IdxProps, Doc);
- false ->
- []
- end.
-
-
-get_index_partial_filter_selector(IdxProps) ->
- case couch_util:get_value(<<"partial_filter_selector">>, IdxProps, {[]}) of
- {[]} ->
- % this is to support legacy text indexes that had the partial_filter_selector
- % set as selector
- couch_util:get_value(<<"selector">>, IdxProps, {[]});
- Else ->
- Else
- end.
-
-
-get_text_entries0(IdxProps, Doc) ->
- DefaultEnabled = get_default_enabled(IdxProps),
- IndexArrayLengths = get_index_array_lengths(IdxProps),
- FieldsList = get_text_field_list(IdxProps),
- TAcc = #tacc{
- index_array_lengths = IndexArrayLengths,
- fields = FieldsList
- },
- Fields0 = get_text_field_values(Doc, TAcc),
- Fields = if not DefaultEnabled -> Fields0; true ->
- add_default_text_field(Fields0)
- end,
- FieldNames = get_field_names(Fields),
- Converted = convert_text_fields(Fields),
- FieldNames ++ Converted.
-
-
-get_text_field_values({Props}, TAcc) when is_list(Props) ->
- get_text_field_values_obj(Props, TAcc, []);
-
-get_text_field_values(Values, TAcc) when is_list(Values) ->
- IndexArrayLengths = TAcc#tacc.index_array_lengths,
- NewPath = ["[]" | TAcc#tacc.path],
- NewTAcc = TAcc#tacc{path = NewPath},
- case IndexArrayLengths of
- true ->
- % We bypass make_text_field and directly call make_text_field_name
- % because the length field name is not part of the path.
- LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>),
- LengthField = [{LengthFieldName, <<"length">>, length(Values)}],
- get_text_field_values_arr(Values, NewTAcc, LengthField);
- _ ->
- get_text_field_values_arr(Values, NewTAcc, [])
- end;
-
-get_text_field_values(Bin, TAcc) when is_binary(Bin) ->
- make_text_field(TAcc, <<"string">>, Bin);
-
-get_text_field_values(Num, TAcc) when is_number(Num) ->
- make_text_field(TAcc, <<"number">>, Num);
-
-get_text_field_values(Bool, TAcc) when is_boolean(Bool) ->
- make_text_field(TAcc, <<"boolean">>, Bool);
-
-get_text_field_values(null, TAcc) ->
- make_text_field(TAcc, <<"null">>, true).
-
-
-get_text_field_values_obj([], _, FAcc) ->
- FAcc;
-get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) ->
- NewPath = [Key | TAcc#tacc.path],
- NewTAcc = TAcc#tacc{path = NewPath},
- Fields = get_text_field_values(Val, NewTAcc),
- get_text_field_values_obj(Rest, TAcc, Fields ++ FAcc).
-
-
-get_text_field_values_arr([], _, FAcc) ->
- FAcc;
-get_text_field_values_arr([Value | Rest], TAcc, FAcc) ->
- Fields = get_text_field_values(Value, TAcc),
- get_text_field_values_arr(Rest, TAcc, Fields ++ FAcc).
-
-
-get_default_enabled(Props) ->
- case couch_util:get_value(<<"default_field">>, Props, {[]}) of
- Bool when is_boolean(Bool) ->
- Bool;
- {[]} ->
- true;
- {Opts}->
- couch_util:get_value(<<"enabled">>, Opts, true)
- end.
-
-
-get_index_array_lengths(Props) ->
- couch_util:get_value(<<"index_array_lengths">>, Props, true).
-
-
-add_default_text_field(Fields) ->
- DefaultFields = add_default_text_field(Fields, []),
- DefaultFields ++ Fields.
-
-
-add_default_text_field([], Acc) ->
- Acc;
-add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) ->
- NewAcc = [{<<"$default">>, <<"string">>, Value} | Acc],
- add_default_text_field(Rest, NewAcc);
-add_default_text_field([_ | Rest], Acc) ->
- add_default_text_field(Rest, Acc).
-
-
-%% index of all field names
-get_field_names(Fields) ->
- FieldNameSet = lists:foldl(fun({Name, _, _}, Set) ->
- gb_sets:add([<<"$fieldnames">>, Name, []], Set)
- end, gb_sets:new(), Fields),
- gb_sets:to_list(FieldNameSet).
-
-
-convert_text_fields([]) ->
- [];
-convert_text_fields([{Name, _Type, Value} | Rest]) ->
- [[Name, Value, []] | convert_text_fields(Rest)].
-
-
-should_index(Selector, Doc) ->
- % We should do this
- NormSelector = mango_selector:normalize(Selector),
- Matches = mango_selector:match(NormSelector, Doc),
- IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
- <<"_design/", _/binary>> -> true;
- _ -> false
- end,
- Matches and not IsDesign.
-
-
-get_text_field_list(IdxProps) ->
- case couch_util:get_value(<<"fields">>, IdxProps) of
- Fields when is_list(Fields) ->
- RawList = lists:flatmap(fun get_text_field_info/1, Fields),
- [mango_util:lucene_escape_user(Field) || Field <- RawList];
- _ ->
- all_fields
- end.
-
-
-get_text_field_info({Props}) ->
- Name = couch_util:get_value(<<"name">>, Props),
- Type0 = couch_util:get_value(<<"type">>, Props),
- if not is_binary(Name) -> []; true ->
- Type = get_text_field_type(Type0),
- [iolist_to_binary([Name, ":", Type])]
- end.
-
-
-get_text_field_type(<<"number">>) ->
- <<"number">>;
-get_text_field_type(<<"boolean">>) ->
- <<"boolean">>;
-get_text_field_type(_) ->
- <<"string">>.
-
-
-make_text_field(TAcc, Type, Value) ->
- FieldName = make_text_field_name(TAcc#tacc.path, Type),
- Fields = TAcc#tacc.fields,
- case Fields == all_fields orelse lists:member(FieldName, Fields) of
- true ->
- [{FieldName, Type, Value}];
- false ->
- []
- end.
-
-
-make_text_field_name([P | Rest], Type) ->
- Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
- Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
- iolist_to_binary(mango_util:join(".", Escaped)).
-
-
-validate_index_info(IndexInfo) ->
- IdxTypes = [mango_idx_view, mango_idx_text],
- Results = lists:foldl(fun(IdxType, Results0) ->
- try
- IdxType:validate_index_def(IndexInfo),
- [valid_index | Results0]
- catch _:_ ->
- [invalid_index | Results0]
- end
- end, [], IdxTypes),
- lists:member(valid_index, Results).
-
-
--ifdef(TEST).
-
--include_lib("eunit/include/eunit.hrl").
-
-handle_garbage_collect_cast_test() ->
- ?assertEqual({noreply, []}, handle_cast(garbage_collect, [])).
-
-handle_stop_cast_test() ->
- ?assertEqual({stop, normal, []}, handle_cast(stop, [])).
-
-handle_invalid_cast_test() ->
- ?assertEqual({stop, {invalid_cast, random}, []}, handle_cast(random, [])).
-
--endif.
[couchdb] 16/18: Update mango test creds to same as elixir tests
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1bc7c54ac2aee5302ce53a7a0c41867495f34704
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 11:39:32 2020 +0200
Update mango test creds to same as elixir tests
---
Makefile | 2 +-
src/mango/test/README.md | 4 ++--
src/mango/test/mango.py | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index b3eb64c..9622c0e 100644
--- a/Makefile
+++ b/Makefile
@@ -349,7 +349,7 @@ mango-test: devclean all
@cd src/mango && \
python3 -m venv .venv && \
.venv/bin/python3 -m pip install -r requirements.txt
- @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=testuser:testpass '.venv/bin/python3 -m nose --with-xunit'
+ @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=adm:pass '.venv/bin/python3 -m nose --with-xunit'
################################################################################
# Developing
diff --git a/src/mango/test/README.md b/src/mango/test/README.md
index 509e32e..08693a2 100644
--- a/src/mango/test/README.md
+++ b/src/mango/test/README.md
@@ -23,7 +23,7 @@ Test configuration
The following environment variables can be used to configure the test fixtures:
* `COUCH_HOST` - root url (including port) of the CouchDB instance to run the tests against. Default is `"http://127.0.0.1:15984"`.
- * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"testuser"`.
- * `COUCH_PASSWORD` - CouchDB password. Default is `"testpass"`.
+ * `COUCH_USER` - CouchDB username (with admin premissions). Default is `"adm"`.
+ * `COUCH_PASSWORD` - CouchDB password. Default is `"pass"`.
* `COUCH_AUTH_HEADER` - Optional Authorization header value. If specified, this is used instead of basic authentication with the username/password variables above.
* `MANGO_TEXT_INDEXES` - Set to `"1"` to run the tests only applicable to text indexes.
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 638de47..db0fab0 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -48,8 +48,8 @@ class Database(object):
dbname,
host="127.0.0.1",
port="15984",
- user="testuser",
- password="testpass",
+ user="adm",
+ password="pass",
):
root_url = get_from_environment("COUCH_HOST", "http://{}:{}".format(host, port))
auth_header = get_from_environment("COUCH_AUTH_HEADER", None)
[couchdb] 10/18: remove partition opts from mango
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e28eb6b791d3421753f6806d9c0a42e9d0f7b4b1
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 24 14:23:09 2020 +0200
remove partition opts from mango
---
src/mango/src/mango_cursor.erl | 1 -
src/mango/src/mango_cursor_text.erl | 9 ---
src/mango/src/mango_cursor_view.erl | 6 --
src/mango/src/mango_error.erl | 14 ----
src/mango/src/mango_httpd.erl | 21 +-----
src/mango/src/mango_idx.erl | 126 +++---------------------------------
src/mango/src/mango_idx.hrl | 1 -
src/mango/src/mango_idx_text.erl | 1 -
src/mango/src/mango_idx_view.erl | 1 -
src/mango/src/mango_opts.erl | 30 ---------
10 files changed, 10 insertions(+), 200 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index b1cb414..f16765b 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -72,7 +72,6 @@ explain(#cursor{}=Cursor) ->
{[
{dbname, mango_idx:dbname(Idx)},
{index, mango_idx:to_json(Idx)},
- {partitioned, mango_idx:partitioned(Idx)},
{selector, Selector},
{opts, {Opts}},
{limit, Limit},
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 43ef84e..ccf58ad 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -77,7 +77,6 @@ explain(Cursor) ->
} = Cursor,
[
{'query', mango_selector_text:convert(Selector)},
- {partition, get_partition(Opts, null)},
{sort, sort_query(Opts, Selector)}
].
@@ -95,7 +94,6 @@ execute(Cursor, UserFun, UserAcc) ->
Query = mango_selector_text:convert(Selector),
QueryArgs = #index_query_args{
q = Query,
- partition = get_partition(Opts, nil),
sort = sort_query(Opts, Selector),
raw_bookmark = true
},
@@ -250,13 +248,6 @@ sort_query(Opts, Selector) ->
end.
-get_partition(Opts, Default) ->
- case couch_util:get_value(partition, Opts) of
- <<>> -> Default;
- Else -> Else
- end.
-
-
get_bookmark(Opts) ->
case lists:keyfind(bookmark, 1, Opts) of
{_, BM} when is_list(BM), BM /= [] ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 240ef50..bced842 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -73,7 +73,6 @@ explain(Cursor) ->
{include_docs, Args#mrargs.include_docs},
{view_type, Args#mrargs.view_type},
{reduce, Args#mrargs.reduce},
- {partition, couch_mrview_util:get_extra(Args, partition, null)},
{start_key, maybe_replace_max_json(Args#mrargs.start_key)},
{end_key, maybe_replace_max_json(Args#mrargs.end_key)},
{direction, Args#mrargs.direction},
@@ -410,11 +409,6 @@ apply_opts([{update, false} | Rest], Args) ->
update = false
},
apply_opts(Rest, NewArgs);
-apply_opts([{partition, <<>>} | Rest], Args) ->
- apply_opts(Rest, Args);
-apply_opts([{partition, Partition} | Rest], Args) when is_binary(Partition) ->
- NewArgs = couch_mrview_util:set_extra(Args, partition, Partition),
- apply_opts(Rest, NewArgs);
apply_opts([{_, _} | Rest], Args) ->
% Ignore unknown options
apply_opts(Rest, Args).
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
index bb545ad..9ac8f63 100644
--- a/src/mango/src/mango_error.erl
+++ b/src/mango/src/mango_error.erl
@@ -28,13 +28,6 @@ info(mango_idx, {no_usable_index, missing_sort_index}) ->
<<"No index exists for this sort, "
"try indexing by the sort fields.">>
};
-info(mango_idx, {no_usable_index, missing_sort_index_partitioned}) ->
- {
- 400,
- <<"no_usable_index">>,
- <<"No partitioned index exists for this sort, "
- "try indexing by the sort fields.">>
- };
info(mango_idx, {no_usable_index, missing_sort_index_global}) ->
{
400,
@@ -118,13 +111,6 @@ info(mango_idx, {invalid_index_type, BadType}) ->
<<"invalid_index">>,
fmt("Invalid type for index: ~s", [BadType])
};
-info(mango_idx, {partitioned_option_mismatch, BadDDoc}) ->
- {
- 400,
- <<"invalid_partitioned_option">>,
- fmt("Requested partitioned option does not match existing value on"
- " design document ~s", [BadDDoc])
- };
info(mango_idx, invalid_query_ddoc_language) ->
{
400,
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 379d2e1..946d7e4 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -170,7 +170,7 @@ handle_index_req(#httpd{path_parts=[_, _, _DDocId0, _Type, _Name]}=Req, _Db) ->
handle_explain_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- Body = maybe_set_partition(Req),
+ Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
Resp = mango_crud:explain(Db, Sel, Opts),
@@ -182,7 +182,7 @@ handle_explain_req(Req, _Db) ->
handle_find_req(#httpd{method='POST'}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
- Body = maybe_set_partition(Req),
+ Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
{ok, Resp0} = start_find_resp(Req),
@@ -231,23 +231,6 @@ get_idx_del_opts(Req) ->
end.
-maybe_set_partition(Req) ->
- {Props} = chttpd:json_body_obj(Req),
- case chttpd:qs_value(Req, "partition", undefined) of
- undefined ->
- {Props};
- Partition ->
- case couch_util:get_value(<<"partition">>, Props) of
- undefined ->
- {[{<<"partition">>, ?l2b(Partition)} | Props]};
- Partition ->
- {Props};
- OtherPartition ->
- ?MANGO_ERROR({bad_partition, OtherPartition})
- end
- end.
-
-
convert_to_design_id(DDocId) ->
case DDocId of
<<"_design/", _/binary>> -> DDocId;
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 5d06a8f..0f79bdb 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -33,7 +33,6 @@
name/1,
type/1,
def/1,
- partitioned/1,
opts/1,
columns/1,
is_usable/3,
@@ -64,13 +63,12 @@ get_usable_indexes(Db, Selector, Opts) ->
ExistingIndexes
),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes0 = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
- UsableIndexes1 = filter_partition_indexes(UsableIndexes0, Opts),
+ UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
SortFields = get_sort_fields(Opts),
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
- case lists:filter(UsableFilter, UsableIndexes1) of
+ case lists:filter(UsableFilter, UsableIndexes) of
[] ->
mango_sort_error(Db, Opts);
UsableIndexes ->
@@ -78,15 +76,8 @@ get_usable_indexes(Db, Selector, Opts) ->
end.
-mango_sort_error(Db, Opts) ->
- case {fabric_util:is_partitioned(Db), is_opts_partitioned(Opts)} of
- {false, _} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index});
- {true, true} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_partitioned});
- {true, false} ->
- ?MANGO_ERROR({no_usable_index, missing_sort_index_global})
- end.
+mango_sort_error(_Db, _Opts) ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index}).
recover(Db) ->
@@ -124,7 +115,6 @@ new(Db, Opts) ->
name = IdxName,
type = Type,
def = Def,
- partitioned = get_idx_partitioned(Opts),
opts = filter_opts(Opts)
}}.
@@ -136,11 +126,10 @@ validate_new(Idx, Db) ->
add(DDoc, Idx) ->
Mod = idx_mod(Idx),
- {ok, NewDDoc1} = Mod:add(DDoc, Idx),
- NewDDoc2 = set_ddoc_partitioned(NewDDoc1, Idx),
+ {ok, NewDDoc} = Mod:add(DDoc, Idx),
% Round trip through JSON for normalization
- Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)),
- {ok, NewDDoc2#doc{body = Body}}.
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
+ {ok, NewDDoc#doc{body = Body}}.
remove(DDoc, Idx) ->
@@ -192,8 +181,7 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc,
- partitioned = get_idx_partitioned(Db, Props)
+ ddoc = DDoc
}
end, Idxs).
@@ -230,10 +218,6 @@ def(#idx{def=Def}) ->
Def.
-partitioned(#idx{partitioned=Partitioned}) ->
- Partitioned.
-
-
opts(#idx{opts=Opts}) ->
Opts.
@@ -350,97 +334,6 @@ gen_name(Idx, Opts0) ->
mango_util:enc_hex(Sha).
-get_idx_partitioned(Opts) ->
- case proplists:get_value(partitioned, Opts) of
- B when is_boolean(B) ->
- B;
- db_default ->
- % Default to the partitioned setting on
- % the database.
- undefined
- end.
-
-
-set_ddoc_partitioned(DDoc, Idx) ->
- % We have to verify that the new index being added
- % to this design document either matches the current
- % ddoc's design options *or* this is a new design doc
- #doc{
- id = DDocId,
- revs = Revs,
- body = {BodyProps}
- } = DDoc,
- OldDOpts = couch_util:get_value(<<"options">>, BodyProps),
- OldOpt = case OldDOpts of
- {OldDOptProps} when is_list(OldDOptProps) ->
- couch_util:get_value(<<"partitioned">>, OldDOptProps);
- _ ->
- undefined
- end,
- % If new matches old we're done
- if Idx#idx.partitioned == OldOpt -> DDoc; true ->
- % If we're creating a ddoc then we can set the options
- case Revs == {0, []} of
- true when Idx#idx.partitioned /= undefined ->
- set_ddoc_partitioned_option(DDoc, Idx#idx.partitioned);
- true when Idx#idx.partitioned == undefined ->
- DDoc;
- false ->
- ?MANGO_ERROR({partitioned_option_mismatch, DDocId})
- end
- end.
-
-
-set_ddoc_partitioned_option(DDoc, Partitioned) ->
- #doc{
- body = {BodyProps}
- } = DDoc,
- NewProps = case couch_util:get_value(<<"options">>, BodyProps) of
- {Existing} when is_list(Existing) ->
- Opt = {<<"partitioned">>, Partitioned},
- New = lists:keystore(<<"partitioned">>, 1, Existing, Opt),
- lists:keystore(<<"options">>, 1, BodyProps, {<<"options">>, New});
- undefined ->
- New = {<<"options">>, {[{<<"partitioned">>, Partitioned}]}},
- lists:keystore(<<"options">>, 1, BodyProps, New)
- end,
- DDoc#doc{body = {NewProps}}.
-
-
-get_idx_partitioned(Db, DDocProps) ->
- Default = fabric_util:is_partitioned(Db),
- case couch_util:get_value(<<"options">>, DDocProps) of
- {DesignOpts} ->
- case couch_util:get_value(<<"partitioned">>, DesignOpts) of
- P when is_boolean(P) ->
- P;
- undefined ->
- Default
- end;
- undefined ->
- Default
- end.
-
-is_opts_partitioned(Opts) ->
- case couch_util:get_value(partition, Opts, <<>>) of
- <<>> ->
- false;
- Partition when is_binary(Partition) ->
- true
- end.
-
-
-filter_partition_indexes(Indexes, Opts) ->
- PFilt = case is_opts_partitioned(Opts) of
- false ->
- fun(#idx{partitioned = P}) -> not P end;
- true ->
- fun(#idx{partitioned = P}) -> P end
- end,
- Filt = fun(Idx) -> type(Idx) == <<"special">> orelse PFilt(Idx) end,
- lists:filter(Filt, Indexes).
-
-
filter_opts([]) ->
[];
filter_opts([{user_ctx, _} | Rest]) ->
@@ -453,8 +346,6 @@ filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{w, _} | Rest]) ->
filter_opts(Rest);
-filter_opts([{partitioned, _} | Rest]) ->
- filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
@@ -488,7 +379,6 @@ index(SelectorName, Selector) ->
<<"Selected">>,<<"json">>,
{[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
{SelectorName,{Selector}}]},
- false,
[{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
}.
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 9725950..712031b 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,6 +16,5 @@
name,
type,
def,
- partitioned,
opts
}).
diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl
index 1d4becf..71eaf11 100644
--- a/src/mango/src/mango_idx_text.erl
+++ b/src/mango/src/mango_idx_text.erl
@@ -100,7 +100,6 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {partitioned, Idx#idx.partitioned},
{def, {def_to_json(Idx#idx.def)}}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 3791149..2d784b6 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -104,7 +104,6 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {partitioned, Idx#idx.partitioned},
{def, {def_to_json(Idx#idx.def)}}
]}.
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 92c07f7..7bae9c9 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -34,7 +34,6 @@
validate_sort/1,
validate_fields/1,
validate_bulk_delete/1,
- validate_partitioned/1,
default_limit/0
]).
@@ -71,12 +70,6 @@ validate_idx_create({Props}) ->
{optional, true},
{default, 2},
{validator, fun is_pos_integer/1}
- ]},
- {<<"partitioned">>, [
- {tag, partitioned},
- {optional, true},
- {default, db_default},
- {validator, fun validate_partitioned/1}
]}
],
validate(Props, Opts).
@@ -124,12 +117,6 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
- {<<"partition">>, [
- {tag, partition},
- {optional, true},
- {default, <<>>},
- {validator, fun validate_partition/1}
- ]},
{<<"r">>, [
{tag, r},
{optional, true},
@@ -309,23 +296,6 @@ validate_fields(Value) ->
mango_fields:new(Value).
-validate_partitioned(true) ->
- {ok, true};
-validate_partitioned(false) ->
- {ok, false};
-validate_partitioned(db_default) ->
- {ok, db_default};
-validate_partitioned(Else) ->
- ?MANGO_ERROR({invalid_partitioned_value, Else}).
-
-
-validate_partition(<<>>) ->
- {ok, <<>>};
-validate_partition(Partition) ->
- couch_partition:validate_partition(Partition),
- {ok, Partition}.
-
-
validate_opts([], Props, Acc) ->
{Props, lists:reverse(Acc)};
validate_opts([{Name, Desc} | Rest], Props, Acc) ->
[couchdb] 13/18: Remove quorum stats
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit dd4444209aaf43c8b7cc6aa422d1b7eeb96dad26
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:04:41 2020 +0200
Remove quorum stats
Removing quorum stats since they are not relevant with FDB.
---
src/mango/src/mango_cursor.erl | 5 +----
src/mango/src/mango_execution_stats.erl | 8 --------
src/mango/src/mango_execution_stats.hrl | 1 -
3 files changed, 1 insertion(+), 13 deletions(-)
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index f16765b..db4e981 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -205,12 +205,9 @@ invalid_index_warning_int(_, _) ->
% returned, implying a lot of in-memory filtering
index_scan_warning(#execution_stats {
totalDocsExamined = Docs,
- totalQuorumDocsExamined = DocsQuorum,
resultsReturned = ResultCount
}) ->
- % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
- DocsScanned = Docs + DocsQuorum,
- Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount),
+ Ratio = calculate_index_scan_ratio(Docs, ResultCount),
Threshold = config:get_integer("mango", "index_scan_warning_threshold", 10),
case Threshold > 0 andalso Ratio > Threshold of
true ->
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 5878a31..fe9d27b 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -18,7 +18,6 @@
incr_keys_examined/1,
incr_docs_examined/1,
incr_docs_examined/2,
- incr_quorum_docs_examined/1,
incr_results_returned/1,
log_start/1,
log_end/1,
@@ -33,7 +32,6 @@ to_json(Stats) ->
{[
{total_keys_examined, Stats#execution_stats.totalKeysExamined},
{total_docs_examined, Stats#execution_stats.totalDocsExamined},
- {total_quorum_docs_examined, Stats#execution_stats.totalQuorumDocsExamined},
{results_returned, Stats#execution_stats.resultsReturned},
{execution_time_ms, Stats#execution_stats.executionTimeMs}
]}.
@@ -55,12 +53,6 @@ incr_docs_examined(Stats, N) ->
}.
-incr_quorum_docs_examined(Stats) ->
- Stats#execution_stats {
- totalQuorumDocsExamined = Stats#execution_stats.totalQuorumDocsExamined + 1
- }.
-
-
incr_results_returned(Stats) ->
couch_stats:increment_counter([mango, results_returned]),
Stats#execution_stats {
diff --git a/src/mango/src/mango_execution_stats.hrl b/src/mango/src/mango_execution_stats.hrl
index ea5ed5e..783c1e7 100644
--- a/src/mango/src/mango_execution_stats.hrl
+++ b/src/mango/src/mango_execution_stats.hrl
@@ -13,7 +13,6 @@
-record(execution_stats, {
totalKeysExamined = 0,
totalDocsExamined = 0,
- totalQuorumDocsExamined = 0,
resultsReturned = 0,
executionStartTime,
executionTimeMs
[couchdb] 15/18: Update Mango query to work with couch_views
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 377b0c26c845a5c5152ef493f833889b45625f4a
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 16:21:22 2020 +0200
Update Mango query to work with couch_views
---
src/couch_views/test/couch_views_map_test.erl | 41 +++++++++-
src/mango/src/mango_crud.erl | 24 ++----
src/mango/src/mango_cursor.erl | 4 +-
src/mango/src/mango_cursor_view.erl | 70 +++++------------
src/mango/src/mango_httpd.erl | 105 ++++++++++++++------------
src/mango/src/mango_idx.erl | 13 +++-
src/mango/src/mango_idx_view.erl | 5 +-
src/mango/src/mango_idx_view.hrl | 13 ----
src/mango/src/mango_util.erl | 11 +--
9 files changed, 139 insertions(+), 147 deletions(-)
diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl
index f8ba183..7d1e94b 100644
--- a/src/couch_views/test/couch_views_map_test.erl
+++ b/src/couch_views/test/couch_views_map_test.erl
@@ -14,6 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
+-include("couch_views.hrl").
-define(TDEF(A), {atom_to_list(A), fun A/0}).
@@ -56,7 +57,8 @@ map_views_test_() ->
?TDEF(should_map_duplicate_keys),
?TDEF(should_map_with_doc_emit),
?TDEF(should_map_update_is_false),
- ?TDEF(should_map_update_is_lazy)
+ ?TDEF(should_map_update_is_lazy),
+ ?TDEF(should_map_wait_for_interactive)
% fun should_give_ext_size_seq_indexed_test/1
]
}
@@ -419,6 +421,25 @@ should_map_update_is_lazy() ->
?assertEqual(Expect, Result2).
+should_map_wait_for_interactive() ->
+ DbName = ?tempdb(),
+ {ok, Db} = fabric2_db:create(DbName, [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_interactive_ddoc(),
+ Docs = make_docs(101),
+
+ fabric2_db:update_docs(Db, Docs),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ Result = couch_views:query(Db, DDoc, <<"idx_01">>, fun default_cb/2, [],
+ #{limit => 3}),
+ ?assertEqual({ok, [
+ {row, [{id, <<"1">>}, {key, 1}, {value, 1}]},
+ {row, [{id, <<"2">>}, {key, 2}, {value, 2}]},
+ {row, [{id, <<"3">>}, {key, 3}, {value, 3}]}
+ ]}, Result).
+
+
% should_give_ext_size_seq_indexed_test(Db) ->
% DDoc = couch_doc:from_json_obj({[
% {<<"_id">>, <<"_design/seqdoc">>},
@@ -510,6 +531,24 @@ create_ddoc() ->
]}}
]}).
+create_interactive_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc_interactive">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, <<
+ "function(doc) {"
+ "if (doc.val) {"
+ "emit(doc.val, doc.val);"
+ "}"
+ "}">>}
+ ]}}
+ ]}},
+ {<<"autoupdate">>, false},
+ {<<"interactive">>, true}
+ ]}).
+
make_docs(Count) ->
[doc(I) || I <- lists:seq(1, Count)].
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 42717ff..66cef65 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -33,9 +33,8 @@ insert(Db, #doc{}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
-insert(Db, Docs, Opts0) when is_list(Docs) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- case fabric:update_docs(Db, Docs, Opts1) of
+insert(Db, Docs, Opts) when is_list(Docs) ->
+ case fabric2_db:update_docs(Db, Docs, Opts) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -45,9 +44,8 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
end.
-find(Db, Selector, Callback, UserAcc, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+find(Db, Selector, Callback, UserAcc, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -97,21 +95,11 @@ delete(Db, Selector, Options) ->
end.
-explain(Db, Selector, Opts0) ->
- Opts1 = maybe_add_user_ctx(Db, Opts0),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
+explain(Db, Selector, Opts) ->
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts),
mango_cursor:explain(Cursor).
-maybe_add_user_ctx(Db, Opts) ->
- case lists:keyfind(user_ctx, 1, Opts) of
- {user_ctx, _} ->
- Opts;
- false ->
- [{user_ctx, couch_db:get_user_ctx(Db)} | Opts]
- end.
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index db4e981..63b449c 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -48,7 +48,9 @@
create(Db, Selector0, Opts) ->
Selector = mango_selector:normalize(Selector0),
- UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+ UsableIndexes = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ mango_idx:get_usable_indexes(TxDb, Selector, Opts)
+ end),
case mango_cursor:maybe_filter_indexes_by_ddoc(UsableIndexes, Opts) of
[] ->
% use_index doesn't match a valid index - fall back to a valid one
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index b88f6ea..4960fa1 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -31,9 +31,7 @@
-include_lib("fabric/include/fabric.hrl").
-include("mango_cursor.hrl").
--include("mango_idx_view.hrl").
--define(HEARTBEAT_INTERVAL_IN_USEC, 4000000).
create(Db, Indexes, Selector, Opts) ->
FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -91,7 +89,8 @@ maybe_replace_max_json(?MAX_STR) ->
<<"<MAX>">>;
maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
- H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+ MAX_VAL = couch_views_encoding:max(),
+ H1 = if H == MAX_VAL -> <<"<MAX>">>;
true -> H
end,
[H1 | maybe_replace_max_json(T)];
@@ -100,7 +99,7 @@ maybe_replace_max_json(EndKey) ->
EndKey.
-base_args(#cursor{index = Idx, selector = Selector} = Cursor) ->
+base_args(#cursor{index = Idx} = Cursor) ->
{StartKey, EndKey} = case Cursor#cursor.ranges of
[empty] ->
{null, null};
@@ -132,18 +131,19 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
#cursor{opts = Opts, bookmark = Bookmark} = Cursor,
Args0 = apply_opts(Opts, BaseArgs),
Args = mango_json_bookmark:update_args(Bookmark, Args0),
- UserCtx = couch_util:get_value(user_ctx, Opts, #user_ctx{}),
- DbOpts = [{user_ctx, UserCtx}],
Result = case mango_idx:def(Idx) of
all_docs ->
CB = fun ?MODULE:handle_all_docs_message/2,
- fabric:all_docs(Db, DbOpts, CB, Cursor, Args);
+ AllDocOpts = fabric2_util:all_docs_view_opts(Args)
+ ++ [{restart_tx, true}],
+ fabric2_db:fold_docs(Db, CB, Cursor, AllDocOpts);
_ ->
CB = fun ?MODULE:handle_message/2,
% Normal view
- DDoc = ddocid(Idx),
+ DDocId = mango_idx:ddoc(Idx),
+ {ok, DDoc} = fabric2_db:open_doc(Db, DDocId),
Name = mango_idx:name(Idx),
- fabric:query_view(Db, DbOpts, DDoc, Name, CB, Cursor, Args)
+ couch_views:query(Db, DDoc, Name, CB, Cursor, Args)
end,
case Result of
{ok, LastCursor} ->
@@ -227,7 +227,7 @@ choose_best_index(_DbName, IndexRanges) ->
handle_message({meta, _}, Cursor) ->
{ok, Cursor};
handle_message({row, Props}, Cursor) ->
- case doc_member(Cursor, Props) of
+ case match_doc(Cursor, Props) of
{ok, Doc, {execution_stats, Stats}} ->
Cursor1 = Cursor#cursor {
execution_stats = Stats
@@ -280,15 +280,6 @@ handle_doc(C, _Doc) ->
{stop, C}.
-ddocid(Idx) ->
- case mango_idx:ddoc(Idx) of
- <<"_design/", Rest/binary>> ->
- Rest;
- Else ->
- Else
- end.
-
-
apply_opts([], Args) ->
Args;
apply_opts([{conflicts, true} | Rest], Args) ->
@@ -340,41 +331,18 @@ apply_opts([{_, _} | Rest], Args) ->
apply_opts(Rest, Args).
-doc_member(Cursor, RowProps) ->
- Db = Cursor#cursor.db,
- Opts = Cursor#cursor.opts,
- ExecutionStats = Cursor#cursor.execution_stats,
- Selector = Cursor#cursor.selector,
- case couch_util:get_value(doc, RowProps) of
- {DocProps} ->
- % only matching documents are returned; the selector
- % is evaluated at the shard level in view_cb({row, Row},
- {ok, {DocProps}, {execution_stats, ExecutionStats}};
- undefined ->
- % an undefined doc was returned, indicating we should
- % perform a quorum fetch
- ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
- couch_stats:increment_counter([mango, quorum_docs_examined]),
- Id = couch_util:get_value(id, RowProps),
- case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
- {ok, #doc{}=DocProps} ->
- Doc = couch_doc:to_json_obj(DocProps, []),
- match_doc(Selector, Doc, ExecutionStats1);
- Else ->
- Else
- end;
- _ ->
- % no doc, no match
- {no_match, null, {execution_stats, ExecutionStats}}
- end.
-
-
-match_doc(Selector, Doc, ExecutionStats) ->
+match_doc(Cursor, RowProps) ->
+ #cursor{
+ execution_stats = Stats0,
+ selector = Selector
+ } = Cursor,
+ Stats1 = mango_execution_stats:incr_docs_examined(Stats0, 1),
+ Doc = couch_util:get_value(doc, RowProps),
case mango_selector:match(Selector, Doc) of
true ->
- {ok, Doc, {execution_stats, ExecutionStats}};
+ {ok, Doc, {execution_stats, Stats1}};
false ->
- {no_match, Doc, {execution_stats, ExecutionStats}}
+ {no_match, Doc, {execution_stats, Stats1}}
end.
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 1054c74..94aa866 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -32,9 +32,8 @@
threshold = 1490
}).
-handle_req(#httpd{} = Req, Db0) ->
+handle_req(#httpd{} = Req, Db) ->
try
- Db = set_user_ctx(Req, Db0),
handle_req_int(Req, Db)
catch
throw:{mango_error, Module, Reason} ->
@@ -61,7 +60,9 @@ handle_req_int(_, _) ->
handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
chttpd:qs(Req)),
- Idxs = lists:sort(mango_idx:list(Db)),
+ Idxs = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ lists:sort(mango_idx:list(TxDb))
+ end),
JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
TotalRows = length(JsonIdxs0),
Limit = case couch_util:get_value(limit, Params, TotalRows) of
@@ -87,25 +88,27 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, Idx0} = mango_idx:new(Db, Opts),
{ok, Idx} = mango_idx:validate_new(Idx0, Db),
DbOpts = [{user_ctx, Req#httpd.user_ctx}, deleted, ejson_body],
- {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), DbOpts),
- Id = Idx#idx.ddoc,
- Name = Idx#idx.name,
- Status = case mango_idx:add(DDoc, Idx) of
- {ok, DDoc} ->
- <<"exists">>;
- {ok, NewDDoc} ->
- case mango_crud:insert(Db, NewDDoc, Opts) of
- {ok, [{RespProps}]} ->
- case lists:keyfind(error, 1, RespProps) of
- {error, Reason} ->
- ?MANGO_ERROR({error_saving_ddoc, Reason});
- _ ->
- <<"created">>
- end;
- _ ->
- ?MANGO_ERROR(error_saving_ddoc)
- end
- end,
+ Id = mango_idx:ddoc(Idx),
+ Name = mango_idx:name(Idx),
+ Status = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ {ok, DDoc} = mango_util:load_ddoc(TxDb, Id, DbOpts),
+ case mango_idx:add(DDoc, Idx) of
+ {ok, DDoc} ->
+ <<"exists">>;
+ {ok, NewDDoc} ->
+ case mango_crud:insert(TxDb, NewDDoc, Opts) of
+ {ok, [{RespProps}]} ->
+ case lists:keyfind(error, 1, RespProps) of
+ {error, Reason} ->
+ ?MANGO_ERROR({error_saving_ddoc, Reason});
+ _ ->
+ <<"created">>
+ end;
+ _ ->
+ ?MANGO_ERROR(error_saving_ddoc)
+ end
+ end
+ end),
chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) ->
@@ -118,19 +121,21 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
<<"_bulk_delete">>]}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
- Idxs = mango_idx:list(Db),
- DDocs = get_bulk_delete_ddocs(Opts),
- {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
- Id = {<<"id">>, DDocId},
- case mango_idx:delete(Filt, Db, Idxs, Opts) of
- {ok, true} ->
- {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
- {error, Error} ->
- {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
- end
- end, {[], []}, DDocs),
+ {Success, Fail} = fabric2_fdb:transactional(Db, fun (TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocs = get_bulk_delete_ddocs(Opts),
+ lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
+ Id = {<<"id">>, DDocId},
+ case mango_idx:delete(Filt, TxDb, Idxs, Opts) of
+ {ok, true} ->
+ {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
+ {error, Error} ->
+ {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
+ end
+ end, {[], []}, DDocs)
+ end),
chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]});
handle_index_req(#httpd{path_parts=[_, <<"_index">>,
@@ -144,15 +149,18 @@ handle_index_req(#httpd{method='DELETE',
handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
- Idxs = mango_idx:list(Db),
- DDocId = convert_to_design_id(DDocId0),
- Filt = fun(Idx) ->
- IsDDoc = mango_idx:ddoc(Idx) == DDocId,
- IsType = mango_idx:type(Idx) == Type,
- IsName = mango_idx:name(Idx) == Name,
- IsDDoc andalso IsType andalso IsName
- end,
- case mango_idx:delete(Filt, Db, Idxs, []) of
+ Result = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Idxs = mango_idx:list(TxDb),
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) ->
+ IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+ IsType = mango_idx:type(Idx) == Type,
+ IsName = mango_idx:name(Idx) == Name,
+ IsDDoc andalso IsType andalso IsName
+ end,
+ mango_idx:delete(Filt, TxDb, Idxs, [])
+ end),
+ case Result of
{ok, true} ->
chttpd:send_json(Req, {[{ok, true}]});
{error, not_found} ->
@@ -170,7 +178,9 @@ handle_explain_req(#httpd{method='POST'}=Req, Db) ->
Body = chttpd:json_body_obj(Req),
{ok, Opts0} = mango_opts:validate_find(Body),
{value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
- Resp = mango_crud:explain(Db, Sel, Opts),
+ Resp = fabric2_fdb:transactional(Db, fun(TxDb) ->
+ mango_crud:explain(TxDb, Sel, Opts)
+ end),
chttpd:send_json(Req, Resp);
handle_explain_req(Req, _Db) ->
@@ -195,11 +205,6 @@ handle_find_req(Req, _Db) ->
chttpd:send_method_not_allowed(Req, "POST").
-set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
- {ok, NewDb} = couch_db:set_user_ctx(Db, Ctx),
- NewDb.
-
-
get_bulk_delete_ddocs(Opts) ->
case lists:keyfind(docids, 1, Opts) of
{docids, DDocs} when is_list(DDocs) ->
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index ba9f68f..99a3588 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -86,7 +86,7 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
- BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
+ BuiltIndexes = remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
@@ -399,6 +399,17 @@ get_legacy_selector(Def) ->
Selector -> Selector
end.
+% remove any interactive indexes that are not built. If an index is not
+% interactive than we do not remove it as it will be built when queried
+remove_unbuilt_indexes(Indexes) ->
+ lists:filter(fun(Idx) ->
+ case Idx#idx.interactive of
+ true -> Idx#idx.build_status == ?INDEX_READY;
+ _ -> true
+ end
+ end, Indexes).
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 84be418..f80cc21 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,7 +34,6 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
--include("mango_idx_view.hrl").
validate_new(#idx{}=Idx, _Db) ->
@@ -183,11 +182,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
end_key([]) ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
end_key([{_, _, '$lt', Key} | Rest]) ->
case mango_json:special(Key) of
true ->
- [?MAX_JSON_OBJ];
+ [couch_views_encoding:max()];
false ->
[Key | end_key(Rest)]
end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
deleted file mode 100644
index 0d213e5..0000000
--- a/src/mango/src/mango_idx_view.hrl
+++ /dev/null
@@ -1,13 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 0d31f15..18a6439 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -85,14 +85,7 @@ open_doc(Db, DocId) ->
open_doc(Db, DocId, Options) ->
- case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
- {ok, Doc} ->
- {ok, Doc};
- {not_found, _} ->
- not_found;
- _ ->
- ?MANGO_ERROR({error_loading_doc, DocId})
- end.
+ fabric2_db:open_doc(Db, DocId, Options).
open_ddocs(Db) ->
@@ -111,7 +104,7 @@ load_ddoc(Db, DDocId, DbOpts) ->
case open_doc(Db, DDocId, DbOpts) of
{ok, Doc} ->
{ok, check_lang(Doc)};
- not_found ->
+ {not_found, missing} ->
Body = {[
{<<"language">>, <<"query">>}
]},
[couchdb] 02/18: move all_doc view options to fabric2_util
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cc2d2320c3acac05e76b3ce67dc42482e29aeaad
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 18 09:40:24 2020 +0200
move all_doc view options to fabric2_util
---
src/chttpd/src/chttpd_db.erl | 30 +-----------------------------
src/fabric/src/fabric2_util.erl | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 29 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 174d464..e9b33f0 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -867,7 +867,7 @@ all_docs_view(Req, Db, Keys, OP) ->
send_all_docs(Db, #mrargs{keys = undefined} = Args, VAcc0) ->
- Opts0 = all_docs_view_opts(Args),
+ Opts0 = fabric2_util:all_docs_view_opts(Args),
Opts = Opts0 ++ [{restart_tx, true}],
NS = couch_util:get_value(namespace, Opts),
FoldFun = case NS of
@@ -939,34 +939,6 @@ send_all_docs_keys(Db, #mrargs{} = Args, VAcc0) ->
end, VAcc1, Keys).
-all_docs_view_opts(Args) ->
- NS = couch_util:get_value(namespace, Args#mrargs.extra),
- StartKey = case Args#mrargs.start_key of
- undefined -> Args#mrargs.start_key_docid;
- SKey -> SKey
- end,
- EndKey = case Args#mrargs.end_key of
- undefined -> Args#mrargs.end_key_docid;
- EKey -> EKey
- end,
- StartKeyOpts = case StartKey of
- undefined -> [];
- _ -> [{start_key, fabric2_util:encode_all_doc_key(StartKey)}]
- end,
- EndKeyOpts = case {EndKey, Args#mrargs.inclusive_end} of
- {undefined, _} -> [];
- {_, false} -> [{end_key_gt, fabric2_util:encode_all_doc_key(EndKey)}];
- {_, true} -> [{end_key, fabric2_util:encode_all_doc_key(EndKey)}]
- end,
- [
- {dir, Args#mrargs.direction},
- {limit, Args#mrargs.limit},
- {skip, Args#mrargs.skip},
- {update_seq, Args#mrargs.update_seq},
- {namespace, NS}
- ] ++ StartKeyOpts ++ EndKeyOpts.
-
-
apply_args_to_keylist(Args, Keys0) ->
Keys1 = case Args#mrargs.direction of
fwd -> Keys0;
diff --git a/src/fabric/src/fabric2_util.erl b/src/fabric/src/fabric2_util.erl
index d74ef27..97bfedc 100644
--- a/src/fabric/src/fabric2_util.erl
+++ b/src/fabric/src/fabric2_util.erl
@@ -38,6 +38,7 @@
uuid/0,
encode_all_doc_key/1,
+ all_docs_view_opts/1,
pmap/2,
pmap/3
@@ -45,6 +46,7 @@
-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
revinfo_to_revs(RevInfo) ->
@@ -299,6 +301,42 @@ encode_all_doc_key(Term) when Term < <<>> -> <<>>;
encode_all_doc_key(_) -> <<255>>.
+all_docs_view_opts(#mrargs{} = Args) ->
+ NS = couch_util:get_value(namespace, Args#mrargs.extra),
+ StartKey = case Args#mrargs.start_key of
+ undefined -> Args#mrargs.start_key_docid;
+ SKey -> SKey
+ end,
+ EndKey = case Args#mrargs.end_key of
+ undefined -> Args#mrargs.end_key_docid;
+ EKey -> EKey
+ end,
+ StartKeyOpts = case StartKey of
+ undefined -> [];
+ _ -> [{start_key, encode_all_doc_key(StartKey)}]
+ end,
+ EndKeyOpts = case {EndKey, Args#mrargs.inclusive_end} of
+ {undefined, _} -> [];
+ {_, false} -> [{end_key_gt, encode_all_doc_key(EndKey)}];
+ {_, true} -> [{end_key, encode_all_doc_key(EndKey)}]
+ end,
+
+ DocOpts = case Args#mrargs.conflicts of
+ true -> [conflicts | Args#mrargs.doc_options];
+ _ -> Args#mrargs.doc_options
+ end,
+
+ [
+ {dir, Args#mrargs.direction},
+ {limit, Args#mrargs.limit},
+ {skip, Args#mrargs.skip},
+ {update_seq, Args#mrargs.update_seq},
+ {namespace, NS},
+ {include_docs, Args#mrargs.include_docs},
+ {doc_opts, DocOpts}
+ ] ++ StartKeyOpts ++ EndKeyOpts.
+
+
pmap(Fun, Args) ->
pmap(Fun, Args, []).
[couchdb] 05/18: Add couch_views_indexer build to creation
versionstamp
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9d27c6e817692ee98acdd1369b771ccb1b8bd79d
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 14:28:37 2020 +0200
Add couch_views_indexer build to creation versionstamp
This creates a versionstamp for when an indexed was created
and build status for indexes. if the index has a creation_vs, then
couch_views_indexer will built the index to this creation versionstamp.
---
src/couch_views/include/couch_views.hrl | 6 ++
src/couch_views/src/couch_views_fdb.erl | 76 +++++++++++++++++++++
src/couch_views/src/couch_views_indexer.erl | 77 ++++++++++++++++------
src/couch_views/src/couch_views_jobs.erl | 21 ++++--
src/couch_views/test/couch_views_indexer_test.erl | 42 +++++++++++-
.../test/couch_views_trace_index_test.erl | 5 +-
src/fabric/src/fabric2_fdb.erl | 12 ++--
7 files changed, 206 insertions(+), 33 deletions(-)
diff --git a/src/couch_views/include/couch_views.hrl b/src/couch_views/include/couch_views.hrl
index c40bb02..3d0110f 100644
--- a/src/couch_views/include/couch_views.hrl
+++ b/src/couch_views/include/couch_views.hrl
@@ -18,6 +18,8 @@
-define(VIEW_UPDATE_SEQ, 0).
-define(VIEW_ROW_COUNT, 1).
-define(VIEW_KV_SIZE, 2).
+-define(VIEW_BUILD_STATUS, 3).
+-define(VIEW_CREATION_VS, 4).
% Data keys
-define(VIEW_ID_RANGE, 0).
@@ -25,3 +27,7 @@
% jobs api
-define(INDEX_JOB_TYPE, <<"views">>).
+
+% indexing progress
+-define(INDEX_BUILDING, <<"building">>).
+-define(INDEX_READY, <<"ready">>).
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index a0224b2..3b008d4 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -13,6 +13,12 @@
-module(couch_views_fdb).
-export([
+ new_interactive_index/3,
+ new_creation_vs/3,
+ get_creation_vs/2,
+ get_build_status/2,
+ set_build_status/3,
+
get_update_seq/2,
set_update_seq/3,
@@ -39,6 +45,60 @@
-include_lib("fabric/include/fabric2.hrl").
+new_interactive_index(Db, Mrst, VS) ->
+ couch_views_fdb:new_creation_vs(Db, Mrst, VS),
+ couch_views_fdb:set_build_status(Db, Mrst, ?INDEX_BUILDING).
+
+
+%Interactive View Creation Versionstamp
+%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig) = VS
+
+new_creation_vs(TxDb, #mrst{} = Mrst, VS) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = creation_vs_key(TxDb, Mrst#mrst.sig),
+ Value = erlfdb_tuple:pack_vs({VS}),
+ ok = erlfdb:set_versionstamped_value(Tx, Key, Value).
+
+
+get_creation_vs(TxDb, #mrst{} = Mrst) ->
+ get_creation_vs(TxDb, Mrst#mrst.sig);
+
+get_creation_vs(TxDb, Sig) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = creation_vs_key(TxDb, Sig),
+ case erlfdb:wait(erlfdb:get(Tx, Key)) of
+ not_found ->
+ not_found;
+ EK ->
+ {VS} = erlfdb_tuple:unpack(EK),
+ VS
+ end.
+
+
+%Interactive View Build Status
+%(<db>, ?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig) = INDEX_BUILDING | INDEX_READY
+
+get_build_status(TxDb, #mrst{sig = Sig}) ->
+ #{
+ tx := Tx
+ } = TxDb,
+ Key = build_status_key(TxDb, Sig),
+ erlfdb:wait(erlfdb:get(Tx, Key)).
+
+
+set_build_status(TxDb, #mrst{sig = Sig}, State) ->
+ #{
+ tx := Tx
+ } = TxDb,
+
+ Key = build_status_key(TxDb, Sig),
+ ok = erlfdb:set(Tx, Key, State).
+
+
% View Build Sequence Access
% (<db>, ?DB_VIEWS, Sig, ?VIEW_UPDATE_SEQ) = Sequence
@@ -340,6 +400,22 @@ map_idx_range(DbPrefix, Sig, ViewId, MapKey, DocId) ->
erlfdb_tuple:range(Key, DbPrefix).
+creation_vs_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_CREATION_VS, Sig},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
+build_status_key(Db, Sig) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ Key = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_BUILD_STATUS, Sig},
+ erlfdb_tuple:pack(Key, DbPrefix).
+
+
process_rows(Rows) ->
Encoded = lists:map(fun({K, V}) ->
EK1 = couch_views_encoding:encode(K, key),
diff --git a/src/couch_views/src/couch_views_indexer.erl b/src/couch_views/src/couch_views_indexer.erl
index 0127bac..ab5aaad 100644
--- a/src/couch_views/src/couch_views_indexer.erl
+++ b/src/couch_views/src/couch_views_indexer.erl
@@ -18,7 +18,9 @@
-export([
- init/0
+ init/0,
+ map_docs/2,
+ write_docs/4
]).
-ifdef(TEST).
@@ -80,6 +82,7 @@ init() ->
db_seq => undefined,
view_seq => undefined,
last_seq => undefined,
+ view_vs => undefined,
job => Job,
job_data => Data,
count => 0,
@@ -174,22 +177,7 @@ update(#{} = Db, Mrst0, State0) ->
do_update(Db, Mrst0, State0) ->
fabric2_fdb:transactional(Db, fun(TxDb) ->
- % In the first iteration of update we need
- % to populate our db and view sequences
- State1 = case State0 of
- #{db_seq := undefined} ->
- ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst0),
- State0#{
- tx_db := TxDb,
- db_seq := fabric2_db:get_update_seq(TxDb),
- view_seq := ViewSeq,
- last_seq := ViewSeq
- };
- _ ->
- State0#{
- tx_db := TxDb
- }
- end,
+ State1 = get_update_start_state(TxDb, Mrst0, State0),
{ok, State2} = fold_changes(State1),
@@ -198,7 +186,8 @@ do_update(Db, Mrst0, State0) ->
doc_acc := DocAcc,
last_seq := LastSeq,
limit := Limit,
- limiter := Limiter
+ limiter := Limiter,
+ view_vs := ViewVS
} = State2,
DocAcc1 = fetch_docs(TxDb, DocAcc),
couch_rate:in(Limiter, Count),
@@ -210,6 +199,8 @@ do_update(Db, Mrst0, State0) ->
case Count < Limit of
true ->
+ maybe_set_build_status(TxDb, Mrst1, ViewVS,
+ ?INDEX_READY),
report_progress(State2, finished),
{Mrst1, finished};
false ->
@@ -224,6 +215,33 @@ do_update(Db, Mrst0, State0) ->
end).
+maybe_set_build_status(_TxDb, _Mrst1, not_found, _State) ->
+ ok;
+
+maybe_set_build_status(TxDb, Mrst1, _ViewVS, State) ->
+ couch_views_fdb:set_build_status(TxDb, Mrst1, State).
+
+
+% In the first iteration of update we need
+% to populate our db and view sequences
+get_update_start_state(TxDb, Mrst, #{db_seq := undefined} = State) ->
+ ViewVS = couch_views_fdb:get_creation_vs(TxDb, Mrst),
+ ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
+
+ State#{
+ tx_db := TxDb,
+ db_seq := fabric2_db:get_update_seq(TxDb),
+ view_vs := ViewVS,
+ view_seq := ViewSeq,
+ last_seq := ViewSeq
+ };
+
+get_update_start_state(TxDb, _Idx, State) ->
+ State#{
+ tx_db := TxDb
+ }.
+
+
fold_changes(State) ->
#{
view_seq := SinceSeq,
@@ -240,7 +258,8 @@ process_changes(Change, Acc) ->
#{
doc_acc := DocAcc,
count := Count,
- design_opts := DesignOpts
+ design_opts := DesignOpts,
+ view_vs := ViewVS
} = Acc,
#{
@@ -263,8 +282,22 @@ process_changes(Change, Acc) ->
last_seq := LastSeq
}
end,
- {ok, Acc1}.
+ DocVS = fabric2_fdb:seq_to_vs(LastSeq),
+
+ Go = maybe_stop_at_vs(ViewVS, DocVS),
+ {Go, Acc1}.
+
+
+maybe_stop_at_vs({versionstamp, _} = ViewVS, DocVS) when DocVS >= ViewVS ->
+ stop;
+
+maybe_stop_at_vs(_, _) ->
+ ok.
+
+
+map_docs(Mrst, []) ->
+ {Mrst, []};
map_docs(Mrst, Docs) ->
% Run all the non deleted docs through the view engine and
@@ -328,7 +361,9 @@ write_docs(TxDb, Mrst, Docs, State) ->
N + 1
end, 0, Docs),
- couch_views_fdb:set_update_seq(TxDb, Sig, LastSeq),
+ if LastSeq == false -> ok; true ->
+ couch_views_fdb:set_update_seq(TxDb, Sig, LastSeq)
+ end,
DocsNumber.
diff --git a/src/couch_views/src/couch_views_jobs.erl b/src/couch_views/src/couch_views_jobs.erl
index 1604841..b97e7ce 100644
--- a/src/couch_views/src/couch_views_jobs.erl
+++ b/src/couch_views/src/couch_views_jobs.erl
@@ -40,11 +40,12 @@ build_view(TxDb, Mrst, UpdateSeq) ->
end.
-build_view_async(TxDb, Mrst) ->
- JobId = job_id(TxDb, Mrst),
- JobData = job_data(TxDb, Mrst),
- DbUUID = fabric2_db:get_uuid(TxDb),
- couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(), fun(JTx) ->
+build_view_async(TxDb0, Mrst) ->
+ JobId = job_id(TxDb0, Mrst),
+ JobData = job_data(TxDb0, Mrst),
+ DbUUID = fabric2_db:get_uuid(TxDb0),
+ TxDb1 = ensure_correct_tx(TxDb0),
+ couch_jobs_fdb:tx(couch_jobs_fdb:get_jtx(TxDb1), fun(JTx) ->
case couch_jobs:get_job_data(JTx, ?INDEX_JOB_TYPE, JobId) of
{error, not_found} ->
ok;
@@ -59,6 +60,16 @@ build_view_async(TxDb, Mrst) ->
{ok, JobId}.
+ensure_correct_tx(#{tx := undefined} = TxDb) ->
+ TxDb;
+
+ensure_correct_tx(#{tx := Tx} = TxDb) ->
+ case erlfdb:is_read_only(Tx) of
+ true -> TxDb#{tx := undefined};
+ false -> TxDb
+ end.
+
+
wait_for_job(JobId, UpdateSeq) ->
case couch_jobs:subscribe(?INDEX_JOB_TYPE, JobId) of
{ok, Subscription, _State, _Data} ->
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index 43b5828..8ddb64b 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -51,7 +51,8 @@ indexer_test_() ->
?TDEF_FE(index_autoupdater_callback),
?TDEF_FE(handle_db_recreated_when_running),
?TDEF_FE(handle_db_recreated_after_finished),
- ?TDEF_FE(index_budget_is_changing)
+ ?TDEF_FE(index_budget_is_changing),
+ ?TDEF_FE(index_can_recover_from_crash, 60)
]
}
}
@@ -508,6 +509,41 @@ handle_db_recreated_after_finished(Db) ->
], Out2).
+index_can_recover_from_crash(Db) ->
+ ok = meck:new(config, [passthrough]),
+ ok = meck:expect(config, get_integer, fun(Section, Key, Default) ->
+ case Section == "couch_views" andalso Key == "change_limit" of
+ true -> 1;
+ _ -> Default
+ end
+ end),
+ meck:new(couch_eval, [passthrough]),
+ meck:expect(couch_eval, map_docs, fun(State, Docs) ->
+ Doc = hd(Docs),
+ case Doc#doc.id == <<"2">> of
+ true ->
+ % remove the mock so that next time the doc is processed
+ % it will work
+ meck:unload(couch_eval),
+ throw({fake_crash, test_jobs_restart});
+ false ->
+ meck:passthrough([State, Docs])
+ end
+ end),
+
+ DDoc = create_ddoc(),
+ Docs = make_docs(3),
+ {ok, _} = fabric2_db:update_doc(Db, DDoc, []),
+ {ok, _} = fabric2_db:update_docs(Db, Docs, []),
+
+ {ok, Out} = run_query(Db, DDoc, ?MAP_FUN1),
+ ?assertEqual([
+ row(<<"1">>, 1, 1),
+ row(<<"2">>, 2, 2),
+ row(<<"3">>, 3, 3)
+ ], Out).
+
+
row(Id, Key, Value) ->
{row, [
{id, Id},
@@ -603,6 +639,10 @@ create_ddoc(multi_emit_key_limit) ->
]}).
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
doc(Id) ->
doc(Id, Id).
diff --git a/src/couch_views/test/couch_views_trace_index_test.erl b/src/couch_views/test/couch_views_trace_index_test.erl
index f8a5ce5..5b15a4c 100644
--- a/src/couch_views/test/couch_views_trace_index_test.erl
+++ b/src/couch_views/test/couch_views_trace_index_test.erl
@@ -77,10 +77,13 @@ trace_single_doc(Db) ->
{ok, _} = fabric2_db:update_doc(Db, Doc, []),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+ HexSig = fabric2_util:to_hex(Mrst#mrst.sig),
JobData = #{
<<"db_name">> => DbName,
+ <<"db_uuid">> => fabric2_db:get_uuid(Db),
<<"ddoc_id">> => <<"_design/bar">>,
- <<"sig">> => fabric2_util:to_hex(Mrst#mrst.sig)
+ <<"sig">> => HexSig,
+ <<"retries">> => 0
},
meck:expect(couch_jobs, accept, 2, {ok, job, JobData}),
meck:expect(couch_jobs, update, 3, {ok, job}),
diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 912d4df..2295a56 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -64,6 +64,8 @@
seq_to_vs/1,
next_vs/1,
+ new_versionstamp/1,
+
debug_cluster/0,
debug_cluster/2
]).
@@ -1021,6 +1023,11 @@ next_vs({versionstamp, VS, Batch, TxId}) ->
{versionstamp, V, B, T}.
+new_versionstamp(Tx) ->
+ TxId = erlfdb:get_next_tx_id(Tx),
+ {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
+
+
debug_cluster() ->
debug_cluster(<<>>, <<16#FE, 16#FF, 16#FF>>).
@@ -1763,11 +1770,6 @@ get_transaction_id(Tx, LayerPrefix) ->
end.
-new_versionstamp(Tx) ->
- TxId = erlfdb:get_next_tx_id(Tx),
- {versionstamp, 16#FFFFFFFFFFFFFFFF, 16#FFFF, TxId}.
-
-
on_commit(Tx, Fun) when is_function(Fun, 0) ->
% Here we rely on Tx objects matching. However they contain a nif resource
% object. Before Erlang 20.0 those would have been represented as empty
[couchdb] 17/18: update mango tests to work with Mango on FDB
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 29ef7fb37007821edd8a0673226090cd47d48bab
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 11:39:58 2020 +0200
update mango tests to work with Mango on FDB
---
Makefile | 5 ++-
src/mango/test/01-index-crud-test.py | 15 ++++++++
src/mango/test/02-basic-find-test.py | 15 --------
src/mango/test/05-index-selection-test.py | 7 +++-
src/mango/test/12-use-correct-index-test.py | 38 +++++++++++--------
src/mango/test/13-stable-update-test.py | 51 --------------------------
src/mango/test/13-users-db-find-test.py | 7 +++-
src/mango/test/15-execution-stats-test.py | 7 +---
src/mango/test/17-multi-type-value-test.py | 4 +-
src/mango/test/19-find-conflicts.py | 7 ++--
src/mango/test/20-no-timeout-test.py | 32 ----------------
src/mango/test/22-build-wait-selected-index.py | 50 +++++++++++++++++++++++++
src/mango/test/mango.py | 20 +++++++---
src/mango/test/user_docs.py | 7 ++--
14 files changed, 129 insertions(+), 136 deletions(-)
diff --git a/Makefile b/Makefile
index 9622c0e..cc33c61 100644
--- a/Makefile
+++ b/Makefile
@@ -160,9 +160,10 @@ endif
.PHONY: check-fdb
check-fdb:
- make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric
+ make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric,mango
make elixir tests=test/elixir/test/basics_test.exs,test/elixir/test/replication_test.exs,test/elixir/test/map_test.exs,test/elixir/test/all_docs_test.exs,test/elixir/test/bulk_docs_test.exs
make exunit tests=src/couch_rate/test/exunit/
+ make mango-test
.PHONY: eunit
# target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options
@@ -349,7 +350,7 @@ mango-test: devclean all
@cd src/mango && \
python3 -m venv .venv && \
.venv/bin/python3 -m pip install -r requirements.txt
- @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=adm:pass '.venv/bin/python3 -m nose --with-xunit'
+ @cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=adm:pass --erlang-config=rel/files/eunit.config '.venv/bin/python3 -m nose -v --with-xunit'
################################################################################
# Developing
diff --git a/src/mango/test/01-index-crud-test.py b/src/mango/test/01-index-crud-test.py
index b602399..13ae300 100644
--- a/src/mango/test/01-index-crud-test.py
+++ b/src/mango/test/01-index-crud-test.py
@@ -113,6 +113,21 @@ class IndexCrudTests(mango.DbPerClass):
return
raise AssertionError("index not created")
+ def test_ignore_design_docs(self):
+ fields = ["baz", "foo"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+ self.db.save_doc({
+ "_id": "_design/ignore",
+ "views": {
+ "view1": {
+ "map": "function (doc) { emit(doc._id, 1)}"
+ }
+ }
+ })
+ indexes = self.db.list_indexes()
+ self.assertEqual(len(indexes), 2)
+
def test_read_idx_doc(self):
self.db.create_index(["foo", "bar"], name="idx_01")
self.db.create_index(["hello", "bar"])
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index afdba03..2a03a3a 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -100,16 +100,6 @@ class BasicFindTests(mango.UserDocsTests):
else:
raise AssertionError("bad find")
- def test_bad_r(self):
- bad_rs = ([None, True, False, 1.2, "no limit!", {"foo": "bar"}, [2]],)
- for br in bad_rs:
- try:
- self.db.find({"int": {"$gt": 2}}, r=br)
- except Exception as e:
- assert e.response.status_code == 400
- else:
- raise AssertionError("bad find")
-
def test_bad_conflicts(self):
bad_conflicts = ([None, 1.2, "no limit!", {"foo": "bar"}, [2]],)
for bc in bad_conflicts:
@@ -262,11 +252,6 @@ class BasicFindTests(mango.UserDocsTests):
assert sorted(d.keys()) == ["location", "user_id"]
assert sorted(d["location"].keys()) == ["address"]
- def test_r(self):
- for r in [1, 2, 3]:
- docs = self.db.find({"age": {"$gt": 0}}, r=r)
- assert len(docs) == 15
-
def test_empty(self):
docs = self.db.find({})
# 15 users
diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py
index cb4d329..bae3d58 100644
--- a/src/mango/test/05-index-selection-test.py
+++ b/src/mango/test/05-index-selection-test.py
@@ -14,6 +14,8 @@ import mango
import user_docs
import unittest
+import requests
+
class IndexSelectionTests:
def test_basic(self):
@@ -201,8 +203,11 @@ class IndexSelectionTests:
}
},
}
- with self.assertRaises(KeyError):
+ try:
self.db.save_doc(design_doc)
+ assert False, "Should not get here."
+ except requests.exceptions.HTTPError as e:
+ self.assertEqual(e.response.json()['error'], 'invalid_design_doc')
def test_explain_sort_reverse(self):
selector = {"manager": {"$gt": None}}
diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py
index c21ad6c..a7f07b5 100644
--- a/src/mango/test/12-use-correct-index-test.py
+++ b/src/mango/test/12-use-correct-index-test.py
@@ -54,36 +54,41 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs(copy.deepcopy(DOCS))
def test_choose_index_with_one_field_in_index(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie"}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
def test_choose_index_with_two(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_alphabetically(self):
- self.db.create_index(["name"], ddoc="aaa")
- self.db.create_index(["name"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/aaa")
def test_choose_index_most_accurate(self):
- self.db.create_index(["name", "age", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "age", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/bbb")
def test_choose_index_most_accurate_in_memory_selector(self):
- self.db.create_index(["name", "location", "user_id"], ddoc="aaa")
- self.db.create_index(["name", "age", "user_id"], ddoc="bbb")
- self.db.create_index(["name"], ddoc="zzz")
+ self.db.create_index(["name", "location", "user_id"], ddoc="aaa", wait_for_built_index=False)
+ self.db.create_index(["name", "age", "user_id"], ddoc="bbb", wait_for_built_index=False)
+ self.db.create_index(["name"], ddoc="zzz", wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True)
self.assertEqual(explain["index"]["ddoc"], "_design/zzz")
@@ -100,8 +105,9 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
def test_chooses_idxA(self):
DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}]
self.db.save_docs(copy.deepcopy(DOCS2))
- self.db.create_index(["a", "b", "c"])
- self.db.create_index(["a", "d", "e"])
+ self.db.create_index(["a", "b", "c"], wait_for_built_index=False)
+ self.db.create_index(["a", "d", "e"], wait_for_built_index=False)
+ self.db.wait_for_built_indexes()
explain = self.db.find(
{"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True
)
diff --git a/src/mango/test/13-stable-update-test.py b/src/mango/test/13-stable-update-test.py
deleted file mode 100644
index 303f3fa..0000000
--- a/src/mango/test/13-stable-update-test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import copy
-import mango
-
-DOCS1 = [
- {
- "_id": "54af50626de419f5109c962f",
- "user_id": 0,
- "age": 10,
- "name": "Jimi",
- "location": "UK",
- "number": 4,
- },
- {
- "_id": "54af50622071121b25402dc3",
- "user_id": 1,
- "age": 12,
- "name": "Eddie",
- "location": "ZAR",
- "number": 2,
- },
-]
-
-
-class SupportStableAndUpdate(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- # Hack to prevent auto-indexer from foiling update=False test
- # https://github.com/apache/couchdb/issues/2313
- self.db.save_doc(
- {"_id": "_design/foo", "language": "query", "autoupdate": False}
- )
- self.db.create_index(["name"], ddoc="foo")
- self.db.save_docs(copy.deepcopy(DOCS1))
-
- def test_update_updates_view_when_specified(self):
- docs = self.db.find({"name": "Eddie"}, update=False)
- assert len(docs) == 0
- docs = self.db.find({"name": "Eddie"}, update=True)
- assert len(docs) == 1
diff --git a/src/mango/test/13-users-db-find-test.py b/src/mango/test/13-users-db-find-test.py
index 73d15ea..9f9b53a 100644
--- a/src/mango/test/13-users-db-find-test.py
+++ b/src/mango/test/13-users-db-find-test.py
@@ -12,10 +12,15 @@
# the License.
-import mango, requests
+import mango, requests, unittest
+# Re-enable once the _users db is implemented
class UsersDbFindTests(mango.UsersDbTests):
+ @classmethod
+ def setUpClass(klass):
+ raise unittest.SkipTest("Re-enable once the _users db is implemented")
+
def test_simple_find(self):
docs = self.db.find({"name": {"$eq": "demo02"}})
assert len(docs) == 1
diff --git a/src/mango/test/15-execution-stats-test.py b/src/mango/test/15-execution-stats-test.py
index 537a19a..6ccc04b 100644
--- a/src/mango/test/15-execution-stats-test.py
+++ b/src/mango/test/15-execution-stats-test.py
@@ -22,7 +22,6 @@ class ExecutionStatsTests(mango.UserDocsTests):
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -35,12 +34,11 @@ class ExecutionStatsTests(mango.UserDocsTests):
def test_quorum_json_index(self):
resp = self.db.find(
- {"age": {"$lt": 35}}, return_raw=True, r=3, executionStats=True
+ {"age": {"$lt": 35}}, return_raw=True, executionStats=True
)
self.assertEqual(len(resp["docs"]), 3)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_docs_examined"], 0)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 3)
+ self.assertEqual(resp["execution_stats"]["total_docs_examined"], 3)
self.assertEqual(resp["execution_stats"]["results_returned"], 3)
# See https://github.com/apache/couchdb/issues/1732
# Erlang os:timestamp() only has ms accuracy on Windows!
@@ -70,7 +68,6 @@ class ExecutionStatsTests_Text(mango.UserDocsTextTests):
self.assertEqual(len(resp["docs"]), 1)
self.assertEqual(resp["execution_stats"]["total_keys_examined"], 0)
self.assertEqual(resp["execution_stats"]["total_docs_examined"], 1)
- self.assertEqual(resp["execution_stats"]["total_quorum_docs_examined"], 0)
self.assertEqual(resp["execution_stats"]["results_returned"], 1)
self.assertGreater(resp["execution_stats"]["execution_time_ms"], 0)
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
index 21e7afd..5a8fced 100644
--- a/src/mango/test/17-multi-type-value-test.py
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -53,9 +53,9 @@ class MultiValueFieldTests:
class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
def setUp(self):
self.db.recreate()
+ self.db.create_index(["name"], wait_for_built_index=False)
+ self.db.create_index(["age", "name"], wait_for_built_index=True)
self.db.save_docs(copy.deepcopy(DOCS))
- self.db.create_index(["name"])
- self.db.create_index(["age", "name"])
# @unittest.skipUnless(mango.has_text_service(), "requires text service")
diff --git a/src/mango/test/19-find-conflicts.py b/src/mango/test/19-find-conflicts.py
index bf865d6..3bf3c06 100644
--- a/src/mango/test/19-find-conflicts.py
+++ b/src/mango/test/19-find-conflicts.py
@@ -12,11 +12,12 @@
import mango
import copy
+import unittest
-DOC = [{"_id": "doc", "a": 2}]
+DOC = [{"_id": "doc", "a": 2}, {"_id": "doc1", "b": 2}]
CONFLICT = [{"_id": "doc", "_rev": "1-23202479633c2b380f79507a776743d5", "a": 1}]
-
+CONFLICT2 = [{"_id": "doc1", "_rev": "1-23202479633c2b380f79507a776743d5", "b": 1}]
class ChooseCorrectIndexForDocs(mango.DbPerClass):
def setUp(self):
@@ -25,7 +26,7 @@ class ChooseCorrectIndexForDocs(mango.DbPerClass):
self.db.save_docs_with_conflicts(copy.deepcopy(CONFLICT))
def test_retrieve_conflicts(self):
- self.db.create_index(["_conflicts"])
+ self.db.create_index(["_conflicts"], wait_for_built_index=False)
result = self.db.find({"_conflicts": {"$exists": True}}, conflicts=True)
self.assertEqual(
result[0]["_conflicts"][0], "1-23202479633c2b380f79507a776743d5"
diff --git a/src/mango/test/20-no-timeout-test.py b/src/mango/test/20-no-timeout-test.py
deleted file mode 100644
index cffdfc3..0000000
--- a/src/mango/test/20-no-timeout-test.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import mango
-import copy
-import unittest
-
-
-class LongRunningMangoTest(mango.DbPerClass):
- def setUp(self):
- self.db.recreate()
- docs = []
- for i in range(100000):
- docs.append({"_id": str(i), "another": "field"})
- if i % 20000 == 0:
- self.db.save_docs(docs)
- docs = []
-
- # This test should run to completion and not timeout
- def test_query_does_not_time_out(self):
- selector = {"_id": {"$gt": 0}, "another": "wrong"}
- docs = self.db.find(selector)
- self.assertEqual(len(docs), 0)
diff --git a/src/mango/test/22-build-wait-selected-index.py b/src/mango/test/22-build-wait-selected-index.py
new file mode 100644
index 0000000..fd856f4
--- /dev/null
+++ b/src/mango/test/22-build-wait-selected-index.py
@@ -0,0 +1,50 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import copy
+import unittest
+
+
+class BuildAndWaitOnSelectedIndex(mango.DbPerClass):
+ def setUp(self):
+ self.db.recreate()
+ docs = []
+ for i in range(1000):
+ docs.append({"_id": str(i), "val": i})
+ if len(docs) == 250:
+ self.db.save_docs(docs)
+ docs = []
+
+ def test_wait_for_query(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+
+ explain = self.db.find({'val': {"$gt": 990}}, use_index="my-ddoc", explain=True)
+ self.assertEqual(explain["index"]["ddoc"], "_design/my-ddoc")
+
+ docs = self.db.find({'val': {"$gte": 990}}, limit=10)
+
+ self.assertEqual(len(docs), 10)
+
+ def test_dont_wait(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+
+ explain = self.db.find({'val': {"$gt": 990}}, explain=True)
+ self.assertEqual(explain["index"]["name"], "_all_docs")
+
+ docs = self.db.find({'val': {"$gte": 990}})
+ self.assertEqual(len(docs), 10)
+
+ def test_update_false(self):
+ self.db.create_index(["val"], ddoc="my-ddoc", wait_for_built_index=False)
+ docs = self.db.find({'val': {"$gte": 990}}, update=False, use_index="my-ddoc")
+ self.assertEqual(docs, [])
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index db0fab0..e013e0e 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -139,8 +139,9 @@ class Database(object):
ddoc=None,
partial_filter_selector=None,
selector=None,
+ wait_for_built_index=True,
):
- body = {"index": {"fields": fields}, "type": idx_type, "w": 3}
+ body = {"index": {"fields": fields}, "type": idx_type}
if name is not None:
body["name"] = name
if ddoc is not None:
@@ -156,13 +157,22 @@ class Database(object):
assert r.json()["name"] is not None
created = r.json()["result"] == "created"
- if created:
- # wait until the database reports the index as available
- while len(self.get_index(r.json()["id"], r.json()["name"])) < 1:
- delay(t=0.1)
+ if created and wait_for_built_index:
+ # wait until the database reports the index as available and build
+ while True:
+ idx = self.get_index(r.json()["id"], r.json()["name"])[0]
+ if idx["build_status"] == "ready":
+ break
+ delay(t=0.2)
return created
+ def wait_for_built_indexes(self):
+ while True:
+ if all(idx["build_status"] == "ready" for idx in self.list_indexes()):
+ break
+ delay(t=0.2)
+
def create_text_index(
self,
analyzer=None,
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
index 8f0ed2e..d69e6d6 100644
--- a/src/mango/test/user_docs.py
+++ b/src/mango/test/user_docs.py
@@ -61,12 +61,11 @@ def setup_users(db, **kwargs):
def setup(db, index_type="view", **kwargs):
db.recreate()
- db.save_docs(copy.deepcopy(DOCS))
if index_type == "view":
add_view_indexes(db, kwargs)
elif index_type == "text":
add_text_indexes(db, kwargs)
-
+ db.save_docs(copy.deepcopy(DOCS))
def add_view_indexes(db, kwargs):
indexes = [
@@ -90,7 +89,9 @@ def add_view_indexes(db, kwargs):
(["ordered"], "ordered"),
]
for (idx, name) in indexes:
- assert db.create_index(idx, name=name, ddoc=name) is True
+ assert db.create_index(idx, name=name, ddoc=name,
+ wait_for_built_index=False) is True
+ db.wait_for_built_indexes()
def add_text_indexes(db, kwargs):
[couchdb] 03/18: add include_docs option to fold_docs
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a8b930d7d94316b86243034b8faa537cf8486661
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 10 18:30:44 2020 +0200
add include_docs option to fold_docs
---
src/fabric/src/fabric2_db.erl | 27 +++++++++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
diff --git a/src/fabric/src/fabric2_db.erl b/src/fabric/src/fabric2_db.erl
index ca9f037..fb6ae51 100644
--- a/src/fabric/src/fabric2_db.erl
+++ b/src/fabric/src/fabric2_db.erl
@@ -832,11 +832,23 @@ fold_docs(Db, UserFun, UserAcc0, Options) ->
UserAcc2 = fabric2_fdb:fold_range(TxDb, Prefix, fun({K, V}, Acc) ->
{DocId} = erlfdb_tuple:unpack(K, Prefix),
RevId = erlfdb_tuple:unpack(V),
- maybe_stop(UserFun({row, [
+ Row0 = [
{id, DocId},
{key, DocId},
{value, {[{rev, couch_doc:rev_to_str(RevId)}]}}
- ]}, Acc))
+ ],
+
+ DocOpts = couch_util:get_value(doc_opts, Options, []),
+ OpenOpts = [deleted | DocOpts],
+
+ Row1 = case lists:keyfind(include_docs, 1, Options) of
+ {include_docs, true} ->
+ Row0 ++ open_json_doc(Db, DocId, OpenOpts, DocOpts);
+ _ ->
+ Row0
+ end,
+
+ maybe_stop(UserFun({row, Row1}, Acc))
end, UserAcc1, Options),
{ok, maybe_stop(UserFun(complete, UserAcc2))}
@@ -1878,3 +1890,14 @@ stem_revisions(#{} = Db, #doc{} = Doc) ->
true -> Doc#doc{revs = {RevPos, lists:sublist(Revs, RevsLimit)}};
false -> Doc
end.
+
+
+open_json_doc(Db, DocId, OpenOpts, DocOpts) ->
+ case fabric2_db:open_doc(Db, DocId, OpenOpts) of
+ {not_found, missing} ->
+ [];
+ {ok, #doc{deleted = true}} ->
+ [{doc, null}];
+ {ok, #doc{} = Doc} ->
+ [{doc, couch_doc:to_json_obj(Doc, DocOpts)}]
+ end.
[couchdb] 14/18: Add mango indexing
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c483652fee917ae3c6d4065cc6deb28296e40065
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 15:19:44 2020 +0200
Add mango indexing
This uses couch_views_updater to create mango indexes in the doc update
along with the couch_views_indexer to update the indexes in the
background up to the creation versionstamp.
---
rel/overlay/etc/default.ini | 1 +
src/couch_eval/src/couch_eval.erl | 3 +
src/mango/src/mango_eval.erl | 115 ++++++++++++++++++++++++++++++++++++
src/mango/src/mango_idx.erl | 57 ++++++++++++++----
src/mango/src/mango_idx.hrl | 4 +-
src/mango/src/mango_idx_special.erl | 4 +-
src/mango/src/mango_idx_view.erl | 22 +++++--
7 files changed, 187 insertions(+), 19 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index fd0aa77..d2a2c72 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -355,6 +355,7 @@ os_process_limit = 100
; beahvior for executing provided code in design
; documents.
javascript = couch_js
+query = mango_eval
[mango]
; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_eval/src/couch_eval.erl b/src/couch_eval/src/couch_eval.erl
index 23ca263..3541a5b 100644
--- a/src/couch_eval/src/couch_eval.erl
+++ b/src/couch_eval/src/couch_eval.erl
@@ -75,6 +75,9 @@ acquire_map_context(DbName, DDocId, Language, Sig, Lib, MapFuns) ->
-spec release_map_context(context()) -> ok | {error, any()}.
+release_map_context(nil) ->
+ ok;
+
release_map_context({ApiMod, Ctx}) ->
ApiMod:release_map_context(Ctx).
diff --git a/src/mango/src/mango_eval.erl b/src/mango/src/mango_eval.erl
new file mode 100644
index 0000000..59d784b
--- /dev/null
+++ b/src/mango/src/mango_eval.erl
@@ -0,0 +1,115 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(mango_eval).
+-behavior(couch_eval).
+
+
+-export([
+ acquire_map_context/1,
+ release_map_context/1,
+ map_docs/2
+]).
+
+
+-export([
+ index_doc/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_idx.hrl").
+
+
+acquire_map_context(Opts) ->
+ #{
+ db_name := DbName,
+ ddoc_id := DDocId,
+ map_funs := MapFuns
+ } = Opts,
+ Indexes = lists:map(fun (Def) ->
+ #idx{
+ type = <<"json">>,
+ dbname = DbName,
+ ddoc = DDocId,
+ def = Def
+ }
+ end, MapFuns),
+ {ok, Indexes}.
+
+
+release_map_context(_) ->
+ ok.
+
+
+map_docs(Indexes, Docs) ->
+ {ok, lists:map(fun(Doc) ->
+ Json = couch_doc:to_json_obj(Doc, []),
+ Results = index_doc(Indexes, Json),
+ {Doc#doc.id, Results}
+ end, Docs)}.
+
+
+index_doc(Indexes, Doc) ->
+ lists:map(fun(Idx) ->
+ {IdxDef} = mango_idx:def(Idx),
+ Results = get_index_entries(IdxDef, Doc),
+ case lists:member(not_found, Results) of
+ true ->
+ [];
+ false ->
+ [{Results, null}]
+ end
+ end, Indexes).
+
+
+get_index_entries(IdxDef, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxDef),
+ Selector = get_index_partial_filter_selector(IdxDef),
+ case should_index(Selector, Doc) of
+ false ->
+ [not_found];
+ true ->
+ get_index_values(Fields, Doc)
+ end.
+
+
+get_index_values(Fields, Doc) ->
+ lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Value -> Value
+ end
+ end, Fields).
+
+
+get_index_partial_filter_selector(IdxDef) ->
+ case couch_util:get_value(<<"partial_filter_selector">>, IdxDef, {[]}) of
+ {[]} ->
+ % this is to support legacy text indexes that had the
+ % partial_filter_selector set as selector
+ couch_util:get_value(<<"selector">>, IdxDef, {[]});
+ Else ->
+ Else
+ end.
+
+
+should_index(Selector, Doc) ->
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index a26a685..ba9f68f 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -50,11 +50,35 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
list(Db) ->
- {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
- Indexes.
+ DDocs = couch_views_ddoc:get_mango_list(Db),
+ DbName = fabric2_db:name(Db),
+ Indexes = lists:foldl(fun(DDoc, Acc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+
+ case proplists:get_value(<<"language">>, Props) == <<"query">> of
+ true ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+ IsInteractive = couch_views_ddoc:is_interactive(DDoc),
+ BuildState = couch_views_fdb:get_build_status(Db, Mrst),
+
+ Idxs = lists:map(fun(Idx) ->
+ Idx#idx{
+ build_status = BuildState,
+ interactive = IsInteractive
+ }
+ end, from_ddoc(Db, DDoc)),
+ Acc ++ Idxs;
+ false ->
+ Acc
+ end
+
+ end, [], DDocs),
+ Indexes ++ special(Db).
get_usable_indexes(Db, Selector, Opts) ->
@@ -62,13 +86,14 @@ get_usable_indexes(Db, Selector, Opts) ->
GlobalIndexes = mango_cursor:remove_indexes_with_partial_filter_selector(
ExistingIndexes
),
+ BuiltIndexes = mango_cursor:remove_unbuilt_indexes(GlobalIndexes),
UserSpecifiedIndex = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
- UsableIndexes = lists:usort(GlobalIndexes ++ UserSpecifiedIndex),
+ UsableIndexes0 = lists:usort(BuiltIndexes ++ UserSpecifiedIndex),
SortFields = get_sort_fields(Opts),
UsableFilter = fun(I) -> is_usable(I, Selector, SortFields) end,
- case lists:filter(UsableFilter, UsableIndexes) of
+ case lists:filter(UsableFilter, UsableIndexes0) of
[] ->
mango_sort_error(Db, Opts);
UsableIndexes ->
@@ -162,16 +187,17 @@ delete(Filt, Db, Indexes, DelOpts) ->
end.
-from_ddoc(Db, {Props}) ->
+from_ddoc(Db, #doc{id = DDocId} = DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
DbName = db_to_name(Db),
- DDoc = proplists:get_value(<<"_id">>, Props),
+ DDocId = proplists:get_value(<<"_id">>, Props),
case proplists:get_value(<<"language">>, Props) of
<<"query">> -> ok;
_ ->
?MANGO_ERROR(invalid_query_ddoc_language)
end,
- IdxMods = case clouseau_rpc:connected() of
+ IdxMods = case is_text_service_available() of
true ->
[mango_idx_view, mango_idx_text];
false ->
@@ -181,7 +207,7 @@ from_ddoc(Db, {Props}) ->
lists:map(fun(Idx) ->
Idx#idx{
dbname = DbName,
- ddoc = DDoc
+ ddoc = DDocId
}
end, Idxs).
@@ -192,7 +218,8 @@ special(Db) ->
name = <<"_all_docs">>,
type = <<"special">>,
def = all_docs,
- opts = []
+ opts = [],
+ build_status = ?INDEX_READY
},
% Add one for _update_seq
[AllDocs].
@@ -278,7 +305,7 @@ db_to_name(Name) when is_binary(Name) ->
db_to_name(Name) when is_list(Name) ->
iolist_to_binary(Name);
db_to_name(Db) ->
- couch_db:name(Db).
+ fabric2_db:name(Db).
get_idx_def(Opts) ->
@@ -293,7 +320,7 @@ get_idx_def(Opts) ->
get_idx_type(Opts) ->
case proplists:get_value(type, Opts) of
<<"json">> -> <<"json">>;
- <<"text">> -> case clouseau_rpc:connected() of
+ <<"text">> -> case is_text_service_available() of
true ->
<<"text">>;
false ->
@@ -306,6 +333,11 @@ get_idx_type(Opts) ->
end.
+is_text_service_available() ->
+ erlang:function_exported(clouseau_rpc, connected, 0) andalso
+ clouseau_rpc:connected().
+
+
get_idx_ddoc(Idx, Opts) ->
case proplists:get_value(ddoc, Opts) of
<<"_design/", _Rest/binary>> = Name ->
@@ -377,7 +409,8 @@ index(SelectorName, Selector) ->
<<"Selected">>,<<"json">>,
{[{<<"fields">>,{[{<<"location">>,<<"asc">>}]}},
{SelectorName,{Selector}}]},
- [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}]
+ [{<<"def">>,{[{<<"fields">>,[<<"location">>]}]}}],
+ <<"ready">>
}.
get_partial_filter_all_docs_test() ->
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
index 712031b..68e5aaa 100644
--- a/src/mango/src/mango_idx.hrl
+++ b/src/mango/src/mango_idx.hrl
@@ -16,5 +16,7 @@
name,
type,
def,
- opts
+ opts,
+ build_status,
+ interactive
}).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
index ac6efc7..3548372 100644
--- a/src/mango/src/mango_idx_special.erl
+++ b/src/mango/src/mango_idx_special.erl
@@ -28,6 +28,7 @@
-include_lib("couch/include/couch_db.hrl").
-include("mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
validate(_) ->
@@ -55,7 +56,8 @@ to_json(#idx{def=all_docs}) ->
{<<"fields">>, [{[
{<<"_id">>, <<"asc">>}
]}]}
- ]}}
+ ]}},
+ {build_status, ?INDEX_READY}
]}.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 2d784b6..84be418 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -54,7 +54,16 @@ add(#doc{body={Props0}}=DDoc, Idx) ->
NewView = make_view(Idx),
Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
- {ok, DDoc#doc{body={Props1}}}.
+
+ {Opts0} = proplists:get_value(<<"options">>, Props1, {[]}),
+ Opts1 = case lists:keymember(<<"interactive">>, 1, Opts0) of
+ true -> Opts0;
+ false -> Opts0 ++ [{<<"interactive">>, true}]
+ end,
+ Props2 = lists:keystore(<<"options">>, 1, Props1, {<<"options">>, {Opts1}}),
+
+ Props3 = [{<<"autoupdate">>, false}],
+ {ok, DDoc#doc{body={Props2 ++ Props3}}}.
remove(#doc{body={Props0}}=DDoc, Idx) ->
@@ -68,13 +77,15 @@ remove(#doc{body={Props0}}=DDoc, Idx) ->
if Views2 /= Views1 -> ok; true ->
?MANGO_ERROR({index_not_found, Idx#idx.name})
end,
- Props1 = case Views2 of
+ Props3 = case Views2 of
[] ->
- lists:keydelete(<<"views">>, 1, Props0);
+ Props1 = lists:keydelete(<<"views">>, 1, Props0),
+ Props2 = lists:keydelete(<<"options">>, 1, Props1),
+ lists:keydelete(<<"autoupdate">>, 1, Props2);
_ ->
lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}})
end,
- {ok, DDoc#doc{body={Props1}}}.
+ {ok, DDoc#doc{body={Props3}}}.
from_ddoc({Props}) ->
@@ -104,7 +115,8 @@ to_json(Idx) ->
{ddoc, Idx#idx.ddoc},
{name, Idx#idx.name},
{type, Idx#idx.type},
- {def, {def_to_json(Idx#idx.def)}}
+ {def, {def_to_json(Idx#idx.def)}},
+ {build_status, Idx#idx.build_status}
]}.
[couchdb] 11/18: remove unneeded r/w parameter
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit bd1667d3bf1b5328ac7b0c4a4b50d1b8b91a9ec2
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Mon Mar 23 15:35:22 2020 +0200
remove unneeded r/w parameter
---
src/mango/src/mango_crud.erl | 17 +++--------------
src/mango/src/mango_cursor_view.erl | 12 ------------
src/mango/src/mango_httpd.erl | 28 +++-------------------------
src/mango/src/mango_idx.erl | 2 --
src/mango/src/mango_opts.erl | 12 ------------
src/mango/test/mango.py | 2 --
6 files changed, 6 insertions(+), 67 deletions(-)
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
index 41a4d14..42717ff 100644
--- a/src/mango/src/mango_crud.erl
+++ b/src/mango/src/mango_crud.erl
@@ -35,8 +35,7 @@ insert(Db, {_}=Doc, Opts) ->
insert(Db, [Doc], Opts);
insert(Db, Docs, Opts0) when is_list(Docs) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(w, Opts1),
- case fabric:update_docs(Db, Docs, Opts2) of
+ case fabric:update_docs(Db, Docs, Opts1) of
{ok, Results0} ->
{ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
{accepted, Results0} ->
@@ -48,8 +47,7 @@ insert(Db, Docs, Opts0) when is_list(Docs) ->
find(Db, Selector, Callback, UserAcc, Opts0) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
mango_cursor:execute(Cursor, Callback, UserAcc).
@@ -101,8 +99,7 @@ delete(Db, Selector, Options) ->
explain(Db, Selector, Opts0) ->
Opts1 = maybe_add_user_ctx(Db, Opts0),
- Opts2 = maybe_int_to_str(r, Opts1),
- {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts1),
mango_cursor:explain(Cursor).
@@ -115,14 +112,6 @@ maybe_add_user_ctx(Db, Opts) ->
end.
-maybe_int_to_str(_Key, []) ->
- [];
-maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) ->
- [{Key, integer_to_list(Val)} | maybe_int_to_str(Key, Rest)];
-maybe_int_to_str(Key, [KV | Rest]) ->
- [KV | maybe_int_to_str(Key, Rest)].
-
-
result_to_json(#doc{id=Id}, Result) ->
result_to_json(Id, Result);
result_to_json({Props}, Result) ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index bced842..5187c10 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -353,18 +353,6 @@ ddocid(Idx) ->
apply_opts([], Args) ->
Args;
-apply_opts([{r, RStr} | Rest], Args) ->
- IncludeDocs = case list_to_integer(RStr) of
- 1 ->
- true;
- R when R > 1 ->
- % We don't load the doc in the view query because
- % we have to do a quorum read in the coordinator
- % so there's no point.
- false
- end,
- NewArgs = Args#mrargs{include_docs = IncludeDocs},
- apply_opts(Rest, NewArgs);
apply_opts([{conflicts, true} | Rest], Args) ->
NewArgs = Args#mrargs{conflicts = true},
apply_opts(Rest, NewArgs);
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 946d7e4..1054c74 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -94,8 +94,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, DDoc} ->
<<"exists">>;
{ok, NewDDoc} ->
- CreateOpts = get_idx_w_opts(Opts),
- case mango_crud:insert(Db, NewDDoc, CreateOpts) of
+ case mango_crud:insert(Db, NewDDoc, Opts) of
{ok, [{RespProps}]} ->
case lists:keyfind(error, 1, RespProps) of
{error, Reason} ->
@@ -121,12 +120,11 @@ handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
{ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
Idxs = mango_idx:list(Db),
DDocs = get_bulk_delete_ddocs(Opts),
- DelOpts = get_idx_w_opts(Opts),
{Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
DDocId = convert_to_design_id(DDocId0),
Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
Id = {<<"id">>, DDocId},
- case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ case mango_idx:delete(Filt, Db, Idxs, Opts) of
{ok, true} ->
{[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
{error, Error} ->
@@ -148,14 +146,13 @@ handle_index_req(#httpd{method='DELETE',
path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
Idxs = mango_idx:list(Db),
DDocId = convert_to_design_id(DDocId0),
- DelOpts = get_idx_del_opts(Req),
Filt = fun(Idx) ->
IsDDoc = mango_idx:ddoc(Idx) == DDocId,
IsType = mango_idx:type(Idx) == Type,
IsName = mango_idx:name(Idx) == Name,
IsDDoc andalso IsType andalso IsName
end,
- case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ case mango_idx:delete(Filt, Db, Idxs, []) of
{ok, true} ->
chttpd:send_json(Req, {[{ok, true}]});
{error, not_found} ->
@@ -203,15 +200,6 @@ set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
NewDb.
-get_idx_w_opts(Opts) ->
- case lists:keyfind(w, 1, Opts) of
- {w, N} when is_integer(N), N > 0 ->
- [{w, integer_to_list(N)}];
- _ ->
- [{w, "2"}]
- end.
-
-
get_bulk_delete_ddocs(Opts) ->
case lists:keyfind(docids, 1, Opts) of
{docids, DDocs} when is_list(DDocs) ->
@@ -221,16 +209,6 @@ get_bulk_delete_ddocs(Opts) ->
end.
-get_idx_del_opts(Req) ->
- try
- WStr = chttpd:qs_value(Req, "w", "2"),
- _ = list_to_integer(WStr),
- [{w, WStr}]
- catch _:_ ->
- [{w, "2"}]
- end.
-
-
convert_to_design_id(DDocId) ->
case DDocId of
<<"_design/", _/binary>> -> DDocId;
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 0f79bdb..a26a685 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -344,8 +344,6 @@ filter_opts([{name, _} | Rest]) ->
filter_opts(Rest);
filter_opts([{type, _} | Rest]) ->
filter_opts(Rest);
-filter_opts([{w, _} | Rest]) ->
- filter_opts(Rest);
filter_opts([Opt | Rest]) ->
[Opt | filter_opts(Rest)].
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
index 7bae9c9..e357676 100644
--- a/src/mango/src/mango_opts.erl
+++ b/src/mango/src/mango_opts.erl
@@ -64,12 +64,6 @@ validate_idx_create({Props}) ->
{optional, true},
{default, auto_name},
{validator, fun validate_idx_name/1}
- ]},
- {<<"w">>, [
- {tag, w},
- {optional, true},
- {default, 2},
- {validator, fun is_pos_integer/1}
]}
],
validate(Props, Opts).
@@ -117,12 +111,6 @@ validate_find({Props}) ->
{default, []},
{validator, fun validate_fields/1}
]},
- {<<"r">>, [
- {tag, r},
- {optional, true},
- {default, 1},
- {validator, fun mango_opts:is_pos_integer/1}
- ]},
{<<"conflicts">>, [
{tag, conflicts},
{optional, true},
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 03cb85f..638de47 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -244,7 +244,6 @@ class Database(object):
skip=0,
sort=None,
fields=None,
- r=1,
conflicts=False,
use_index=None,
explain=False,
@@ -258,7 +257,6 @@ class Database(object):
"use_index": use_index,
"limit": limit,
"skip": skip,
- "r": r,
"conflicts": conflicts,
}
if sort is not None:
[couchdb] 07/18: Add couch_views_updater interactive indexer
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b856501628359fba0a08087b4ce75a0606cae7a9
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 25 15:38:53 2020 +0200
Add couch_views_updater interactive indexer
This adds the ability for couch_views to index an index in the docs
update transaction. This only happens if a design doc has the
field <<"interactive">> = true.
---
rel/apps/couch_epi.config | 1 +
src/couch_views/src/couch_views.app.src | 1 +
src/couch_views/src/couch_views.erl | 16 +-
src/couch_views/src/couch_views_ddoc.erl | 42 ++++
src/couch_views/src/couch_views_epi.erl | 58 ++++++
.../couch_views/src/couch_views_fabric2_plugin.erl | 23 ++-
src/couch_views/src/couch_views_sup.erl | 2 +-
src/couch_views/src/couch_views_updater.erl | 101 +++++++++
src/couch_views/test/couch_views_updater_test.erl | 230 +++++++++++++++++++++
9 files changed, 458 insertions(+), 16 deletions(-)
diff --git a/rel/apps/couch_epi.config b/rel/apps/couch_epi.config
index 0f3d2da..d371163 100644
--- a/rel/apps/couch_epi.config
+++ b/rel/apps/couch_epi.config
@@ -15,6 +15,7 @@
fabric2_epi,
chttpd_epi,
couch_index_epi,
+ couch_views_epi,
dreyfus_epi,
global_changes_epi,
mango_epi,
diff --git a/src/couch_views/src/couch_views.app.src b/src/couch_views/src/couch_views.app.src
index b704c97..cb8285a 100644
--- a/src/couch_views/src/couch_views.app.src
+++ b/src/couch_views/src/couch_views.app.src
@@ -22,6 +22,7 @@
kernel,
stdlib,
erlfdb,
+ couch_epi,
couch_log,
config,
couch_stats,
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 2268052..2acba00 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -37,6 +37,7 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
end,
DbName = fabric2_db:name(Db),
+ IsInteractive = couch_views_ddoc:is_interactive(DDoc),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
#mrst{
@@ -54,7 +55,7 @@ query(Db, DDoc, ViewName, Callback, Acc0, Args0) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
- ok = maybe_update_view(TxDb, Mrst, Args3),
+ ok = maybe_update_view(TxDb, Mrst, IsInteractive, Args3),
read_view(TxDb, Mrst, ViewName, Callback, Acc0, Args3)
end)
catch throw:{build_view, WaitSeq} ->
@@ -127,13 +128,20 @@ read_view(Db, Mrst, ViewName, Callback, Acc0, Args) ->
end).
-maybe_update_view(_Db, _Mrst, #mrargs{update = false}) ->
+maybe_update_view(_Db, _Mrst, _, #mrargs{update = false}) ->
ok;
-maybe_update_view(_Db, _Mrst, #mrargs{update = lazy}) ->
+maybe_update_view(_Db, _Mrst, _, #mrargs{update = lazy}) ->
ok;
-maybe_update_view(TxDb, Mrst, _Args) ->
+maybe_update_view(TxDb, Mrst, true, _Args) ->
+ BuildState = couch_views_fdb:get_build_status(TxDb, Mrst),
+ if BuildState == ?INDEX_READY -> ok; true ->
+ VS = couch_views_fdb:get_creation_vs(TxDb, Mrst),
+ throw({build_view, fabric2_fdb:vs_to_seq(VS)})
+ end;
+
+maybe_update_view(TxDb, Mrst, false, _Args) ->
DbSeq = fabric2_db:get_update_seq(TxDb),
ViewSeq = couch_views_fdb:get_update_seq(TxDb, Mrst),
case DbSeq == ViewSeq of
diff --git a/src/couch_views/src/couch_views_ddoc.erl b/src/couch_views/src/couch_views_ddoc.erl
new file mode 100644
index 0000000..fae4a34
--- /dev/null
+++ b/src/couch_views/src/couch_views_ddoc.erl
@@ -0,0 +1,42 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_views_ddoc).
+
+
+-export([
+ get_interactive_list/1,
+ get_mango_list/1,
+ is_interactive/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+% TODO: build a ddoc cache that checks the md_version
+get_interactive_list(Db) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ lists:filter(fun is_interactive/1, DDocs).
+
+
+get_mango_list(Db) ->
+ DDocs = fabric2_db:get_design_docs(Db),
+ lists:filter(fun (DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+ fabric2_util:get_value(<<"language">>, Props) == <<"query">>
+ end, DDocs).
+
+
+is_interactive(#doc{} = DDoc) ->
+ {Props} = couch_doc:to_json_obj(DDoc, []),
+ {Opts} = fabric2_util:get_value(<<"options">>, Props, {[]}),
+ fabric2_util:get_value(<<"interactive">>, Opts, false).
diff --git a/src/couch_views/src/couch_views_epi.erl b/src/couch_views/src/couch_views_epi.erl
new file mode 100644
index 0000000..6d39d9a
--- /dev/null
+++ b/src/couch_views/src/couch_views_epi.erl
@@ -0,0 +1,58 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+
+-module(couch_views_epi).
+
+
+-behaviour(couch_epi_plugin).
+
+
+-export([
+ app/0,
+ providers/0,
+ services/0,
+ data_subscriptions/0,
+ data_providers/0,
+ processes/0,
+ notify/3
+]).
+
+
+app() ->
+ couch_views.
+
+
+providers() ->
+ [
+ {fabric2_db, couch_views_fabric2_plugin}
+ ].
+
+
+services() ->
+ [].
+
+
+data_subscriptions() ->
+ [].
+
+
+data_providers() ->
+ [].
+
+
+processes() ->
+ [].
+
+
+notify(_Key, _Old, _New) ->
+ ok.
diff --git a/rel/apps/couch_epi.config b/src/couch_views/src/couch_views_fabric2_plugin.erl
similarity index 66%
copy from rel/apps/couch_epi.config
copy to src/couch_views/src/couch_views_fabric2_plugin.erl
index 0f3d2da..cae0e1f 100644
--- a/rel/apps/couch_epi.config
+++ b/src/couch_views/src/couch_views_fabric2_plugin.erl
@@ -10,14 +10,15 @@
% License for the specific language governing permissions and limitations under
% the License.
-{plugins, [
- couch_db_epi,
- fabric2_epi,
- chttpd_epi,
- couch_index_epi,
- dreyfus_epi,
- global_changes_epi,
- mango_epi,
- mem3_epi,
- setup_epi
-]}.
+
+-module(couch_views_fabric2_plugin).
+
+
+-export([
+ after_doc_write/6
+]).
+
+
+after_doc_write(Db, Doc, NewWinner, OldWinner, NewRevId, Seq)->
+ couch_views_updater:index(Db, Doc, NewWinner, OldWinner, NewRevId, Seq),
+ [Db, Doc, NewWinner, OldWinner, NewRevId, Seq].
diff --git a/src/couch_views/src/couch_views_sup.erl b/src/couch_views/src/couch_views_sup.erl
index 2a40f0a..9453189 100644
--- a/src/couch_views/src/couch_views_sup.erl
+++ b/src/couch_views/src/couch_views_sup.erl
@@ -42,7 +42,7 @@ init(normal) ->
id => couch_views_server,
start => {couch_views_server, start_link, []}
}
- ],
+ ] ++ couch_epi:register_service(couch_views_epi, []),
{ok, {flags(), Children}};
init(builds_disabled) ->
diff --git a/src/couch_views/src/couch_views_updater.erl b/src/couch_views/src/couch_views_updater.erl
new file mode 100644
index 0000000..f405123
--- /dev/null
+++ b/src/couch_views/src/couch_views_updater.erl
@@ -0,0 +1,101 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+-module(couch_views_updater).
+
+-export([
+ index/6
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+
+% If the doc revision doesn't not match the NewRevId passed here we can ignore
+% the document since it is then a conflict document and it doesn't need
+% to be indexed.
+index(Db, #doc{id = Id, revs = Revs} = Doc, _NewWinner, _OldWinner, NewRevId,
+ Seq) ->
+ try
+ {Depth, [FirstRev | _]} = Revs,
+ DocRev = {Depth, FirstRev},
+ if DocRev /= NewRevId -> ok; true ->
+ index_int(Db, Doc, Seq)
+ end
+ catch
+ Error:Reason ->
+ DbName = fabric2_db:name(Db),
+ couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+ [DbName, Id, Error, Reason])
+ end.
+
+
+% Check if design doc is an interactive index and kick off background worker
+% to build the new index up to the creation_vs
+index_int(Db, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>,
+ deleted = false} = DDoc, Seq) ->
+ DbName = fabric2_db:name(Db),
+
+ case couch_views_ddoc:is_interactive(DDoc) of
+ true ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+ case couch_views_fdb:get_creation_vs(Db, Mrst) of
+ not_found ->
+ couch_views_fdb:new_interactive_index(Db, Mrst, Seq),
+ {ok, _} = couch_views_jobs:build_view_async(Db, Mrst);
+ _ ->
+ ok
+ end;
+ false ->
+ ok
+ end,
+ write_doc(Db, DDoc);
+
+
+index_int(Db, #doc{} = Doc, _Seq) ->
+ write_doc(Db, Doc).
+
+
+write_doc(Db, #doc{deleted = Deleted} = Doc) ->
+ DbName = fabric2_db:name(Db),
+ DDocs = couch_views_ddoc:get_interactive_list(Db),
+
+ Result0 = [#{
+ id => Doc#doc.id,
+ results => [],
+ deleted => Deleted,
+ doc => Doc
+ }],
+
+ %% Interactive updates do not update the views update_seq
+ State = #{
+ last_seq => false
+ },
+
+ lists:foreach(fun(DDoc) ->
+ {ok, Mrst} = couch_mrview_util:ddoc_to_mrst(DbName, DDoc),
+
+ case should_index_doc(Doc, Mrst) of
+ true ->
+ {Mrst1, Result1} = couch_views_indexer:map_docs(Mrst, Result0),
+ couch_views_indexer:write_docs(Db, Mrst1, Result1, State),
+ couch_eval:release_map_context(Mrst1#mrst.qserver);
+ false ->
+ ok
+ end
+ end, DDocs).
+
+
+should_index_doc(<<?DESIGN_DOC_PREFIX, _/binary>>, Mrst) ->
+ lists:keymember(<<"include_design">>, 1, Mrst#mrst.design_opts);
+
+should_index_doc(_, _) ->
+ true.
diff --git a/src/couch_views/test/couch_views_updater_test.erl b/src/couch_views/test/couch_views_updater_test.erl
new file mode 100644
index 0000000..e456225
--- /dev/null
+++ b/src/couch_views/test/couch_views_updater_test.erl
@@ -0,0 +1,230 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_updater_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+-include_lib("mango/src/mango_idx.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
+
+
+indexer_test_() ->
+ {
+ "Test indexing",
+ {
+ setup,
+ fun setup/0,
+ fun cleanup/1,
+ {
+ foreach,
+ fun foreach_setup/0,
+ fun foreach_teardown/1,
+ [
+ ?TDEF_FE(index_docs),
+ ?TDEF_FE(update_doc),
+ ?TDEF_FE(delete_doc),
+ ?TDEF_FE(includes_design_docs)
+ ]
+ }
+ }
+ }.
+
+
+setup() ->
+ Ctx = test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views,
+ mango
+ ]),
+ Ctx.
+
+
+cleanup(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+ {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+ DDoc = create_idx_ddoc(),
+ fabric2_db:update_docs(Db, [DDoc]),
+ % make sure the index is built for the first time so the background
+ % indexer doesn't build the index
+ wait_while_ddoc_builds(Db),
+
+ Docs = make_docs(3),
+ fabric2_db:update_docs(Db, Docs),
+ {Db, DDoc}.
+
+
+foreach_teardown({Db, _}) ->
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"2">>}, {value, 2}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+update_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}],
+ [{id, <<"2">>}, {value, 4}]
+ ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+ {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+ JsonDoc = couch_doc:to_json_obj(Doc, []),
+ JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+ Doc2 = couch_doc:from_json_obj(JsonDoc2),
+ fabric2_db:update_doc(Db, Doc2),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"1">>}, {value, 1}],
+ [{id, <<"3">>}, {value, 3}]
+ ], Docs).
+
+
+includes_design_docs({Db, _}) ->
+ DDoc = create_idx_include_ddocs(),
+ fabric2_db:update_docs(Db, [DDoc]),
+
+ IndexDDoc0 = create_idx_ddoc(),
+ IndexDDoc = IndexDDoc0#doc{
+ id = <<"_design/to_be_indexed">>
+ },
+
+ fabric2_db:update_docs(Db, [IndexDDoc]),
+
+ Docs = run_query(Db, DDoc),
+ ?assertEqual([
+ [{id, <<"_design/ddoc_that_indexes_ddocs">>}, {value, 1}],
+ [{id, <<"_design/to_be_indexed">>}, {value, 1}]
+ ], Docs).
+
+
+run_query(Db, DDoc) ->
+ Args = #mrargs{
+ view_type = map,
+ reduce = false,
+ include_docs = true,
+ update = false
+ },
+ CB = fun query_cb/2,
+ {ok, Acc} = couch_views:query(Db, DDoc, <<"idx_01">>, CB, [], Args),
+ lists:map(fun ({Props}) ->
+ [
+ {id, couch_util:get_value(<<"_id">>, Props)},
+ {value, couch_util:get_value(<<"value">>, Props, 1)}
+ ]
+
+ end, Acc).
+
+
+create_idx_ddoc() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc1">>},
+ {<<"language">>, <<"query">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, {[
+ {<<"fields">>, {[{<<"value">>, <<"asc">>}]}}
+ ]}},
+ {<<"reduce">>, <<"_count">>},
+ {<<"options">>, {[
+ {<<"def">>,
+ {[{<<"fields">>,
+ {[{<<"value">>, <<"asc">>}]}}]}}
+ ]}}
+ ]}}
+ ]}
+ },
+ {<<"autoupdate">>, false},
+ {<<"options">>, {[{<<"interactive">>, true}]}}
+ ]}).
+
+
+create_idx_include_ddocs() ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, <<"_design/ddoc_that_indexes_ddocs">>},
+ {<<"language">>, <<"javascript">>},
+ {<<"views">>, {[
+ {<<"idx_01">>, {[
+ {<<"map">>, <<
+ "function(doc) {"
+ "if (doc.language) {"
+ "emit(doc.language, 1);"
+ "}"
+ "}">>}
+ ]}}
+ ]}},
+ {<<"autoupdate">>, false},
+ {<<"options">>, {[
+ {<<"include_design">>, true},
+ {<<"interactive">>, true}
+ ]}}
+ ]}).
+
+
+wait_while_ddoc_builds(Db) ->
+ Fun = fun () ->
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ Ready = lists:filter(fun (Idx) ->
+ Idx#idx.build_status == ?INDEX_READY
+ end, mango_idx:list(TxDb)),
+
+ if length(Ready) > 1 -> ok; true ->
+ wait
+ end
+ end)
+ end,
+ test_util:wait(Fun).
+
+
+
+make_docs(Count) ->
+ [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+ couch_doc:from_json_obj({[
+ {<<"_id">>, list_to_binary(integer_to_list(Id))},
+ {<<"value">>, Id}
+ ]}).
+
+
+query_cb({row, Props}, Acc) ->
+ Doc = couch_util:get_value(doc, Props),
+ {ok, Acc ++ [Doc]};
+
+query_cb(_, Acc) ->
+ {ok, Acc}.
+
[couchdb] 06/18: All couch_view queries to run across transactions
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 34ca5e40cfac4bb6fbdc8d9084602781c17d87de
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 25 15:10:00 2020 +0200
All couch_view queries to run across transactions
---
src/couch_views/src/couch_views_reader.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/couch_views/src/couch_views_reader.erl b/src/couch_views/src/couch_views_reader.erl
index 76dbed1..ce7f163 100644
--- a/src/couch_views/src/couch_views_reader.erl
+++ b/src/couch_views/src/couch_views_reader.erl
@@ -184,7 +184,8 @@ mrargs_to_fdb_options(Args) ->
[
{dir, Direction},
{limit, Limit + Skip},
- {streaming_mode, want_all}
+ {streaming_mode, want_all},
+ {restart_tx, true}
] ++ StartKeyOpts ++ EndKeyOpts.
[couchdb] 18/18: remove defer and load_ddocs from mango_utils
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 5652e72e43406b7e4b743ee3fe7e2570aec77e95
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Tue Mar 31 15:26:03 2020 +0200
remove defer and load_ddocs from mango_utils
Clean up unused mango_utils functions.
---
src/mango/src/mango_idx.erl | 16 ---------------
src/mango/src/mango_util.erl | 47 --------------------------------------------
2 files changed, 63 deletions(-)
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index 99a3588..9907cc7 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -19,7 +19,6 @@
-export([
list/1,
- recover/1,
new/2,
validate_new/2,
@@ -105,21 +104,6 @@ mango_sort_error(_Db, _Opts) ->
?MANGO_ERROR({no_usable_index, missing_sort_index}).
-recover(Db) ->
- {ok, DDocs0} = mango_util:open_ddocs(Db),
- Pred = fun({Props}) ->
- case proplists:get_value(<<"language">>, Props) of
- <<"query">> -> true;
- _ -> false
- end
- end,
- DDocs = lists:filter(Pred, DDocs0),
- Special = special(Db),
- {ok, Special ++ lists:flatmap(fun(Doc) ->
- from_ddoc(Db, Doc)
- end, DDocs)}.
-
-
get_sort_fields(Opts) ->
case lists:keyfind(sort, 1, Opts) of
{sort, Sort} ->
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
index 18a6439..d649f95 100644
--- a/src/mango/src/mango_util.erl
+++ b/src/mango/src/mango_util.erl
@@ -15,13 +15,9 @@
-export([
open_doc/2,
- open_ddocs/1,
load_ddoc/2,
load_ddoc/3,
- defer/3,
- do_defer/3,
-
assert_ejson/1,
to_lower/1,
@@ -88,15 +84,6 @@ open_doc(Db, DocId, Options) ->
fabric2_db:open_doc(Db, DocId, Options).
-open_ddocs(Db) ->
- case mango_util:defer(fabric, design_docs, [Db]) of
- {ok, Docs} ->
- {ok, Docs};
- _ ->
- ?MANGO_ERROR(error_loading_ddocs)
- end.
-
-
load_ddoc(Db, DDocId) ->
load_ddoc(Db, DDocId, [deleted, ejson_body]).
@@ -112,40 +99,6 @@ load_ddoc(Db, DDocId, DbOpts) ->
end.
-defer(Mod, Fun, Args) ->
- {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]),
- receive
- {'DOWN', Ref, process, Pid, {mango_defer_ok, Value}} ->
- Value;
- {'DOWN', Ref, process, Pid, {mango_defer_throw, Value}} ->
- erlang:throw(Value);
- {'DOWN', Ref, process, Pid, {mango_defer_error, Value}} ->
- erlang:error(Value);
- {'DOWN', Ref, process, Pid, {mango_defer_exit, Value}} ->
- erlang:exit(Value)
- end.
-
-
-do_defer(Mod, Fun, Args) ->
- try erlang:apply(Mod, Fun, Args) of
- Resp ->
- erlang:exit({mango_defer_ok, Resp})
- catch
- throw:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{throw, Error}, Stack]),
- erlang:exit({mango_defer_throw, Error});
- error:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{error, Error}, Stack]),
- erlang:exit({mango_defer_error, Error});
- exit:Error ->
- Stack = erlang:get_stacktrace(),
- couch_log:error("Defered error: ~w~n ~p", [{exit, Error}, Stack]),
- erlang:exit({mango_defer_exit, Error})
- end.
-
-
assert_ejson({Props}) ->
assert_ejson_obj(Props);
assert_ejson(Vals) when is_list(Vals) ->
[couchdb] 01/18: fix all_docs call to return row
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
garren pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c22d0075df034d7b0bf501ae28a85713233c8f1e
Author: Garren Smith <ga...@gmail.com>
AuthorDate: Wed Mar 18 09:39:50 2020 +0200
fix all_docs call to return row
---
src/chttpd/src/chttpd_db.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index dea992c..174d464 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -985,7 +985,8 @@ apply_args_to_keylist(Args, Keys0) ->
view_cb({row, Row}, {iter, Db, Args, VAcc}) ->
NewRow = case lists:keymember(doc, 1, Row) of
true ->
- chttpd_stats:incr_reads();
+ chttpd_stats:incr_reads(),
+ Row;
false when Args#mrargs.include_docs ->
{id, DocId} = lists:keyfind(id, 1, Row),
chttpd_stats:incr_reads(),