You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2016/12/06 20:43:33 UTC
couchdb-mango git commit: Add config parameter to reject index all
text indexes
Repository: couchdb-mango
Updated Branches:
refs/heads/79577-add-config-index-all [created] a4cce0032
Add config parameter to reject index all text indexes
Text indexes that index all fields can sometimes lead to OOM issues
when users have documents with nested array fields. This change adds
in a config parameter to provide the ability to log users who do this
and also reject new requests. Note that we need to pass in a db
record to validate_new because it contains user and db name info that
will be logged.
COUCHDB-3249
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/a4cce003
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/a4cce003
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/a4cce003
Branch: refs/heads/79577-add-config-index-all
Commit: a4cce0032a9ca43301038eb46bfd4dc1dd6a4a49
Parents: 50066bc
Author: Tony Sun <to...@cloudant.com>
Authored: Tue Dec 6 00:46:51 2016 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Tue Dec 6 12:51:25 2016 -0800
----------------------------------------------------------------------
src/mango_error.erl | 6 ++++
src/mango_httpd.erl | 2 +-
src/mango_idx.erl | 6 ++--
src/mango_idx_text.erl | 83 +++++++++++++++++++++++++++++++++++++++++++--
src/mango_idx_view.erl | 4 +--
5 files changed, 93 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index f0a8ee2..76a8362 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -174,6 +174,12 @@ info(mango_idx_text, {index_not_found, BadIdx}) ->
<<"index_not_found">>,
fmt("Text index ~s not found in this design doc.", [BadIdx])
};
+info(mango_idx_text, index_all_disabled) ->
+ {
+ 403,
+ <<"index_all_disabled">>,
+ <<"New text indexes are forbidden to index all fields.">>
+ };
info(mango_opts, {invalid_bulk_docs, Val}) ->
{
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index bde3850..a088276 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -84,7 +84,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
chttpd:validate_ctype(Req, "application/json"),
{ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
{ok, Idx0} = mango_idx:new(Db, Opts),
- {ok, Idx} = mango_idx:validate_new(Idx0),
+ {ok, Idx} = mango_idx:validate_new(Idx0, Db),
{ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
Id = Idx#idx.ddoc,
Name = Idx#idx.name,
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 11c713b..bc88b97 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -23,7 +23,7 @@
for_sort/2,
new/2,
- validate_new/1,
+ validate_new/2,
add/2,
remove/2,
from_ddoc/2,
@@ -136,9 +136,9 @@ new(Db, Opts) ->
}}.
-validate_new(Idx) ->
+validate_new(Idx, Db) ->
Mod = idx_mod(Idx),
- Mod:validate_new(Idx).
+ Mod:validate_new(Idx, Db).
add(DDoc, Idx) ->
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl
index 40a1fd3..4cfda5a 100644
--- a/src/mango_idx_text.erl
+++ b/src/mango_idx_text.erl
@@ -14,7 +14,7 @@
-export([
- validate_new/1,
+ validate_new/2,
validate_fields/1,
validate_index_def/1,
add/2,
@@ -32,8 +32,9 @@
-include("mango_idx.hrl").
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, Db) ->
{ok, Def} = do_validate(Idx#idx.def),
+ maybe_reject_index_all_req(Def, Db),
{ok, Idx#idx{def=Def}}.
@@ -327,3 +328,81 @@ indexable_fields(Fields, {op_null, {_, _}}) ->
indexable_fields(Fields, {op_default, _}) ->
[<<"$default">> | Fields].
+
+
+maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) ->
+ User = Ctx#user_ctx.name,
+ Fields = couch_util:get_value(fields, Def),
+ case {Fields, forbid_index_all()} of
+ {all_fields, "true"} ->
+ ?MANGO_ERROR(index_all_disabled);
+ {all_fields, "warn"} ->
+ couch_log:warning("User ~p is indexing all fields in db ~p",
+ [User, DbName]);
+ _ ->
+ ok
+ end.
+
+
+forbid_index_all() ->
+ config:get("mango", "index_all_disabled", "false").
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+
+setup() ->
+ test_util:start_couch(),
+ meck:expect(couch_log, warning, 2,
+ fun(_,_) ->
+ throw({test_error, logged_warning})
+ end),
+ %default index all def that generates {fields, all_fields}
+ Index = #idx{def={[]}},
+ Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}},
+ {Index, Db}.
+
+
+teardown(_) ->
+ ok = config:delete("mango", "index_all_disabled"),
+ test_util:stop_couch().
+
+
+index_all_test_() ->
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun forbid_index_all/1,
+ fun default_and_false_index_all/1,
+ fun warn_index_all/1
+ ]
+
+ }.
+
+
+forbid_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "true"),
+ ?_assertThrow({mango_error, ?MODULE, index_all_disabled},
+ validate_new(Idx, Db)
+ ).
+
+
+default_and_false_index_all({Idx, Db}) ->
+ {ok, #idx{def={Def}}} = validate_new(Idx, Db),
+ Fields = couch_util:get_value(fields, Def),
+ ?_assertEqual(all_fields, Fields),
+ ok = config:set("mango", "index_all_disabled", "false"),
+ {ok, #idx{def={Def2}}} = validate_new(Idx, Db),
+ Fields2 = couch_util:get_value(fields, Def2),
+ ?_assertEqual(all_fields, Fields2).
+
+
+warn_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "warn"),
+ ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)).
+
+
+-endif.
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a4cce003/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index 0cc713f..8bad34c 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -14,7 +14,7 @@
-export([
- validate_new/1,
+ validate_new/2,
validate_index_def/1,
add/2,
remove/2,
@@ -36,7 +36,7 @@
-include("mango_idx.hrl").
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, _Db) ->
{ok, Def} = do_validate(Idx#idx.def),
{ok, Idx#idx{def=Def}}.