You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2015/10/05 22:47:23 UTC
[1/2] couchdb-mango git commit: Update Erlang releases for build
matrix
Repository: couchdb-mango
Updated Branches:
refs/heads/2816-malformed-docs 87020c220 -> 06cf106dc (forced update)
Update Erlang releases for build matrix
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/ea662ba9
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/ea662ba9
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/ea662ba9
Branch: refs/heads/2816-malformed-docs
Commit: ea662ba968ee8e1cf4075ff8ad9f097418f7d5a5
Parents: 46b58a3
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Oct 5 22:54:36 2015 +0300
Committer: Alexander Shorin <kx...@apache.org>
Committed: Mon Oct 5 22:54:36 2015 +0300
----------------------------------------------------------------------
.travis.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ea662ba9/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 2c6b2f0..d613012 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,11 +19,11 @@ before_script:
matrix:
include:
+ - otp_release: 18.1
+ python: 2.7
- otp_release: 17.5
python: 2.7
- otp_release: R16B03-1
python: 2.7
- - otp_release: R14B04
- python: 2.7
cache: apt
[2/2] couchdb-mango git commit: Handle malformed manual design docs.
Posted by to...@apache.org.
Handle malformed manual design docs.
We separate index validation into three phases. The first phase is
index creation via our _index api. This validation piece will throw
an error for invalid index definitions. The second phase is during
the indexing of documents. If an index definition is not valid, we
will not use the definition to index documents. We silently log the
error. Finally, during the query phase, design documents are again
validated to ensure correct indexes are used. Again, we log an error
but silently ignore invalid index definitions. Our validation will
integrate into a consolidated validation of all indexers.
COUCHDB-2816
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/06cf106d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/06cf106d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/06cf106d
Branch: refs/heads/2816-malformed-docs
Commit: 06cf106dc47188ce399b62c82c786197ef85a382
Parents: ea662ba
Author: Tony Sun <to...@cloudant.com>
Authored: Wed Sep 16 16:42:47 2015 -0700
Committer: Tony Sun <to...@cloudant.com>
Committed: Mon Oct 5 13:45:10 2015 -0700
----------------------------------------------------------------------
src/mango_httpd.erl | 2 +-
src/mango_idx.erl | 6 ++--
src/mango_idx_text.erl | 42 ++++++++++++++++------
src/mango_idx_view.erl | 46 +++++++++++++++++-------
src/mango_native_proc.erl | 38 ++++++++++++++++++--
test/05-index-selection-test.py | 69 ++++++++++++++++++++++++++++++++++++
6 files changed, 174 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index 8ac3d3c..ef83bda 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -83,7 +83,7 @@ handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
{ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
{ok, Idx0} = mango_idx:new(Db, Opts),
- {ok, Idx} = mango_idx:validate(Idx0),
+ {ok, Idx} = mango_idx:validate_new(Idx0),
{ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
Id = Idx#idx.ddoc,
Name = Idx#idx.name,
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 867cbef..377af91 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -23,7 +23,7 @@
for_sort/2,
new/2,
- validate/1,
+ validate_new/1,
add/2,
remove/2,
from_ddoc/2,
@@ -114,9 +114,9 @@ new(Db, Opts) ->
}}.
-validate(Idx) ->
+validate_new(Idx) ->
Mod = idx_mod(Idx),
- Mod:validate(Idx).
+ Mod:validate_new(Idx).
add(DDoc, Idx) ->
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl
index 321889a..fcd2939 100644
--- a/src/mango_idx_text.erl
+++ b/src/mango_idx_text.erl
@@ -14,8 +14,9 @@
-export([
- validate/1,
+ validate_new/1,
validate_fields/1,
+ validate_index_def/1,
add/2,
remove/2,
from_ddoc/1,
@@ -31,11 +32,15 @@
-include("mango_idx.hrl").
-validate(#idx{}=Idx) ->
+validate_new(#idx{}=Idx) ->
{ok, Def} = do_validate(Idx#idx.def),
{ok, Idx#idx{def=Def}}.
+validate_index_def(IndexInfo) ->
+ do_validate(IndexInfo).
+
+
add(#doc{body={Props0}}=DDoc, Idx) ->
Texts1 = case proplists:get_value(<<"indexes">>, Props0) of
{Texts0} -> Texts0;
@@ -72,14 +77,17 @@ from_ddoc({Props}) ->
case lists:keyfind(<<"indexes">>, 1, Props) of
{<<"indexes">>, {Texts}} when is_list(Texts) ->
lists:flatmap(fun({Name, {VProps}}) ->
- Def = proplists:get_value(<<"index">>, VProps),
- I = #idx{
- type = <<"text">>,
- name = Name,
- def = Def
- },
- % TODO: Validate the index definition
- [I]
+ case validate_ddoc(VProps) of
+ invalid_ddoc ->
+ [];
+ Def ->
+ I = #idx{
+ type = <<"text">>,
+ name = Name,
+ def = Def
+ },
+ [I]
+ end
end, Texts);
_ ->
[]
@@ -165,6 +173,8 @@ validate_field_type(<<"boolean">>) ->
<<"boolean">>.
+validate_fields(<<"all_fields">>) ->
+ {ok, all_fields};
validate_fields(Fields) ->
try fields_to_json(Fields) of
_ ->
@@ -174,6 +184,18 @@ validate_fields(Fields) ->
end.
+validate_ddoc(VProps) ->
+ try
+ Def = proplists:get_value(<<"index">>, VProps),
+ validate_index_def(Def),
+ Def
+ catch Error:Reason ->
+ couch_log:error("Invalid Index Def ~p: Error. ~p, Reason: ~p",
+ [VProps, Error, Reason]),
+ invalid_ddoc
+ end.
+
+
opts() ->
[
{<<"default_analyzer">>, [
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index cc5bcb3..0cc713f 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -14,7 +14,8 @@
-export([
- validate/1,
+ validate_new/1,
+ validate_index_def/1,
add/2,
remove/2,
from_ddoc/1,
@@ -35,11 +36,15 @@
-include("mango_idx.hrl").
-validate(#idx{}=Idx) ->
+validate_new(#idx{}=Idx) ->
{ok, Def} = do_validate(Idx#idx.def),
{ok, Idx#idx{def=Def}}.
+validate_index_def(Def) ->
+ def_to_json(Def).
+
+
add(#doc{body={Props0}}=DDoc, Idx) ->
Views1 = case proplists:get_value(<<"views">>, Props0) of
{Views0} -> Views0;
@@ -75,17 +80,18 @@ from_ddoc({Props}) ->
case lists:keyfind(<<"views">>, 1, Props) of
{<<"views">>, {Views}} when is_list(Views) ->
lists:flatmap(fun({Name, {VProps}}) ->
- Def = proplists:get_value(<<"map">>, VProps),
- {Opts0} = proplists:get_value(<<"options">>, VProps),
- Opts = lists:keydelete(<<"sort">>, 1, Opts0),
- I = #idx{
- type = <<"json">>,
- name = Name,
- def = Def,
- opts = Opts
- },
- % TODO: Validate the index definition
- [I]
+ case validate_ddoc(VProps) of
+ invalid_view ->
+ [];
+ {Def, Opts} ->
+ I = #idx{
+ type = <<"json">>,
+ name = Name,
+ def = Def,
+ opts = Opts
+ },
+ [I]
+ end
end, Views);
_ ->
[]
@@ -204,6 +210,20 @@ make_view(Idx) ->
{Idx#idx.name, View}.
+validate_ddoc(VProps) ->
+ try
+ Def = proplists:get_value(<<"map">>, VProps),
+ validate_index_def(Def),
+ {Opts0} = proplists:get_value(<<"options">>, VProps),
+ Opts = lists:keydelete(<<"sort">>, 1, Opts0),
+ {Def, Opts}
+ catch Error:Reason ->
+ couch_log:error("Invalid Index Def ~p. Error: ~p, Reason: ~p",
+ [VProps, Error, Reason]),
+ invalid_view
+ end.
+
+
% This function returns a list of indexes that
% can be used to restrict this query. This works by
% searching the selector looking for field names that
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_native_proc.erl
----------------------------------------------------------------------
diff --git a/src/mango_native_proc.erl b/src/mango_native_proc.erl
index 9481ca4..822d173 100644
--- a/src/mango_native_proc.erl
+++ b/src/mango_native_proc.erl
@@ -14,6 +14,9 @@
-behavior(gen_server).
+-include("mango_idx.hrl").
+
+
-export([
start_link/0,
set_timeout/2,
@@ -72,7 +75,13 @@ handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) ->
{reply, true, St#st{indexes=[]}};
handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
- Indexes = St#st.indexes ++ [IndexInfo],
+ Indexes = case validate_index_info(IndexInfo) of
+ true ->
+ St#st.indexes ++ [IndexInfo];
+ false ->
+ couch_log:error("No Valid Indexes For: ~p", [IndexInfo]),
+ St#st.indexes
+ end,
NewSt = St#st{indexes = Indexes},
{reply, true, NewSt};
@@ -86,7 +95,14 @@ handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) ->
{reply, null, St};
handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
- {reply, index_doc(St, mango_json:to_binary(Doc)), St};
+ Vals = case index_doc(St, mango_json:to_binary(Doc)) of
+ [] ->
+ [[]];
+ Else ->
+ Else
+ end,
+ {reply, Vals, St};
+
handle_call(Msg, _From, St) ->
{stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
@@ -296,3 +312,21 @@ make_text_field_name([P | Rest], Type) ->
Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
iolist_to_binary(mango_util:join(".", Escaped)).
+
+
+validate_index_info(IndexInfo) ->
+ IdxTypes = case module_loaded(dreyfus_index) of
+ true ->
+ [mango_idx_view, mango_idx_text];
+ false ->
+ [mango_idx_view]
+ end,
+ Results = lists:foldl(fun(IdxType, Results0) ->
+ try
+ IdxType:validate_index_def(IndexInfo),
+ [valid_index | Results0]
+ catch _:_ ->
+ [invalid_index | Results0]
+ end
+ end, [], IdxTypes),
+ lists:member(valid_index, Results).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/test/05-index-selection-test.py
----------------------------------------------------------------------
diff --git a/test/05-index-selection-test.py b/test/05-index-selection-test.py
index eecaf17..3a757f8 100644
--- a/test/05-index-selection-test.py
+++ b/test/05-index-selection-test.py
@@ -74,6 +74,75 @@ class IndexSelectionTests(mango.UserDocsTests):
}, use_index=ddocid, explain=True)
assert resp["index"]["ddoc"] == ddocid
+ # This doc will not be saved given the new ddoc validation code
+ # in couch_mrview
+ def test_manual_bad_view_idx01(self):
+ design_doc = {
+ "_id": "_design/bad_view_index",
+ "language": "query",
+ "views": {
+ "queryidx1": {
+ "map": {
+ "fields": {
+ "age": "asc"
+ }
+ },
+ "reduce": "_count",
+ "options": {
+ "def": {
+ "fields": [
+ {
+ "age": "asc"
+ }
+ ]
+ },
+ "w": 2
+ }
+ }
+ },
+ "views" : {
+ "views001" : {
+ "map" : "function(employee){if(employee.training)"
+ + "{emit(employee.number, employee.training);}}"
+ }
+ }
+ }
+ with self.assertRaises(KeyError):
+ self.db.save_doc(design_doc)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_manual_bad_text_idx(self):
+ design_doc = {
+ "_id": "_design/bad_text_index",
+ "language": "query",
+ "indexes": {
+ "text_index": {
+ "default_analyzer": "keyword",
+ "default_field": {},
+ "selector": {},
+ "fields": "all_fields",
+ "analyzer": {
+ "name": "perfield",
+ "default": "keyword",
+ "fields": {
+ "$default": "standard"
+ }
+ }
+ }
+ },
+ "indexes": {
+ "st_index": {
+ "analyzer": "standard",
+ "index": "function(doc){\n index(\"st_index\", doc.geometry);\n}"
+ }
+ }
+ }
+ self.db.save_doc(design_doc)
+ docs= self.db.find({"age" : 48})
+ assert len(docs) == 1
+ assert docs[0]["name"]["first"] == "Stephanie"
+ assert docs[0]["age"] == 48
+
@unittest.skipUnless(mango.has_text_service(), "requires text service")
class MultiTextIndexSelectionTests(mango.UserDocsTests):