You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ro...@apache.org on 2015/02/03 16:13:56 UTC
[49/50] [abbrv] couchdb-mango git commit: Add tests to test for field
names with period
Add tests to test for field names with period
BugzId:43621
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/a7c88e58
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/a7c88e58
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/a7c88e58
Branch: refs/heads/master
Commit: a7c88e589ff6d261cbc6ab9f59edb613df034626
Parents: 8af882b
Author: Tony Sun <to...@cloudant.com>
Authored: Wed Jan 21 18:21:19 2015 -0800
Committer: Tony Sun <to...@cloudant.com>
Committed: Fri Jan 23 15:39:17 2015 -0800
----------------------------------------------------------------------
src/mango_native_proc.erl | 43 ++++++-----------------------
src/mango_selector_text.erl | 24 ++++++----------
src/mango_util.erl | 7 +++++
test/04-key-tests.py | 30 ++++++++++++++++++--
test/07-text-custom-field-list-test.py | 17 +++++++++++-
test/user_docs.py | 3 +-
6 files changed, 71 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/src/mango_native_proc.erl
----------------------------------------------------------------------
diff --git a/src/mango_native_proc.erl b/src/mango_native_proc.erl
index a55a3ac..9481ca4 100644
--- a/src/mango_native_proc.erl
+++ b/src/mango_native_proc.erl
@@ -168,8 +168,7 @@ get_text_field_values(Values, TAcc) when is_list(Values) ->
% We bypass make_text_field and directly call make_text_field_name
% because the length field name is not part of the path.
LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>),
- LengthField = [{escape_name_parts(LengthFieldName), <<"length">>,
- length(Values)}],
+ LengthField = [{LengthFieldName, <<"length">>, length(Values)}],
get_text_field_values_arr(Values, NewTAcc, LengthField);
get_text_field_values(Bin, TAcc) when is_binary(Bin) ->
@@ -258,7 +257,8 @@ should_index(Selector, Doc) ->
get_text_field_list(IdxProps) ->
case couch_util:get_value(<<"fields">>, IdxProps) of
Fields when is_list(Fields) ->
- lists:flatmap(fun get_text_field_info/1, Fields);
+ RawList = lists:flatmap(fun get_text_field_info/1, Fields),
+ [mango_util:lucene_escape_user(Field) || Field <- RawList];
_ ->
all_fields
end.
@@ -283,41 +283,16 @@ get_text_field_type(_) ->
make_text_field(TAcc, Type, Value) ->
FieldName = make_text_field_name(TAcc#tacc.path, Type),
- Fields = case TAcc#tacc.fields of
- Fields0 when is_list(Fields0) ->
- % for field names such as "name\\.first"
- PFields = [mango_doc:parse_field(F) || F <- Fields0],
- [iolist_to_binary(mango_util:join(".", P)) || {ok, P} <- PFields];
- _ ->
- all_fields
- end,
- % need to convert the fieldname to binary but not escape it in order
- % to compare with the user field names.
- BName = iolist_to_binary(FieldName),
- case Fields == all_fields orelse lists:member(BName, Fields) of
+ Fields = TAcc#tacc.fields,
+ case Fields == all_fields orelse lists:member(FieldName, Fields) of
true ->
- [{escape_name_parts(FieldName), Type, Value}];
+ [{FieldName, Type, Value}];
false ->
[]
end.
make_text_field_name([P | Rest], Type) ->
- make_text_field_name0(Rest, [P, ":", Type]).
-
-make_text_field_name0([], Name) ->
- Name;
-make_text_field_name0([P | Rest], Name) ->
- make_text_field_name0(Rest, [P, "." | Name]).
-
-
-escape_name_parts(Name) ->
- EscapedName = lists:map(fun(N) ->
- case N of
- "." ->
- ".";
- Else ->
- mango_util:lucene_escape_field(Else)
- end
- end, Name),
- iolist_to_binary(EscapedName).
+ Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
+ Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
+ iolist_to_binary(mango_util:join(".", Escaped)).
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/src/mango_selector_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_selector_text.erl b/src/mango_selector_text.erl
index 9a69853..bdd82bf 100644
--- a/src/mango_selector_text.erl
+++ b/src/mango_selector_text.erl
@@ -171,9 +171,8 @@ convert(_Path, {Props} = Sel) when length(Props) > 1 ->
to_query({op_and, Args}) when is_list(Args) ->
- Res = ["(", mango_util:join(<<" AND ">>, lists:map(fun to_query/1, Args)),
- ")"],
- Res;
+ QueryArgs = lists:map(fun to_query/1, Args),
+ ["(", mango_util:join(<<" AND ">>, QueryArgs), ")"];
to_query({op_or, Args}) when is_list(Args) ->
["(", mango_util:join(" OR ", lists:map(fun to_query/1, Args)), ")"];
@@ -192,27 +191,21 @@ to_query({op_insert, Arg}) when is_binary(Arg) ->
%% This needs to be resolved.
to_query({op_field, {Name, Value}}) ->
NameBin = iolist_to_binary(Name),
- ["(", escape_name_parts(NameBin), ":", Value, ")"];
+ ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
%% This is for indexable_fields
to_query({op_null, {Name, Value}}) ->
NameBin = iolist_to_binary(Name),
- ["(", escape_name_parts(NameBin), ":", Value, ")"];
+ ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
to_query({op_fieldname, {Name, Wildcard}}) ->
NameBin = iolist_to_binary(Name),
- ["($fieldnames:", escape_name_parts(NameBin), Wildcard, ")"];
+ ["($fieldnames:", mango_util:lucene_escape_user(NameBin), Wildcard, ")"];
to_query({op_default, Value}) ->
["($default:", Value, ")"].
-escape_name_parts(Name) ->
- {ok, ParsedNames} = mango_doc:parse_field(Name),
- EncodedNames = [mango_util:lucene_escape_field(N) || N <- ParsedNames],
- iolist_to_binary(mango_util:join(".", EncodedNames)).
-
-
%% We match on fieldname and fieldname.[]
convert_in(Path, Args) ->
Path0 = [<<"[]">> | Path],
@@ -256,9 +249,10 @@ field_exists_query(Path) ->
% match a path foo.name against foo.name_first (if were to just
% appened * isntead).
Parts = [
+ % We need to remove the period from the path list to indicate that it is
+ % a path separator. We escape the colon because it is not used as a
+ % separator and we escape colons in field names.
{op_fieldname, {[path_str(Path), ":"], "*"}},
- % need to extract out the period because mango_doc:parse_field/1
- % will not accept "name.", also we don't want to escape the .
{op_fieldname, {[path_str(Path)], ".*"}}
],
{op_or, Parts}.
@@ -307,7 +301,7 @@ value_str(null) ->
append_sort_type(RawSortField, Selector) ->
- EncodeField = escape_name_parts(RawSortField),
+ EncodeField = mango_util:lucene_escape_user(RawSortField),
String = mango_util:has_suffix(EncodeField, <<"_3astring">>),
Number = mango_util:has_suffix(EncodeField, <<"_3anumber">>),
case {String, Number} of
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/src/mango_util.erl
----------------------------------------------------------------------
diff --git a/src/mango_util.erl b/src/mango_util.erl
index b1d30fb..f350710 100644
--- a/src/mango_util.erl
+++ b/src/mango_util.erl
@@ -33,6 +33,7 @@
lucene_escape_field/1,
lucene_escape_query_value/1,
+ lucene_escape_user/1,
has_suffix/2,
@@ -283,6 +284,12 @@ lucene_escape_qv(<<C, Rest/binary>>) ->
Out ++ lucene_escape_qv(Rest).
+lucene_escape_user(Field) ->
+ {ok, Path} = mango_doc:parse_field(Field),
+ Escaped = [mango_util:lucene_escape_field(P) || P <- Path],
+ iolist_to_binary(join(".", Escaped)).
+
+
has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) ->
SBin = size(Bin),
SSuffix = size(Suffix),
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/test/04-key-tests.py
----------------------------------------------------------------------
diff --git a/test/04-key-tests.py b/test/04-key-tests.py
index c673cf2..0109737 100644
--- a/test/04-key-tests.py
+++ b/test/04-key-tests.py
@@ -26,7 +26,8 @@ TEST_DOCS = [
"dot.key": "dot's value",
"none": {
"dot": "none dot's value"
- }
+ },
+ "name.first" : "Kvothe"
},
{
"type": "complex_key",
@@ -34,7 +35,8 @@ TEST_DOCS = [
"$key": "peso",
"deep": {
"$key": "deep peso"
- }
+ },
+ "name": {"first" : "Master Elodin"}
},
{
"type": "complex_key",
@@ -121,3 +123,27 @@ class KeyTests(mango.DbPerClass):
assert docs[0]["title"] == "internal_fields_format"
for query in queries:
self.run_check(query, check, indexes=["text"])
+
+ def test_escape_period(self):
+ query = {"name\\.first" : "Kvothe"}
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0]["name.first"] == "Kvothe"
+ self.run_check(query, check, indexes=["text"])
+
+ query = {"name.first" : "Kvothe"}
+ def check_empty(docs):
+ assert len(docs) == 0
+ self.run_check(query, check_empty, indexes=["text"])
+
+ def test_object_period(self):
+ query = {"name.first" : "Master Elodin"}
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0]["title"] == "key with peso"
+ self.run_check(query, check, indexes=["text"])
+
+ query = {"name\\.first" : "Master Elodin"}
+ def check_empty(docs):
+ assert len(docs) == 0
+ self.run_check(query, check_empty, indexes=["text"])
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/test/07-text-custom-field-list-test.py
----------------------------------------------------------------------
diff --git a/test/07-text-custom-field-list-test.py b/test/07-text-custom-field-list-test.py
index 5e5f7cc..a019ea2 100644
--- a/test/07-text-custom-field-list-test.py
+++ b/test/07-text-custom-field-list-test.py
@@ -26,7 +26,8 @@ class CustomFieldsTest(mango.UserDocsTextTests):
{
"name": "location.address.street",
"type": "string"
- }
+ },
+ {"name": "name\\.first", "type": "string"}
]
def test_basic(self):
@@ -60,3 +61,17 @@ class CustomFieldsTest(mango.UserDocsTextTests):
docs = self.db.find({"location.state": "New Hampshire"})
assert len(docs) == 1
assert docs[0]["user_id"] == 10
+
+ # Since our FIELDS list only includes "name\\.first", we should
+ # get an error when we try to search for "name.first", since the index
+ # for that field does not exist.
+ def test_escaped_field(self):
+ docs = self.db.find({"name\\.first": "name dot first"})
+ assert len(docs) == 1
+ assert docs[0]["name.first"] == "name dot first"
+
+ try:
+ self.db.find({"name.first": "name dot first"})
+ raise Exception("Should have thrown an HTTPError")
+ except:
+ return
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/a7c88e58/test/user_docs.py
----------------------------------------------------------------------
diff --git a/test/user_docs.py b/test/user_docs.py
index 05eabbb..baf83f7 100644
--- a/test/user_docs.py
+++ b/test/user_docs.py
@@ -224,7 +224,8 @@ DOCS = [
"C",
"Ruby",
"Ruby"
- ]
+ ],
+ "name.first" : "name dot first"
},
{
"_id": "a33d5457-741a-4dce-a217-3eab28b24e3e",