You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@olingo.apache.org by mi...@apache.org on 2015/12/01 11:21:00 UTC
[1/2] olingo-odata4 git commit: [OLINGO-568] Search grammar rewritten
Repository: olingo-odata4
Updated Branches:
refs/heads/master 8674a1f29 -> e5ac59079
[OLINGO-568] Search grammar rewritten
Project: http://git-wip-us.apache.org/repos/asf/olingo-odata4/repo
Commit: http://git-wip-us.apache.org/repos/asf/olingo-odata4/commit/6dd0a0f3
Tree: http://git-wip-us.apache.org/repos/asf/olingo-odata4/tree/6dd0a0f3
Diff: http://git-wip-us.apache.org/repos/asf/olingo-odata4/diff/6dd0a0f3
Branch: refs/heads/master
Commit: 6dd0a0f3e51d2db7b6ffb9023e81b9fd8038903c
Parents: 8674a1f
Author: Christian Holzer <c....@sap.com>
Authored: Tue Nov 24 17:02:17 2015 +0100
Committer: Christian Holzer <c....@sap.com>
Committed: Tue Nov 24 17:02:17 2015 +0100
----------------------------------------------------------------------
.../core/uri/parser/search/SearchParser.java | 195 +++++++++----------
.../parser/search/SearchParserException.java | 16 +-
.../server-core-exceptions-i18n.properties | 8 +-
.../search/SearchParserAndTokenizerTest.java | 61 ++++--
.../uri/parser/search/SearchParserTest.java | 27 +--
.../uri/parser/search/SearchTokenizerTest.java | 1 -
6 files changed, 154 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java
index 4d93f73..ac0bb7d 100644
--- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java
+++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java
@@ -18,6 +18,9 @@
*/
package org.apache.olingo.server.core.uri.parser.search;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.olingo.server.api.uri.queryoption.SearchOption;
import org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind;
import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression;
@@ -25,8 +28,15 @@ import org.apache.olingo.server.api.uri.queryoption.search.SearchTerm;
import org.apache.olingo.server.core.uri.parser.search.SearchQueryToken.Token;
import org.apache.olingo.server.core.uri.queryoption.SearchOptionImpl;
-import java.util.Iterator;
-import java.util.List;
+/*
+ * Rewritten grammar
+ *
+ * SearchExpr ::= ExprOR
+ * ExprOR ::= ExprAnd ('OR' ExprAnd)*
+ * ExprAnd ::= Term ('AND'? Term)*
+ * Term ::= ('NOT')? (Word | Phrase)
+ * | '(' Expr ')'
+ */
public class SearchParser {
@@ -54,60 +64,58 @@ public class SearchParser {
if (token == null) {
throw new SearchParserException("No search String", SearchParserException.MessageKeys.NO_EXPRESSION_FOUND);
}
- SearchExpression se = processSearchExpression(null);
+ SearchExpression searchExpression = processSearchExpression();
if(!isEof()) {
throw new SearchParserException("Token left after end of search query parsing.",
- SearchParserException.MessageKeys.INVALID_END_OF_QUERY_TOKEN_LEFT, token.getToken().name());
+ SearchParserException.MessageKeys.INVALID_END_OF_QUERY, getTokenAsString());
}
- return se;
+ return searchExpression;
}
- private SearchExpression processSearchExpression(SearchExpression left) throws SearchParserException {
- if (isEof()) {
- return left;
- }
+ private SearchExpression processSearchExpression() throws SearchParserException {
+ return processExprOr();
+ }
- if (left == null && (isToken(SearchQueryToken.Token.AND) || isToken(SearchQueryToken.Token.OR))) {
- throw new SearchParserException(token.getToken() + " needs a left operand.",
- SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION, token.getToken().toString());
+ private SearchExpression processExprOr() throws SearchParserException {
+ SearchExpression left = processExprAnd();
+
+ while(isToken(Token.OR)) {
+ nextToken(); // Match OR
+ final SearchExpression right = processExprAnd();
+ left = new SearchBinaryImpl(left, SearchBinaryOperatorKind.OR, right);
}
+
+ return left;
+ }
- SearchExpression expression = left;
- if (isToken(SearchQueryToken.Token.OPEN)) {
- processOpen();
- expression = processSearchExpression(left);
- if (expression == null) {
- throw new SearchParserException("Brackets must contain an expression.",
- SearchParserException.MessageKeys.NO_EXPRESSION_FOUND);
+ private SearchExpression processExprAnd() throws SearchParserException {
+ SearchExpression left = processTerm();
+
+ while(isToken(Token.AND) || isTerm()) {
+ if(isToken(Token.AND)) {
+ nextToken(); // Match AND
}
- processClose();
- } else if (isTerm()) {
- expression = processTerm();
+ final SearchExpression right = processTerm();
+ left = new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, right);
}
-
-
- if (isToken(SearchQueryToken.Token.AND) || isToken(SearchQueryToken.Token.OPEN) || isTerm()) {
- expression = processAnd(expression);
- } else if (isToken(SearchQueryToken.Token.OR)) {
- expression = processOr(expression);
- } else if (isEof()) {
- return expression;
- }
- return expression;
- }
-
- private boolean isTerm() {
- return isToken(SearchQueryToken.Token.NOT)
- || isToken(SearchQueryToken.Token.PHRASE)
- || isToken(SearchQueryToken.Token.WORD);
+
+ return left;
}
- private boolean isEof() {
- return token == null;
- }
-
- private boolean isToken(SearchQueryToken.Token toCheckToken) {
- return token != null && token.getToken() == toCheckToken;
+ private SearchExpression processTerm() throws SearchParserException {
+ if(isToken(SearchQueryToken.Token.OPEN)) {
+ nextToken(); // Match OPEN
+ final SearchExpression expr = processExprOr();
+ processClose();
+
+ return expr;
+ } else {
+ // ('NOT')? (Word | Phrase)
+ if (isToken(SearchQueryToken.Token.NOT)) {
+ return processNot();
+ }
+ return processWordOrPhrase();
+ }
}
private void processClose() throws SearchParserException {
@@ -119,84 +127,28 @@ public class SearchParser {
}
}
- private void processOpen() {
- nextToken();
- }
-
- private SearchExpression processAnd(SearchExpression left) throws SearchParserException {
- if (isToken(SearchQueryToken.Token.AND)) {
- nextToken();
- }
- SearchExpression se = left;
- if (isTerm()) {
- se = processTerm();
- if(isTerm()) {
- se = processAnd(se);
- }
- se = new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, se);
- return processSearchExpression(se);
- } else {
- if (isToken(SearchQueryToken.Token.AND) || isToken(SearchQueryToken.Token.OR)) {
- throw new SearchParserException("Operators must not be followed by an AND or an OR",
- SearchParserException.MessageKeys.INVALID_OPERATOR_AFTER_AND, token.getToken().name());
- } else if(isEof()) {
- throw new SearchParserException("Missing search expression after AND (found end of search query)",
- SearchParserException.MessageKeys.INVALID_END_OF_QUERY, Token.AND.name());
- }
- se = processSearchExpression(se);
- return new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, se);
- }
- }
-
- public SearchExpression processOr(SearchExpression left) throws SearchParserException {
- if (isToken(SearchQueryToken.Token.OR)) {
- nextToken();
- }
- if(isEof()) {
- throw new SearchParserException("Missing search expression after OR (found end of search query)",
- SearchParserException.MessageKeys.INVALID_END_OF_QUERY, Token.OR.name());
- }
- SearchExpression se = processSearchExpression(left);
- return new SearchBinaryImpl(left, SearchBinaryOperatorKind.OR, se);
- }
-
private SearchExpression processNot() throws SearchParserException {
nextToken();
if (isToken(Token.WORD) || isToken(Token.PHRASE)) {
return new SearchUnaryImpl(processWordOrPhrase());
}
- if(isEof()) {
- throw new SearchParserException("NOT must be followed by a term.",
- SearchParserException.MessageKeys.INVALID_NOT_OPERAND, "EOF");
- }
- throw new SearchParserException("NOT must be followed by a term not a " + token.getToken(),
- SearchParserException.MessageKeys.INVALID_NOT_OPERAND, token.getToken().toString());
- }
-
- private void nextToken() {
- if (tokens.hasNext()) {
- token = tokens.next();
- } else {
- token = null;
- }
- }
- private SearchExpression processTerm() throws SearchParserException {
- if (isToken(SearchQueryToken.Token.NOT)) {
- return processNot();
- }
- return processWordOrPhrase();
+ final String tokenAsString = getTokenAsString();
+ throw new SearchParserException("NOT must be followed by a term not a " + tokenAsString,
+ SearchParserException.MessageKeys.INVALID_NOT_OPERAND, tokenAsString);
}
-
+
private SearchTerm processWordOrPhrase() throws SearchParserException {
if (isToken(Token.PHRASE)) {
return processPhrase();
} else if (isToken(Token.WORD)) {
return processWord();
}
- throw new SearchParserException("Expected PHRASE||WORD found: " + token.getToken(),
+
+ final String tokenName = getTokenAsString();
+ throw new SearchParserException("Expected PHRASE||WORD found: " + tokenName,
SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN,
- Token.PHRASE.name() + "" + Token.WORD.name(), token.getToken().toString());
+ Token.PHRASE.name() + "" + Token.WORD.name(), tokenName);
}
private SearchTerm processWord() {
@@ -210,4 +162,31 @@ public class SearchParser {
nextToken();
return new SearchTermImpl(literal.substring(1,literal.length()-1));
}
+
+ private boolean isTerm() {
+ return isToken(SearchQueryToken.Token.NOT)
+ || isToken(SearchQueryToken.Token.PHRASE)
+ || isToken(SearchQueryToken.Token.WORD)
+ || isToken(SearchQueryToken.Token.OPEN);
+ }
+
+ private boolean isEof() {
+ return token == null;
+ }
+
+ private boolean isToken(SearchQueryToken.Token toCheckToken) {
+ return token != null && token.getToken() == toCheckToken;
+ }
+
+ private void nextToken() {
+ if (tokens.hasNext()) {
+ token = tokens.next();
+ } else {
+ token = null;
+ }
+ }
+
+ private String getTokenAsString() {
+ return token == null ? "<EOF>" : token.getToken().name();
+ }
}
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParserException.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParserException.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParserException.java
index 1bc60dc..d7af4d4 100644
--- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParserException.java
+++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParserException.java
@@ -28,23 +28,15 @@ public class SearchParserException extends UriParserSyntaxException {
NO_EXPRESSION_FOUND,
/** parameter: message */
TOKENIZER_EXCEPTION,
- /** parameter: tokenCharacter */
- INVALID_TOKEN_CHARACTER_FOUND,
- /** parameter: operatorkind */
- INVALID_BINARY_OPERATOR_POSITION,
/** parameter: operatorkind */
INVALID_NOT_OPERAND,
/** parameters: - */
MISSING_CLOSE,
/** parameters: expectedToken actualToken */
EXPECTED_DIFFERENT_TOKEN,
- /** parameters: actualToken */
- INVALID_END_OF_QUERY,
- /** parameters: left_over_token */
- INVALID_END_OF_QUERY_TOKEN_LEFT,
- /** parameter: operatorkind */
- INVALID_OPERATOR_AFTER_AND;
-
+ /** parameter: actual token */
+ INVALID_END_OF_QUERY;
+
@Override
public String getKey() {
return name();
@@ -60,5 +52,5 @@ public class SearchParserException extends UriParserSyntaxException {
final String... parameters) {
super(developmentMessage, cause, messageKey, parameters);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/main/resources/server-core-exceptions-i18n.properties
----------------------------------------------------------------------
diff --git a/lib/server-core/src/main/resources/server-core-exceptions-i18n.properties b/lib/server-core/src/main/resources/server-core-exceptions-i18n.properties
index a2baf0f..3de8814 100644
--- a/lib/server-core/src/main/resources/server-core-exceptions-i18n.properties
+++ b/lib/server-core/src/main/resources/server-core-exceptions-i18n.properties
@@ -37,16 +37,12 @@ UriParserSyntaxException.SYSTEM_QUERY_OPTION_LEVELS_NOT_ALLOWED_HERE=The system
UriParserSyntaxException.SYNTAX=The URI is malformed.
UriParserSyntaxException.DUPLICATED_ALIAS=Duplicated alias. An alias '%1$s' was already specified!.
+SearchParserException.NO_EXPRESSION_FOUND=No expression found.
SearchParserException.TOKENIZER_EXCEPTION=Exception during tokenizer creation with message '%1$s'.
-SearchParserException.INVALID_TOKEN_CHARACTER_FOUND=Invalid token character with value '%1$s' found.
-SearchParserException.INVALID_BINARY_OPERATOR_POSITION=Invalid binary operator position for kind '%1$s' found.
SearchParserException.INVALID_NOT_OPERAND=Invalid not operand for kind '%1$s' found.
+SearchParserException.MISSING_CLOSE=Missing closing bracket after an opening bracket.
SearchParserException.EXPECTED_DIFFERENT_TOKEN=Expected token '%1$s' but found '%2$s'.
-SearchParserException.NO_EXPRESSION_FOUND=No expression found.
-SearchParserException.INVALID_OPERATOR_AFTER_AND=Invalid operator after AND found of kind '%1$s'.
SearchParserException.INVALID_END_OF_QUERY=Invalid end of search query after '%1$s' (query must end with a search phrase or word).
-SearchParserException.INVALID_END_OF_QUERY_TOKEN_LEFT=Invalid end of search query. Found not processed token '%1$s' at the end.
-SearchParserException.MISSING_CLOSE=Missing closing bracket after an opening bracket.
SearchTokenizerException.FORBIDDEN_CHARACTER=Not allowed character '%1$s' found for token '%2$s'.
SearchTokenizerException.NOT_EXPECTED_TOKEN=Not expected token '%1$s' found.
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java
index f6722b8..b341b0e 100644
--- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java
+++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java
@@ -25,6 +25,8 @@ import org.junit.Test;
public class SearchParserAndTokenizerTest {
+ private static final String EOF = "<EOF>";
+
@Test
public void basicParsing() throws Exception {
assertQuery("\"99\"").resultsIn("'99'");
@@ -32,7 +34,12 @@ public class SearchParserAndTokenizerTest {
assertQuery("a AND b").resultsIn("{'a' AND 'b'}");
assertQuery("a AND b AND c").resultsIn("{{'a' AND 'b'} AND 'c'}");
assertQuery("a OR b").resultsIn("{'a' OR 'b'}");
- assertQuery("a OR b OR c").resultsIn("{'a' OR {'b' OR 'c'}}");
+ assertQuery("a OR b OR c").resultsIn("{{'a' OR 'b'} OR 'c'}");
+
+ assertQuery("NOT a NOT b").resultsIn("{{NOT 'a'} AND {NOT 'b'}}");
+ assertQuery("NOT a AND NOT b").resultsIn("{{NOT 'a'} AND {NOT 'b'}}");
+ assertQuery("NOT a OR NOT b").resultsIn("{{NOT 'a'} OR {NOT 'b'}}");
+ assertQuery("NOT a OR NOT b NOT C").resultsIn("{{NOT 'a'} OR {{NOT 'b'} AND {NOT 'C'}}}");
}
@Test
@@ -51,34 +58,44 @@ public class SearchParserAndTokenizerTest {
public void parenthesesParsing() throws Exception {
assertQuery("a AND (b OR c)").resultsIn("{'a' AND {'b' OR 'c'}}");
assertQuery("(a OR b) AND NOT c").resultsIn("{{'a' OR 'b'} AND {NOT 'c'}}");
+ assertQuery("(a OR B) AND (c OR d AND NOT e OR (f))")
+ .resultsIn("{{'a' OR 'B'} AND {{'c' OR {'d' AND {NOT 'e'}}} OR 'f'}}");
+ assertQuery("(a OR B) (c OR d NOT e OR (f))")
+ .resultsIn("{{'a' OR 'B'} AND {{'c' OR {'d' AND {NOT 'e'}}} OR 'f'}}");
+ assertQuery("((((a))))").resultsIn("'a'");
+ assertQuery("((((a)))) ((((a))))").resultsIn("{'a' AND 'a'}");
+ assertQuery("((((a)))) OR ((((a))))").resultsIn("{'a' OR 'a'}");
+ assertQuery("((((((a)))) ((((c))) OR (((C)))) ((((a))))))").resultsIn("{{'a' AND {'c' OR 'C'}} AND 'a'}");
+ assertQuery("((((\"a\")))) OR ((((\"a\"))))").resultsIn("{'a' OR 'a'}");
}
-
+
@Test
public void parseImplicitAnd() throws Exception {
assertQuery("a b").resultsIn("{'a' AND 'b'}");
- assertQuery("a b c").resultsIn("{'a' AND {'b' AND 'c'}}");
- assertQuery("a and b").resultsIn("{'a' AND {'and' AND 'b'}}");
+ assertQuery("a b c").resultsIn("{{'a' AND 'b'} AND 'c'}");
+ assertQuery("a and b").resultsIn("{{'a' AND 'and'} AND 'b'}");
+ assertQuery("hey ANDy warhol").resultsIn("{{'hey' AND 'ANDy'} AND 'warhol'}");
assertQuery("a b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}");
assertQuery("a \"bc123\" OR c").resultsIn("{{'a' AND 'bc123'} OR 'c'}");
- assertQuery("(a OR x) bc c").resultsIn("{{'a' OR 'x'} AND {'bc' AND 'c'}}");
- assertQuery("one ((a OR x) bc c)").resultsIn("{'one' AND {{'a' OR 'x'} AND {'bc' AND 'c'}}}");
+ assertQuery("(a OR x) bc c").resultsIn("{{{'a' OR 'x'} AND 'bc'} AND 'c'}");
+ assertQuery("one ((a OR x) bc c)").resultsIn("{'one' AND {{{'a' OR 'x'} AND 'bc'} AND 'c'}}");
}
@Test
public void invalidSearchQuery() throws Exception {
assertQuery("99").resultsIn(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
assertQuery("NOT").resultsIn(SearchParserException.MessageKeys.INVALID_NOT_OPERAND);
- assertQuery("AND").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
- assertQuery("OR").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
+ assertQuery("AND").resultsInExpectedTerm(SearchQueryToken.Token.AND.name());
+ assertQuery("OR").resultsInExpectedTerm(SearchQueryToken.Token.OR.name());
- assertQuery("NOT a AND").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
- assertQuery("NOT a OR").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
- assertQuery("a AND").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
- assertQuery("a OR").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
+ assertQuery("NOT a AND").resultsInExpectedTerm(EOF);
+ assertQuery("NOT a OR").resultsInExpectedTerm(EOF);
+ assertQuery("a AND").resultsInExpectedTerm(EOF);
+ assertQuery("a OR").resultsInExpectedTerm(EOF);
- assertQuery("a OR b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY_TOKEN_LEFT);
- assertQuery("a NOT b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY_TOKEN_LEFT);
- assertQuery("a AND b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY_TOKEN_LEFT);
+ assertQuery("a OR b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
+ assertQuery("a NOT b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
+ assertQuery("a AND b)").resultsIn(SearchParserException.MessageKeys.INVALID_END_OF_QUERY);
assertQuery("(a OR b").resultsIn(SearchParserException.MessageKeys.MISSING_CLOSE);
assertQuery("(a NOT b").resultsIn(SearchParserException.MessageKeys.MISSING_CLOSE);
@@ -86,6 +103,9 @@ public class SearchParserAndTokenizerTest {
assertQuery("((a AND b OR c)").resultsIn(SearchParserException.MessageKeys.MISSING_CLOSE);
assertQuery("a AND (b OR c").resultsIn(SearchParserException.MessageKeys.MISSING_CLOSE);
assertQuery("(a AND ((b OR c)").resultsIn(SearchParserException.MessageKeys.MISSING_CLOSE);
+
+ assertQuery("NOT NOT a").resultsIn(SearchParserException.MessageKeys.INVALID_NOT_OPERAND);
+ assertQuery("NOT (a)").resultsIn(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
}
private static Validator assertQuery(String searchQuery) {
@@ -125,7 +145,16 @@ public class SearchParserAndTokenizerTest {
}
Assert.fail("SearchParserException with message key " + key.getKey() + " was not thrown.");
}
-
+
+ public void resultsInExpectedTerm(final String actualToken) throws SearchTokenizerException {
+ try {
+ resultsIn(searchQuery);
+ } catch(SearchParserException e) {
+ Assert.assertEquals(SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN, e.getMessageKey());
+ Assert.assertEquals("Expected PHRASE||WORD found: " + actualToken, e.getMessage());
+ }
+ }
+
private void resultsIn(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException {
final SearchExpression searchExpression = getSearchExpression();
Assert.assertEquals(expectedSearchExpression, searchExpression.toString());
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java
index 578f89f..4d2b560 100644
--- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java
+++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java
@@ -184,31 +184,36 @@ public class SearchParserTest extends SearchParser {
@Test
public void doubleAnd() throws Exception {
- runEx(SearchParserException.MessageKeys.INVALID_OPERATOR_AFTER_AND, Token.WORD, Token.AND, Token.AND, Token.WORD);
+ runEx(SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.WORD, Token.AND, Token.AND, Token.WORD);
}
@Test
public void invalidQueryEnds() {
- runEx(MessageKeys.INVALID_END_OF_QUERY, Token.WORD, Token.AND);
- runEx(MessageKeys.INVALID_END_OF_QUERY, Token.WORD, Token.OR);
- runEx(MessageKeys.INVALID_END_OF_QUERY, Token.NOT, Token.WORD, Token.OR);
- runEx(MessageKeys.INVALID_END_OF_QUERY, Token.NOT, Token.WORD, Token.AND);
- runEx(MessageKeys.INVALID_END_OF_QUERY_TOKEN_LEFT, Token.WORD, Token.AND, Token.WORD, Token.CLOSE);
+ runEx(MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.WORD, Token.AND);
+ runEx(MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.WORD, Token.OR);
+ runEx(MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.NOT, Token.WORD, Token.OR);
+ runEx(MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.NOT, Token.WORD, Token.AND);
+ runEx(MessageKeys.INVALID_END_OF_QUERY, Token.WORD, Token.AND, Token.WORD, Token.CLOSE);
}
-
+
+ @Test
+ public void invalidQueryStarts() throws Exception {
+ run(Token.WORD, Token.AND, Token.WORD, Token.AND, Token.WORD);
+ }
+
@Test
public void singleAnd() {
- runEx(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION, Token.AND);
+ runEx(SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.AND);
}
@Test
public void singleOpenBracket() {
- runEx(SearchParserException.MessageKeys.NO_EXPRESSION_FOUND, Token.OPEN);
+ runEx(SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.OPEN);
}
@Test
public void emptyBrackets() {
- runEx(SearchParserException.MessageKeys.NO_EXPRESSION_FOUND, Token.OPEN, Token.CLOSE);
+ runEx(SearchParserException.MessageKeys.EXPECTED_DIFFERENT_TOKEN, Token.OPEN, Token.CLOSE);
}
@Test
@@ -225,7 +230,7 @@ public class SearchParserTest extends SearchParser {
assertEquals(key, e.getMessageKey());
}
}
-
+
private SearchExpression run(SearchQueryToken.Token... tokenArray) throws SearchParserException {
List<SearchQueryToken> tokenList = prepareTokens(tokenArray);
SearchExpression se = parse(tokenList);
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/6dd0a0f3/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
index abf06b7..2340f37 100644
--- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
+++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
@@ -368,7 +368,6 @@ public class SearchTokenizerTest {
this.searchQuery = searchQuery;
}
- @SuppressWarnings("unused")
private Validator enableLogging() {
log = true;
return this;
[2/2] olingo-odata4 git commit: [OLINGO-568] Added support for escape
of escape and quote characters
Posted by mi...@apache.org.
[OLINGO-568] Added support for escape of escape and quote characters
Project: http://git-wip-us.apache.org/repos/asf/olingo-odata4/repo
Commit: http://git-wip-us.apache.org/repos/asf/olingo-odata4/commit/e5ac5907
Tree: http://git-wip-us.apache.org/repos/asf/olingo-odata4/tree/e5ac5907
Diff: http://git-wip-us.apache.org/repos/asf/olingo-odata4/diff/e5ac5907
Branch: refs/heads/master
Commit: e5ac590794148f5195c28cb055f26b7b4b3f5027
Parents: 6dd0a0f
Author: Michael Bolz <mi...@sap.com>
Authored: Mon Nov 30 15:15:00 2015 +0100
Committer: Michael Bolz <mi...@sap.com>
Committed: Mon Nov 30 15:15:00 2015 +0100
----------------------------------------------------------------------
.../core/uri/parser/search/SearchTokenizer.java | 189 +++++++++++--------
.../uri/parser/search/SearchTokenizerTest.java | 8 +
.../core/uri/antlr/TestFullResourcePath.java | 16 +-
.../core/uri/testutil/TestUriValidator.java | 5 +
4 files changed, 137 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/e5ac5907/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java
index 5c42e6d..2146438 100644
--- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java
+++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java
@@ -37,6 +37,13 @@ import java.util.List;
* searchWord = 1*ALPHA ; Actually: any character from the Unicode categories L or Nl,
* ; but not the words AND, OR, and NOT
* </code>
+ *
+ * <b>ATTENTION:</b> For a <code>searchPhrase</code> the percent encoding is not supported by the
+ * <code>SearchTokenizer</code>.<br/>
+ * This was a decision based on that the <code>org.apache.olingo.server.core.uri.parser.Parser</code>
+ * already handles in his <code>parseUri</code> method each query as <code>percent decoded</code> strings (see
+ * line <i>177ff</i> (<code>for (RawUri.QueryOption option : uri.queryOptionListDecoded)</code>).
+ *
*/
public class SearchTokenizer {
@@ -45,6 +52,7 @@ public class SearchTokenizer {
private boolean finished = false;
protected static final char QUOTATION_MARK = '\"';
+ protected static final char PHRASE_ESCAPE_CHAR = '\\';
protected static final char CHAR_N = 'N';
protected static final char CHAR_O = 'O';
protected static final char CHAR_T = 'T';
@@ -126,45 +134,59 @@ public class SearchTokenizer {
}
/**
- * searchPhrase = quotation-mark 1*qchar-no-AMP-DQUOTE quotation-mark
- *
- * qchar-no-AMP-DQUOTE = qchar-unescaped / escape ( escape / quotation-mark )
- *
- * qchar-unescaped = unreserved / pct-encoded-unescaped / other-delims / ":" / "@" / "/" / "?" / "$" / "'" / "="
- *
- * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
- *
- * escape = "\" / "%5C" ; reverse solidus U+005C
- *
- * pct-encoded-unescaped = "%" ( "0" / "1" / "3" / "4" / "6" / "7" / "8" / "9" / A-to-F ) HEXDIG
+ * <code>
+ * <b>searchPhrase</b> = quotation-mark 1*qchar-no-AMP-DQUOTE quotation-mark
+ * <br/><br/>
+ * <b>qchar-no-AMP-DQUOTE</b> = qchar-unescaped / escape ( escape / quotation-mark )
+ * <br/><br/>
+ * <b>qchar-unescaped</b> = unreserved / pct-encoded-unescaped / other-delims /
+ * ":" / "@" / "/" / "?" / "$" / "'" / "="
+ * <br/><br/>
+ * <b>unreserved</b> = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ * <br/><br/>
+ * <b>escape</b> = "\" / "%5C" ; reverse solidus U+005C
+ * <br/><br/>
+ * <b>pct-encoded-unescaped</b> = "%" ( "0" / "1" / "3" / "4" / "6" / "7" / "8" / "9" / A-to-F ) HEXDIG
* / "%" "2" ( "0" / "1" / "3" / "4" / "5" / "6" / "7" / "8" / "9" / A-to-F )
* / "%" "5" ( DIGIT / "A" / "B" / "D" / "E" / "F" )
+ * <br/><br/>
+ * <b>other-delims</b> = "!" / "(" / ")" / "*" / "+" / "," / ";"
+ * <br/><br/>
+ * <b>quotation-mark</b> = DQUOTE / "%22"
+ * <br/><br/>
+ * <b>ALPHA</b> = %x41-5A / %x61-7A
+ * <br/>
+ * <b>DIGIT</b> = %x30-39
+ * <br/>
+ * <b>DQUOTE</b> = %x22
+ * </code>
*
- * other-delims = "!" / "(" / ")" / "*" / "+" / "," / ";"
- *
- * quotation-mark = DQUOTE / "%22"
- *
- * ALPHA = %x41-5A / %x61-7A
- * DIGIT = %x30-39
- * DQUOTE = %x22
+ * Checks if given <code>character</code> is allowed for a search phrase.
+ * <b>ATTENTION:</b> Escaping and percent encoding is not be validated here (and can not be validated on
+ * a single character).<br/>
+ * Hence for the {@link #PHRASE_ESCAPE_CHAR} and the {@link #QUOTATION_MARK} characters this method will
+ * return <code>FALSE</code>.<br/>
+ * <b>Furthermore</b> percent encoded characters are also not validated (and can not be validated on
+ * a single character).<br/>
+ * Hence for the <code>%</code> character this method will return <code>FALSE</code>.<br/>
*
* @param character which is checked
* @return true if character is allowed for a phrase
*/
static boolean isAllowedPhrase(final char character) {
// FIXME mibo: check missing
- return isQCharUnescaped(character) || isEscaped(character);
+ return isQCharUnescaped(character);// || isEscaped(character);
}
- /**
- * escape = "\" / "%5C" ; reverse solidus U+005C
- * @param character which is checked
- * @return true if character is allowed
- */
- private static boolean isEscaped(char character) {
- // TODO: mibo(151117): check how to implement
- return false;
- }
+// /**
+// * escape = "\" / "%5C" ; reverse solidus U+005C
+// * @param character which is checked
+// * @return true if character is allowed
+// */
+// private static boolean isEscaped(char character) {
+// // TODO: mibo(151130): is checked in SearchPhraseState
+// return false;
+// }
/**
* qchar-unescaped = unreserved / pct-encoded-unescaped / other-delims / ":" / "@" / "/" / "?" / "$" / "'" / "="
@@ -173,14 +195,14 @@ public class SearchTokenizer {
*/
private static boolean isQCharUnescaped(char character) {
return isUnreserved(character)
- || isPctEncodedUnescaped(character)
- || isOtherDelims(character)
- || character == ':'
- || character == '@'
- || character == '/'
- || character == '$'
- || character == '\''
- || character == '=';
+// || isPctEncodedUnescaped(character)
+ || isOtherDelims(character)
+ || character == ':'
+ || character == '@'
+ || character == '/'
+ || character == '$'
+ || character == '\''
+ || character == '=';
}
/**
@@ -190,43 +212,43 @@ public class SearchTokenizer {
*/
private static boolean isOtherDelims(char character) {
return character == '!'
- || character == '('
- || character == ')'
- || character == '*'
- || character == '+'
- || character == ','
- || character == ';';
- }
-
- /**
- * pct-encoded-unescaped = "%" ( "0" / "1" / "3" / "4" / "6" / "7" / "8" / "9" / A-to-F ) HEXDIG
- * / "%" "2" ( "0" / "1" / "3" / "4" / "5" / "6" / "7" / "8" / "9" / A-to-F )
- * / "%" "5" ( DIGIT / "A" / "B" / "D" / "E" / "F" )
- *
- * HEXDIG = DIGIT / A-to-F
- *
- * @param character which is checked
- * @return true if character is allowed
- */
- private static boolean isPctEncodedUnescaped(char character) {
- String hex = Integer.toHexString(character);
- char aschar[] = hex.toCharArray();
- if(aschar[0] == '%') {
- if(aschar[1] == '2') {
- return aschar[2] != '2' && isHexDigit(aschar[2]);
- } else if(aschar[1] == '5') {
- return aschar[2] != 'C' && isHexDigit(aschar[2]);
- } else if(isHexDigit(aschar[1])) {
- return isHexDigit(aschar[2]);
- }
- }
- return false;
- }
-
- private static boolean isHexDigit(char character) {
- return 'A' <= character && character <= 'F' // case A..F
- || '0' <= character && character <= '9'; // case 0..9
- }
+ || character == '('
+ || character == ')'
+ || character == '*'
+ || character == '+'
+ || character == ','
+ || character == ';';
+ }
+
+// /**
+// * pct-encoded-unescaped = "%" ( "0" / "1" / "3" / "4" / "6" / "7" / "8" / "9" / A-to-F ) HEXDIG
+// * / "%" "2" ( "0" / "1" / "3" / "4" / "5" / "6" / "7" / "8" / "9" / A-to-F )
+// * / "%" "5" ( DIGIT / "A" / "B" / "D" / "E" / "F" )
+// *
+// * HEXDIG = DIGIT / A-to-F
+// *
+// * @param character which is checked
+// * @return true if character is allowed
+// */
+// private static boolean isPctEncodedUnescaped(char character) {
+// String hex = Integer.toHexString(character);
+// char aschar[] = hex.toCharArray();
+// if(aschar[0] == '%') {
+// if(aschar[1] == '2') {
+// return aschar[2] != '2' && isHexDigit(aschar[2]);
+// } else if(aschar[1] == '5') {
+// return aschar[2] != 'C' && isHexDigit(aschar[2]);
+// } else if(isHexDigit(aschar[1])) {
+// return isHexDigit(aschar[2]);
+// }
+// }
+// return false;
+// }
+
+// private static boolean isHexDigit(char character) {
+// return 'A' <= character && character <= 'F' // case A..F
+// || '0' <= character && character <= '9'; // case 0..9
+// }
/**
* unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
@@ -235,10 +257,10 @@ public class SearchTokenizer {
*/
private static boolean isUnreserved(char character) {
return isAlphaOrDigit(character)
- || character == '-'
- || character == '.'
- || character == '_'
- || character == '~';
+ || character == '-'
+ || character == '.'
+ || character == '_'
+ || character == '~';
}
/**
@@ -256,8 +278,6 @@ public class SearchTokenizer {
// BWS = *( SP / HTAB / "%20" / "%09" ) ; "bad" whitespace
// RWS = 1*( SP / HTAB / "%20" / "%09" ) ; "required" whitespace
static boolean isWhitespace(final char character) {
- // ( SP / HTAB / "%20" / "%09" )
- // TODO mibo: add missing whitespaces
return character == ' ' || character == '\t';
}
@@ -400,6 +420,7 @@ public class SearchTokenizer {
private class SearchPhraseState extends LiteralState {
private boolean closed = false;
+ private boolean escaped = false;
public SearchPhraseState(char c) throws SearchTokenizerException {
super(Token.PHRASE, c);
if (c != QUOTATION_MARK) {
@@ -416,6 +437,16 @@ public class SearchTokenizer {
} else if (isWhitespace(c)) {
return new RwsState();
}
+ } else if(escaped) {
+ escaped = false;
+ if(c == QUOTATION_MARK || c == PHRASE_ESCAPE_CHAR) {
+ return allowed(c);
+ } else {
+ return forbidden(c);
+ }
+ } else if(c == PHRASE_ESCAPE_CHAR) {
+ escaped = true;
+ return this;
} else if (isAllowedPhrase(c)) {
return allowed(c);
} else if (isWhitespace(c)) {
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/e5ac5907/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
----------------------------------------------------------------------
diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
index 2340f37..46c9290 100644
--- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
+++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java
@@ -250,6 +250,14 @@ public class SearchTokenizerTest {
@Test
public void characterInPhrase() throws Exception {
assertQuery("\"123\" OR \"ALPHA-._~\"").resultsIn(PHRASE, OR, PHRASE);
+ //escaped characters
+ assertQuery("\"\\\"123\" OR \"\\\\abc\"").resultsIn(new Validator.Tuple(PHRASE, "\"\"123\""),
+ new Validator.Tuple(OR), new Validator.Tuple(PHRASE, "\"\\abc\""));
+ assertQuery("\"\\\"1\\\\23\"").resultsIn(new Validator.Tuple(PHRASE, "\"\"1\\23\""));
+ // exceptions
+ assertQuery("\"\\\"1\\\\").resultsIn(SearchTokenizerException.MessageKeys.INVALID_TOKEN_STATE);
+ assertQuery("\"1\\\"").resultsIn(SearchTokenizerException.MessageKeys.INVALID_TOKEN_STATE);
+ assertQuery("\"1\\23\"").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER);
}
@Test
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/e5ac5907/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java
----------------------------------------------------------------------
diff --git a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java
index 9f66d66..3c02003 100644
--- a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java
+++ b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java
@@ -45,6 +45,7 @@ import org.apache.olingo.server.core.uri.parser.UriParserException;
import org.apache.olingo.server.core.uri.parser.UriParserSemanticException;
import org.apache.olingo.server.core.uri.parser.UriParserSemanticException.MessageKeys;
import org.apache.olingo.server.core.uri.parser.UriParserSyntaxException;
+import org.apache.olingo.server.core.uri.parser.search.SearchParserException;
import org.apache.olingo.server.core.uri.testutil.FilterValidator;
import org.apache.olingo.server.core.uri.testutil.TestUriValidator;
import org.apache.olingo.server.core.uri.validator.UriValidationException;
@@ -5428,9 +5429,7 @@ public class TestFullResourcePath {
}
@Test
- @Ignore("$search currently not implemented")
public void testSearch() throws Exception {
-
testUri.run("ESTwoKeyNav", "$search=abc");
testUri.run("ESTwoKeyNav", "$search=NOT abc");
@@ -5462,6 +5461,19 @@ public class TestFullResourcePath {
testUri.run("ESTwoKeyNav", "$search=(abc AND def) ghi ");
testUri.run("ESTwoKeyNav", "$search=abc AND (def OR ghi)");
testUri.run("ESTwoKeyNav", "$search=abc AND (def ghi)");
+
+ // escaped characters
+ testUri.run("ESTwoKeyNav", "$search=\"abc\"");
+ testUri.run("ESTwoKeyNav", "$search=\"a\\\"bc\"");
+ testUri.run("ESTwoKeyNav", "$search=%22abc%22");
+ testUri.run("ESTwoKeyNav", "$search=%22a%5C%22bc%22");
+ testUri.run("ESTwoKeyNav", "$search=%22a%5C%5Cbc%22");
+
+ // wrong escaped characters
+ testUri.runEx("ESTwoKeyNav", "$search=%22a%22bc%22")
+ .isExceptionMessage(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
+ testUri.runEx("ESTwoKeyNav", "$search=%22a%5Cbc%22")
+ .isExceptionMessage(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
}
@Test
http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/e5ac5907/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/testutil/TestUriValidator.java
----------------------------------------------------------------------
diff --git a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/testutil/TestUriValidator.java b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/testutil/TestUriValidator.java
index 6a2e5b4..0d5fb4a 100644
--- a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/testutil/TestUriValidator.java
+++ b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/testutil/TestUriValidator.java
@@ -176,6 +176,11 @@ public class TestUriValidator implements TestValidator {
}
}
+ public TestUriValidator isExceptionMessage(final ODataLibraryException.MessageKey messageKey) {
+ assertEquals(messageKey, exception.getMessageKey());
+ return this;
+ }
+
public TestUriValidator isExSyntax(final UriParserSyntaxException.MessageKeys messageKey) {
assertEquals(UriParserSyntaxException.class, exception.getClass());
assertEquals(messageKey, exception.getMessageKey());