You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/01/22 18:24:13 UTC
[2/2] ambari git commit: AMBARI-8605 Query predicate .matches doesn't
work for stacks endpoint with passed logical OR (dsen)
AMBARI-8605 Query predicate .matches doesn't work for stacks endpoint with passed logical OR (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f77be4c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f77be4c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f77be4c4
Branch: refs/heads/trunk
Commit: f77be4c4df25aef336f00a7aa9bc0591b347dfb6
Parents: 9f296a2
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Jan 22 15:13:08 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Jan 22 19:23:35 2015 +0200
----------------------------------------------------------------------
.../ambari/server/api/predicate/QueryLexer.java | 195 +++++++++++++++++--
.../server/api/predicate/QueryLexerTest.java | 51 ++++-
2 files changed, 231 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f77be4c4/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java
index b645040..e7051a1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/predicate/QueryLexer.java
@@ -62,6 +62,7 @@ public class QueryLexer {
*/
private static final Set<String> SET_IGNORE = new HashSet<String>();
+
/**
* Constructor.
* Register token handlers.
@@ -88,14 +89,18 @@ public class QueryLexer {
listHandlers = new ArrayList<TokenHandler>();
listHandlers.add(new CloseBracketTokenHandler());
- listHandlers.add(new ValueOperandTokenHandler());
+ listHandlers.add(new ComplexValueOperandTokenHandler());
TOKEN_HANDLERS.put(Token.TYPE.RELATIONAL_OPERATOR_FUNC, listHandlers);
listHandlers = new ArrayList<TokenHandler>();
listHandlers.add(new CloseBracketTokenHandler());
listHandlers.add(new LogicalOperatorTokenHandler());
- TOKEN_HANDLERS.put(Token.TYPE.VALUE_OPERAND, listHandlers);
TOKEN_HANDLERS.put(Token.TYPE.BRACKET_CLOSE, listHandlers);
+
+ listHandlers = new ArrayList<TokenHandler>(listHandlers);
+ // complex value operands can span multiple tokens
+ listHandlers.add(0, new ComplexValueOperandTokenHandler());
+ TOKEN_HANDLERS.put(Token.TYPE.VALUE_OPERAND, listHandlers);
}
@@ -139,6 +144,9 @@ public class QueryLexer {
tok + "\', previous token type=" + ctx.getLastTokenType());
}
}
+
+ ctx.validateEndState();
+
return ctx.getTokenList().toArray(new Token[ctx.getTokenList().size()]);
}
@@ -234,6 +242,23 @@ public class QueryLexer {
private Set<String> m_propertiesToIgnore = new HashSet<String>();
/**
+ * Bracket score. This score is the difference between the number of
+ * opening brackets and the number of closing brackets processed by
+ * a handler. Only handlers which process values containing brackets
+ * will be interested in this information.
+ */
+ private int bracketScore = 0;
+
+ /**
+ * Intermediate tokens are tokens which are used by a handler which may
+ * process several adjacent tokens. A handler might push intermediate
+ * tokens and then in subsequent invocations combine/alter/remove/etc
+ * these tokens prior to adding them to the context tokens.
+ */
+ private Deque<Token> m_intermediateTokens = new ArrayDeque<Token>();
+
+
+ /**
* Constructor.
*/
private ScanContext() {
@@ -329,6 +354,86 @@ public class QueryLexer {
m_propertiesToIgnore.addAll(ignoredProperties);
}
}
+
+ /**
+ * Add an intermediate token.
+ *
+ * @param token the token to add
+ */
+ public void pushIntermediateToken(Token token) {
+ if (m_ignoreSegmentEndToken == null) {
+ m_intermediateTokens.add(token);
+ } else if (token.getType() == m_ignoreSegmentEndToken) {
+ m_ignoreSegmentEndToken = null;
+ }
+ }
+
+ /**
+ * Return the intermediate tokens if any.
+ *
+ * @return the intermediate tokens. Will never return null.
+ */
+ public Deque<Token> getIntermediateTokens() {
+ return m_intermediateTokens;
+ }
+
+ /**
+ * Move all intermediate tokens to the context tokens.
+ */
+ public void addIntermediateTokens() {
+ m_listTokens.addAll(m_intermediateTokens);
+ m_intermediateTokens.clear();
+ }
+
+ /**
+ * Obtain the bracket score. This count is the number of outstanding opening brackets.
+ * A value of 0 indicates all opening and closing brackets are matched
+ * @return the current bracket score
+ */
+ public int getBracketScore() {
+ return bracketScore;
+ }
+
+ /**
+ * Increment the bracket score by n. This indicates that n unmatched opening brackets
+ * have been encountered.
+ *
+ * @param n amount to increment
+ * @return the new bracket score after incrementing
+ */
+ public int incrementBracketScore(int n) {
+ return bracketScore += n;
+ }
+
+ /**
+ * Decrement the bracket score. This is done when matching a closing bracket with a previously encountered
+ * opening bracket. If the requested decrement would result in a negative number an exception is thrown
+ * as this isn't a valid state.
+ *
+ * @param decValue amount to decrement
+ * @return the new bracket score after decrementing
+ * @throws InvalidQueryException if the decrement operation will result in a negative value
+ */
+ public int decrementBracketScore(int decValue) throws InvalidQueryException {
+ bracketScore -= decValue;
+ if (bracketScore < 0) {
+ throw new InvalidQueryException("Unexpected closing bracket. Last token type: " + getLastTokenType() +
+ ", Current property operand: " + getPropertyOperand() + ", tokens: " + getTokenList());
+ }
+ return bracketScore;
+ }
+
+ //todo: most handlers should implement this
+ /**
+ * Validate the end state of the scan context.
+ * Iterates over each handler associated with the final token type and asks it to validate the context.
+ * @throws InvalidQueryException if the context is determined to in an invalid end state
+ */
+ public void validateEndState() throws InvalidQueryException {
+ for (TokenHandler handler : TOKEN_HANDLERS.get(getLastTokenType())) {
+ handler.validateEndState(this);
+ }
+ }
}
/**
@@ -346,7 +451,7 @@ public class QueryLexer {
* @throws InvalidQueryException if an invalid token is encountered
*/
public boolean handleToken(String token, ScanContext ctx) throws InvalidQueryException {
- if (handles(token, ctx.getLastTokenType())) {
+ if (handles(token, ctx)) {
_handleToken(token, ctx);
ctx.setLastTokenType(getType());
return true;
@@ -355,6 +460,12 @@ public class QueryLexer {
}
}
+ public void validateEndState(ScanContext ctx) throws InvalidQueryException {
+ if (! ctx.getIntermediateTokens().isEmpty()) {
+ throw new InvalidQueryException("Unexpected end of expression.");
+ }
+ }
+
/**
* Process a token.
*
@@ -374,12 +485,12 @@ public class QueryLexer {
/**
* Determine if a handler handles a specific token type.
*
- * @param token the token type
- * @param previousTokenType the previous token type
*
+ * @param token the token type
+ * @param ctx scan context
* @return true if the handler handles the specified type; false otherwise
*/
- public abstract boolean handles(String token, Token.TYPE previousTokenType);
+ public abstract boolean handles(String token, ScanContext ctx);
}
/**
@@ -411,7 +522,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("[^!&\\|<=|>=|!=|=|<|>\\(\\)]+");
}
}
@@ -431,7 +542,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("[^!&\\|<=|>=|!=|=|<|>]+");
}
}
@@ -451,7 +562,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("\\(");
}
}
@@ -471,7 +582,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("\\)");
}
}
@@ -492,7 +603,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("<=|>=|!=|=|<|>");
}
}
@@ -514,11 +625,67 @@ public class QueryLexer {
//todo: add a unary relational operator func
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("\\.[a-zA-Z]+\\(");
}
}
+ /**
+ * Complex Value Operand token handler.
+ * Supports values that span multiple tokens.
+ */
+ private class ComplexValueOperandTokenHandler extends TokenHandler {
+ @Override
+ public void _handleToken(String token, ScanContext ctx) throws InvalidQueryException {
+ if (token.equals(")")) {
+ ctx.decrementBracketScore(1);
+ } else if (token.endsWith("(")) {
+ // .endsWith() is used because of tokens ".matches(",".in(" and".isEmpty("
+ ctx.incrementBracketScore(1);
+ }
+
+ String tokenValue = token;
+ if (ctx.getBracketScore() > 0) {
+ Deque<Token> intermediateTokens = ctx.getIntermediateTokens();
+ if (! intermediateTokens.isEmpty()) {
+ Token lastToken = intermediateTokens.peek();
+ if (lastToken.getType() == Token.TYPE.VALUE_OPERAND) {
+ intermediateTokens.pop();
+ tokenValue = lastToken.getValue() + token;
+ }
+ }
+ ctx.pushIntermediateToken(new Token(Token.TYPE.VALUE_OPERAND, tokenValue));
+ }
+
+ if (ctx.getBracketScore() == 0) {
+ ctx.addIntermediateTokens();
+ ctx.addToken(new Token(Token.TYPE.BRACKET_CLOSE, ")"));
+ }
+ }
+
+ @Override
+ public Token.TYPE getType() {
+ return Token.TYPE.VALUE_OPERAND;
+ }
+
+ @Override
+ public boolean handles(String token, ScanContext ctx) {
+ Token.TYPE lastTokenType = ctx.getLastTokenType();
+ if (lastTokenType == Token.TYPE.RELATIONAL_OPERATOR_FUNC) {
+ ctx.incrementBracketScore(1);
+ return true;
+ } else {
+ return ctx.getBracketScore() > 0;
+ }
+ }
+
+ @Override
+ public void validateEndState(ScanContext ctx) throws InvalidQueryException {
+ if (ctx.getBracketScore() > 0) {
+ throw new InvalidQueryException("Missing closing bracket for function: " + ctx.getTokenList());
+ }
+ }
+ }
/**
* Logical Operator token handler.
@@ -535,7 +702,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return token.matches("[!&\\|]");
}
}
@@ -555,7 +722,7 @@ public class QueryLexer {
}
@Override
- public boolean handles(String token, Token.TYPE previousTokenType) {
+ public boolean handles(String token, ScanContext ctx) {
return "!".equals(token);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f77be4c4/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java
index 2c04d6c..8caa821 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/predicate/QueryLexerTest.java
@@ -22,7 +22,6 @@ package org.apache.ambari.server.api.predicate;
import org.junit.Test;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -360,4 +359,54 @@ public class QueryLexerTest {
//expected
}
}
+
+ @Test
+ public void testTokens_matchesRegexp_simple() throws InvalidQueryException {
+ List<Token> listTokens = new ArrayList<Token>();
+ listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches("));
+ listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type"));
+ listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "(.*USER.*)|(.*GROUP.*)"));
+ listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")"));
+
+ QueryLexer lexer = new QueryLexer();
+ Token[] tokens = lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*))");
+
+ assertArrayEquals(listTokens.toArray(new Token[listTokens.size()]), tokens);
+ }
+
+ @Test
+ public void testTokens_matchesRegexp() throws InvalidQueryException {
+ List<Token> listTokens = new ArrayList<Token>();
+ listTokens.add(new Token(Token.TYPE.BRACKET_OPEN, "("));
+ listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches("));
+ listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type"));
+ listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "(([^=])|=|!=),.in(&).*USER.*.isEmpty(a).matches(b)"));
+ listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")"));
+ listTokens.add(new Token(Token.TYPE.LOGICAL_OPERATOR, "|"));
+ listTokens.add(new Token(Token.TYPE.RELATIONAL_OPERATOR_FUNC, ".matches("));
+ listTokens.add(new Token(Token.TYPE.PROPERTY_OPERAND, "StackConfigurations/property_type"));
+ listTokens.add(new Token(Token.TYPE.VALUE_OPERAND, "fields format to from .*GROUP.*"));
+ listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")"));
+ listTokens.add(new Token(Token.TYPE.BRACKET_CLOSE, ")"));
+
+ QueryLexer lexer = new QueryLexer();
+ Token[] tokens = lexer.tokens("(StackConfigurations/property_type.matches((([^=])|=|!=),.in(&).*USER.*" +
+ ".isEmpty(a).matches(b))|StackConfigurations/property_type.matches(fields format to from .*GROUP.*))");
+
+ assertArrayEquals("All characters between \".matches(\" and corresponding closing \")\" bracket should " +
+ "come to VALUE_OPERAND.", listTokens.toArray(new Token[listTokens.size()]), tokens);
+ }
+
+ @Test(expected = InvalidQueryException.class)
+ public void testTokens_matchesRegexpInvalidQuery() throws InvalidQueryException {
+ QueryLexer lexer = new QueryLexer();
+ lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*)");
+ }
+
+ @Test(expected = InvalidQueryException.class)
+ public void testTokens_matchesRegexpInvalidQuery2() throws InvalidQueryException {
+ QueryLexer lexer = new QueryLexer();
+ lexer.tokens("StackConfigurations/property_type.matches((.*USER.*)|(.*GROUP.*)|StackConfigurations/property_type.matches(.*GROUP.*)");
+ }
+
}