You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by da...@apache.org on 2015/06/10 04:40:31 UTC
[09/21] incubator-nifi git commit: NIFI-625: Fixed bug in expression
language that caused EL not to get evaluated if enclosed within curly braces
NIFI-625: Fixed bug in expression language that caused EL not to get evaluated if enclosed within curly braces
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/86cbfab1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/86cbfab1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/86cbfab1
Branch: refs/heads/NIFI-632
Commit: 86cbfab14ad13ce5adbb591deae9c8d46fd7274e
Parents: b82d142
Author: Mark Payne <ma...@hotmail.com>
Authored: Mon Jun 8 19:52:15 2015 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue Jun 9 18:32:34 2015 -0400
----------------------------------------------------------------------
.../attribute/expression/language/Query.java | 135 ++++++++++---------
.../expression/language/TestQuery.java | 54 +++++---
.../processors/standard/TestReplaceText.java | 18 +++
3 files changed, 123 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/86cbfab1/nifi/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
index 7e40897..77d08c6 100644
--- a/nifi/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
+++ b/nifi/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
@@ -244,51 +244,58 @@ public class Query {
int backslashCount = 0;
charLoop:
- for (int i = 0; i < value.length(); i++) {
- final char c = value.charAt(i);
-
- if (expressionStart > -1 && (c == '\'' || c == '"') && (lastChar != '\\' || backslashCount % 2 == 0)) {
- final int endQuoteIndex = findEndQuoteChar(value, i);
- if (endQuoteIndex < 0) {
- break charLoop;
- }
-
- i = endQuoteIndex;
- continue;
- }
+ for (int i = 0; i < value.length(); i++) {
+ final char c = value.charAt(i);
- if (c == '{') {
- if (oddDollarCount && lastChar == '$') {
- if (embeddedCount == 0) {
- expressionStart = i - 1;
+ if (expressionStart > -1 && (c == '\'' || c == '"') && (lastChar != '\\' || backslashCount % 2 == 0)) {
+ final int endQuoteIndex = findEndQuoteChar(value, i);
+ if (endQuoteIndex < 0) {
+ break charLoop;
}
- }
- embeddedCount++;
- } else if (c == '}') {
- if (embeddedCount <= 0) {
+ i = endQuoteIndex;
continue;
}
- if (--embeddedCount == 0) {
+ if (c == '{') {
+ if (oddDollarCount && lastChar == '$') {
+ if (embeddedCount == 0) {
+ expressionStart = i - 1;
+ }
+ }
+
+ // Keep track of the number of opening curly braces that we are embedded within,
+ // if we are within an Expression. If we are outside of an Expression, we can just ignore
+ // curly braces. This allows us to ignore the first character if the value is something
+ // like: { ${abc} }
+ // However, we will count the curly braces if we have something like: ${ $${abc} }
if (expressionStart > -1) {
- // ended expression. Add a new range.
- final Range range = new Range(expressionStart, i);
- ranges.add(range);
+ embeddedCount++;
+ }
+ } else if (c == '}') {
+ if (embeddedCount <= 0) {
+ continue;
}
- expressionStart = -1;
+ if (--embeddedCount == 0) {
+ if (expressionStart > -1) {
+ // ended expression. Add a new range.
+ final Range range = new Range(expressionStart, i);
+ ranges.add(range);
+ }
+
+ expressionStart = -1;
+ }
+ } else if (c == '$') {
+ oddDollarCount = !oddDollarCount;
+ } else if (c == '\\') {
+ backslashCount++;
+ } else {
+ oddDollarCount = false;
}
- } else if (c == '$') {
- oddDollarCount = !oddDollarCount;
- } else if (c == '\\') {
- backslashCount++;
- } else {
- oddDollarCount = false;
- }
- lastChar = c;
- }
+ lastChar = c;
+ }
return ranges;
}
@@ -420,7 +427,7 @@ public class Query {
}
private static Map<String, String> wrap(final Map<String, String> attributes, final Map<String, String> flowFileProps,
- final Map<String, String> env, final Map<?, ?> sysProps) {
+ final Map<String, String> env, final Map<?, ?> sysProps) {
@SuppressWarnings("rawtypes")
final Map[] maps = new Map[]{attributes, flowFileProps, env, sysProps};
@@ -947,7 +954,7 @@ public class Query {
return new BooleanCastEvaluator((StringEvaluator) evaluator);
default:
throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + evaluator.getResultType() + " to " + ResultType.BOOLEAN
- + (location == null ? "" : " at location [" + location + "]"));
+ + (location == null ? "" : " at location [" + location + "]"));
}
}
@@ -965,12 +972,12 @@ public class Query {
case NUMBER:
return (NumberEvaluator) evaluator;
case STRING:
- return new NumberCastEvaluator((StringEvaluator) evaluator);
+ return new NumberCastEvaluator(evaluator);
case DATE:
return new DateToNumberEvaluator((DateEvaluator) evaluator);
default:
throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + evaluator.getResultType() + " to " + ResultType.NUMBER
- + (location == null ? "" : " at location [" + location + "]"));
+ + (location == null ? "" : " at location [" + location + "]"));
}
}
@@ -1015,27 +1022,27 @@ public class Query {
case SUBSTRING_BEFORE: {
verifyArgCount(argEvaluators, 1, "substringBefore");
return new SubstringBeforeEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to substringBefore"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringBefore"));
}
case SUBSTRING_BEFORE_LAST: {
verifyArgCount(argEvaluators, 1, "substringBeforeLast");
return new SubstringBeforeLastEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to substringBeforeLast"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringBeforeLast"));
}
case SUBSTRING_AFTER: {
verifyArgCount(argEvaluators, 1, "substringAfter");
return new SubstringAfterEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to substringAfter"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringAfter"));
}
case SUBSTRING_AFTER_LAST: {
verifyArgCount(argEvaluators, 1, "substringAfterLast");
return new SubstringAfterLastEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to substringAfterLast"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to substringAfterLast"));
}
case REPLACE_NULL: {
verifyArgCount(argEvaluators, 1, "replaceNull");
return new ReplaceNullEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to replaceNull"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to replaceNull"));
}
case REPLACE_EMPTY: {
verifyArgCount(argEvaluators, 1, "replaceEmtpy");
@@ -1044,34 +1051,34 @@ public class Query {
case REPLACE: {
verifyArgCount(argEvaluators, 2, "replace");
return new ReplaceEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to replace"),
- toStringEvaluator(argEvaluators.get(1), "second argument to replace"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to replace"),
+ toStringEvaluator(argEvaluators.get(1), "second argument to replace"));
}
case REPLACE_ALL: {
verifyArgCount(argEvaluators, 2, "replaceAll");
return new ReplaceAllEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to replaceAll"),
- toStringEvaluator(argEvaluators.get(1), "second argument to replaceAll"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to replaceAll"),
+ toStringEvaluator(argEvaluators.get(1), "second argument to replaceAll"));
}
case APPEND: {
verifyArgCount(argEvaluators, 1, "append");
return new AppendEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to append"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to append"));
}
case PREPEND: {
verifyArgCount(argEvaluators, 1, "prepend");
return new PrependEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to prepend"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to prepend"));
}
case SUBSTRING: {
final int numArgs = argEvaluators.size();
if (numArgs == 1) {
return new SubstringEvaluator(toStringEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to substring"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to substring"));
} else if (numArgs == 2) {
return new SubstringEvaluator(toStringEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to substring"),
- toNumberEvaluator(argEvaluators.get(1), "second argument to substring"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to substring"),
+ toNumberEvaluator(argEvaluators.get(1), "second argument to substring"));
} else {
throw new AttributeExpressionLanguageParsingException("substring() function can take either 1 or 2 arguments but cannot take " + numArgs + " arguments");
}
@@ -1099,27 +1106,27 @@ public class Query {
case STARTS_WITH: {
verifyArgCount(argEvaluators, 1, "startsWith");
return new StartsWithEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to startsWith"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to startsWith"));
}
case ENDS_WITH: {
verifyArgCount(argEvaluators, 1, "endsWith");
return new EndsWithEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to endsWith"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to endsWith"));
}
case CONTAINS: {
verifyArgCount(argEvaluators, 1, "contains");
return new ContainsEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to contains"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to contains"));
}
case FIND: {
verifyArgCount(argEvaluators, 1, "find");
return new FindEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to find"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to find"));
}
case MATCHES: {
verifyArgCount(argEvaluators, 1, "matches");
return new MatchesEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to matches"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to matches"));
}
case EQUALS: {
verifyArgCount(argEvaluators, 1, "equals");
@@ -1128,27 +1135,27 @@ public class Query {
case EQUALS_IGNORE_CASE: {
verifyArgCount(argEvaluators, 1, "equalsIgnoreCase");
return new EqualsIgnoreCaseEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to equalsIgnoreCase"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to equalsIgnoreCase"));
}
case GREATER_THAN: {
verifyArgCount(argEvaluators, 1, "gt");
return new GreaterThanEvaluator(toNumberEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to gt"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to gt"));
}
case GREATER_THAN_OR_EQUAL: {
verifyArgCount(argEvaluators, 1, "ge");
return new GreaterThanOrEqualEvaluator(toNumberEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to ge"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to ge"));
}
case LESS_THAN: {
verifyArgCount(argEvaluators, 1, "lt");
return new LessThanEvaluator(toNumberEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to lt"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to lt"));
}
case LESS_THAN_OR_EQUAL: {
verifyArgCount(argEvaluators, 1, "le");
return new LessThanOrEqualEvaluator(toNumberEvaluator(subjectEvaluator),
- toNumberEvaluator(argEvaluators.get(0), "first argument to le"));
+ toNumberEvaluator(argEvaluators.get(0), "first argument to le"));
}
case LENGTH: {
verifyArgCount(argEvaluators, 0, "length");
@@ -1199,12 +1206,12 @@ public class Query {
case INDEX_OF: {
verifyArgCount(argEvaluators, 1, "indexOf");
return new IndexOfEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to indexOf"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to indexOf"));
}
case LAST_INDEX_OF: {
verifyArgCount(argEvaluators, 1, "lastIndexOf");
return new LastIndexOfEvaluator(toStringEvaluator(subjectEvaluator),
- toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf"));
+ toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf"));
}
case FORMAT: {
return new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format"));
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/86cbfab1/nifi/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
index 4bf614f..f343261 100644
--- a/nifi/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
+++ b/nifi/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
@@ -87,9 +87,9 @@ public class TestQuery {
Query.validateExpression("$${attr}", true);
Query.validateExpression("${filename:startsWith('T8MTXBC')\n"
- + ":or( ${filename:startsWith('C4QXABC')} )\n"
- + ":or( ${filename:startsWith('U6CXEBC')} )"
- + ":or( ${filename:startsWith('KYM3ABC')} )}", false);
+ + ":or( ${filename:startsWith('C4QXABC')} )\n"
+ + ":or( ${filename:startsWith('U6CXEBC')} )"
+ + ":or( ${filename:startsWith('KYM3ABC')} )}", false);
}
@Test
@@ -361,6 +361,8 @@ public class TestQuery {
@Test
public void testExtractExpressionRanges() {
+ assertEquals(29, Query.extractExpressionRanges("${hello:equals( $${goodbye} )}").get(0).getEnd());
+
List<Range> ranges = Query.extractExpressionRanges("hello");
assertTrue(ranges.isEmpty());
@@ -592,13 +594,13 @@ public class TestQuery {
attributes.put("abc", "xyz");
final String expression
- = "# hello, world\n"
- + "${# ref attr\n"
- + "\t"
- + "abc"
- + "\t"
- + "#end ref attr\n"
- + "}";
+ = "# hello, world\n"
+ + "${# ref attr\n"
+ + "\t"
+ + "abc"
+ + "\t"
+ + "#end ref attr\n"
+ + "}";
Query query = Query.compile(expression);
QueryResult<?> result = query.evaluate(attributes);
@@ -1030,16 +1032,16 @@ public class TestQuery {
attributes.put("filename 3", "abcxy");
final String query
- = "${"
- + " 'non-existing':notNull():not():and(" + // true AND (
- " ${filename1:startsWith('y')" + // false
- " :or(" + // or
- " ${ filename1:startsWith('x'):and(false) }" + // false
- " ):or(" + // or
- " ${ filename2:endsWith('xxxx'):or( ${'filename 3':length():gt(1)} ) }" + // true )
- " )}"
- + " )"
- + "}";
+ = "${"
+ + " 'non-existing':notNull():not():and(" + // true AND (
+ " ${filename1:startsWith('y')" + // false
+ " :or(" + // or
+ " ${ filename1:startsWith('x'):and(false) }" + // false
+ " ):or(" + // or
+ " ${ filename2:endsWith('xxxx'):or( ${'filename 3':length():gt(1)} ) }" + // true )
+ " )}"
+ + " )"
+ + "}";
System.out.println(query);
verifyEquals(query, attributes, true);
@@ -1098,6 +1100,18 @@ public class TestQuery {
verifyEquals(query, attributes, true);
}
+ @Test
+ public void testEvaluateWithinCurlyBraces() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("abc", "xyz");
+
+ final String query = "{ ${abc} }";
+ final List<String> expressions = Query.extractExpressions(query);
+ assertEquals(1, expressions.size());
+ assertEquals("${abc}", expressions.get(0));
+ assertEquals("{ xyz }", Query.evaluateExpressions(query, attributes));
+ }
+
private void verifyEquals(final String expression, final Map<String, String> attributes, final Object expectedResult) {
Query.validateExpression(expression, false);
assertEquals(String.valueOf(expectedResult), Query.evaluateExpressions(expression, attributes, null));
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/86cbfab1/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
----------------------------------------------------------------------
diff --git a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
index e340468..203f77a 100644
--- a/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
+++ b/nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
@@ -368,4 +368,22 @@ public class TestReplaceText {
System.out.println(outContent);
}
+ @Test
+ public void testReplaceWithinCurlyBraces() throws IOException {
+ final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(ReplaceText.REGEX, ".+");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "{ ${filename} }");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("filename", "abc.txt");
+ runner.enqueue("Hello".getBytes(), attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
+ final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
+ out.assertContentEquals("{ abc.txt }");
+ }
+
}