You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by oz...@apache.org on 2016/11/17 12:28:41 UTC
nifi git commit: NIFI-2950 Adding support for whole number hex values
and a fromRadix function
Repository: nifi
Updated Branches:
refs/heads/master e5eda6370 -> c4be80068
NIFI-2950 Adding support for whole number hex values and a fromRadix function
NIFI-2950 Fixing typo
This closes #1161
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c4be8006
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c4be8006
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c4be8006
Branch: refs/heads/master
Commit: c4be800688bf23a3bdea8def75b84c0f4ded243d
Parents: e5eda63
Author: jpercivall <jo...@yahoo.com>
Authored: Wed Oct 26 16:34:14 2016 -0400
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Thu Nov 17 07:27:51 2016 -0500
----------------------------------------------------------------------
.../language/antlr/AttributeExpressionLexer.g | 1 +
.../language/antlr/AttributeExpressionParser.g | 2 +-
.../attribute/expression/language/Query.java | 6 ++
.../evaluation/cast/DecimalCastEvaluator.java | 8 ++-
.../evaluation/cast/NumberCastEvaluator.java | 10 +++-
.../cast/WholeNumberCastEvaluator.java | 13 ++++-
.../functions/FromRadixEvaluator.java | 58 ++++++++++++++++++++
.../language/evaluation/util/NumberParsing.java | 3 +-
.../expression/language/TestQuery.java | 19 ++++++-
.../asciidoc/expression-language-guide.adoc | 32 +++++++++++
10 files changed, 141 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
index cf76808..9375579 100644
--- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
@@ -167,6 +167,7 @@ MINUS : 'minus';
MULTIPLY : 'multiply';
DIVIDE : 'divide';
MATH : 'math';
+FROM_RADIX : 'fromRadix';
TO_RADIX : 'toRadix';
OR : 'or';
AND : 'and';
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
index 5542fb0..eb50a28 100644
--- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
+++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
@@ -75,7 +75,7 @@ tokens {
// functions that return Strings
zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
- PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH) LPAREN! anyArg RPAREN!) |
+ PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
(SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
index 49c7518..fb48b0f 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
@@ -47,6 +47,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Equals
import org.apache.nifi.attribute.expression.language.evaluation.functions.CharSequenceTranslatorEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.FindEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.FormatEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.FromRadixEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GetDelimitedFieldEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanOrEqualEvaluator;
@@ -123,6 +124,7 @@ import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.tree.Tree;
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES;
@@ -1241,6 +1243,10 @@ public class Query {
toWholeNumberEvaluator(argEvaluators.get(0)), toWholeNumberEvaluator(argEvaluators.get(1))), "toRadix");
}
}
+ case FROM_RADIX: {
+ return addToken(new FromRadixEvaluator(toStringEvaluator(subjectEvaluator),
+ toWholeNumberEvaluator(argEvaluators.get(0))), "fromRadix");
+ }
case MOD: {
return addToken(new ModEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "mod");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java
index 9418028..bce54f7 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java
@@ -58,7 +58,13 @@ public class DecimalCastEvaluator extends DecimalEvaluator {
case DECIMAL:
return new DecimalQueryResult(Double.valueOf(trimmed));
case WHOLE_NUMBER:
- final Long resultValue = Long.valueOf(trimmed);
+ Long resultValue;
+ try {
+ resultValue = Long.valueOf(trimmed);
+ } catch (NumberFormatException e){
+ // Will only occur if trimmed is a hex number
+ resultValue = Long.decode(trimmed);
+ }
return new DecimalQueryResult(resultValue.doubleValue());
case NOT_NUMBER:
default:
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
index 4073dc4..00c9ef7 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
@@ -64,8 +64,14 @@ public class NumberCastEvaluator extends NumberEvaluator {
case DECIMAL:
return new NumberQueryResult(Double.valueOf(trimmed));
case WHOLE_NUMBER:
- final Long resultValue = Long.valueOf(trimmed);
- return new NumberQueryResult(Long.valueOf(trimmed));
+ Long resultValue;
+ try {
+ resultValue = Long.valueOf(trimmed);
+ } catch (NumberFormatException e){
+ // Will only occur if trimmed is a hex number
+ resultValue = Long.decode(trimmed);
+ }
+ return new NumberQueryResult(resultValue);
case NOT_NUMBER:
default:
return new NumberQueryResult(null);
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java
index 024fb44..736bf2c 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java
@@ -56,10 +56,17 @@ public class WholeNumberCastEvaluator extends WholeNumberEvaluator {
NumberParsing.ParseResultType parseType = NumberParsing.parse(trimmed);
switch (parseType){
case DECIMAL:
- final Double resultValue = Double.valueOf(trimmed);
- return new WholeNumberQueryResult(resultValue.longValue());
+ final Double doubleResultValue = Double.valueOf(trimmed);
+ return new WholeNumberQueryResult(doubleResultValue.longValue());
case WHOLE_NUMBER:
- return new WholeNumberQueryResult(Long.valueOf(trimmed));
+ Long longResultValue;
+ try {
+ longResultValue = Long.valueOf(trimmed);
+ } catch (NumberFormatException e){
+ // Will only occur if trimmed is a hex number
+ longResultValue = Long.decode(trimmed);
+ }
+ return new WholeNumberQueryResult(longResultValue);
case NOT_NUMBER:
default:
return new WholeNumberQueryResult(null);
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java
new file mode 100644
index 0000000..8febd2e
--- /dev/null
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberQueryResult;
+
+import java.util.Map;
+
+public class FromRadixEvaluator extends WholeNumberEvaluator {
+
+ private final Evaluator<String> numberEvaluator;
+ private final Evaluator<Long> radixEvaluator;
+
+ public FromRadixEvaluator(final Evaluator<String> subject, final Evaluator<Long> radixEvaluator) {
+ this.numberEvaluator = subject;
+ this.radixEvaluator = radixEvaluator;
+ }
+
+ @Override
+ public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+ final String result = numberEvaluator.evaluate(attributes).getValue();
+ if (result == null) {
+ return new WholeNumberQueryResult(null);
+ }
+
+ final Long radix = radixEvaluator.evaluate(attributes).getValue();
+ if (radix == null) {
+ return new WholeNumberQueryResult(null);
+ }
+
+ long longValue = Long.parseLong(result, radix.intValue());
+
+ return new WholeNumberQueryResult(longValue);
+ }
+
+ @Override
+ public Evaluator<?> getSubjectEvaluator() {
+ return numberEvaluator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java
index c79c922..bbfd4e2 100644
--- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java
+++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java
@@ -21,7 +21,6 @@ import java.util.regex.Pattern;
public class NumberParsing {
- private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
public static enum ParseResultType {
NOT_NUMBER, WHOLE_NUMBER, DECIMAL;
@@ -69,6 +68,8 @@ public class NumberParsing {
private static final Pattern DOUBLE_PATTERN = Pattern.compile(fpRegex);
+ private static final Pattern NUMBER_PATTERN = Pattern.compile("-?((\\d+)|(0[xX]" + HexDigits + "))");
+
private NumberParsing(){
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
index f124f76..3b6896c 100644
--- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
+++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
@@ -1057,11 +1057,14 @@ public class TestQuery {
verifyEquals("${literal(5):toNumber()}", attributes, 5L);
verifyEquals("${literal(5):toDecimal()}", attributes, 5D);
- // Unquoted doubles are not due to more complicated parsing
verifyEquals("${literal(\"5.5\")}", attributes, "5.5");
- verifyEquals("${literal(\"5.5\"):toNumber()}", attributes, 5L);
- verifyEquals("${literal(\"5.5\"):toDecimal()}", attributes, 5.5D);
+ verifyEquals("${literal(5.5):toNumber()}", attributes, 5L);
+ verifyEquals("${literal(5.5):toDecimal()}", attributes, 5.5D);
+ verifyEquals("${literal('0xF.Fp10'):toDecimal()}", attributes, 0xF.Fp10D);
+
+ verifyEquals("${literal('0xABC'):toNumber()}", attributes, 0xABCL);
+ verifyEquals("${literal('-0xABC'):toNumber()}", attributes, -0xABCL);
}
@Test
@@ -1176,6 +1179,16 @@ public class TestQuery {
}
@Test
+ public void testFromRadix() {
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("test1", "ABCDEF");
+ attributes.put("test2", "123");
+
+ verifyEquals("${test1:fromRadix(16)}", attributes, 0xABCDEFL);
+ verifyEquals("${test2:fromRadix(4)}", attributes, 27L);
+ }
+
+ @Test
public void testBase64Encode(){
final Map<String, String> attributes = new HashMap<>();
attributes.put("userpass", "admin:admin");
http://git-wip-us.apache.org/repos/asf/nifi/blob/c4be8006/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
index 90205e8..c5e2a77 100644
--- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc
@@ -213,7 +213,11 @@ The Expression Language is generally able to automatically coerce a value of one
data type for a function. However, functions do exist to manually coerce a value into a specific data type.
See the <<type_cast>> section for more information.
+Hex values are supported for Number and Decimal types but they must be quoted and prepended with "0x" when being
+interpreted as literals. For example these two expressions are valid (without the quotes or "0x" the expression would fail to run properly):
+ - ${literal("0xF"):toNumber()}
+ - ${literal("0xF.Fp10"):toDecimal()}
@@ -1645,6 +1649,34 @@ Divide. This is to preserve backwards compatibility and to not force rounding er
|=======================================================================================
[.function]
+=== fromRadix
+
+*Description*: [.description]#Converts the Subject from a specified Radix (or number base) to a base ten whole number. The subject will converted as is, without interpretation, and all characters
+must be valid for the base being converted from. For example converting "0xFF" from hex will not work due to "x" being a invalid hex character.
+
+ If a decimal is passed as the subject, it will first be converted to a whole number and then processed.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+ - [.argName]#_Subject Base_# : [.argDesc]#A Number between 2 and 36 (inclusive)#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attributes has a value of 1234A, then the following Expressions will yield
+ the following results:
+
+
+.toRadix Examples
+|=======================================================================================
+| Expression | Value
+| `${fileSize:fromRadix(11)}` | `17720`
+| `${fileSize:fromRadix(16)}` | `74570`
+| `${fileSize:fromRadix(20)}` | `177290`
+|=======================================================================================
+
+[.function]
=== random
*Description*: [.description]#Returns a random whole number ( 0 to 2^63 - 1) using an insecure random number generator.#