You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/08/30 18:27:22 UTC
nifi git commit: NIFI-1923 - AttributesToJson regex property
Repository: nifi
Updated Branches:
refs/heads/master a53a37f9c -> e62417ea6
NIFI-1923 - AttributesToJson regex property
added EL support
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2099
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/e62417ea
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/e62417ea
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/e62417ea
Branch: refs/heads/master
Commit: e62417ea6b189ad0e7522cffd68481dd5df7ffd8
Parents: a53a37f
Author: Pierre Villard <pi...@gmail.com>
Authored: Fri Aug 18 12:19:11 2017 +0200
Committer: Matthew Burgess <ma...@apache.org>
Committed: Wed Aug 30 14:01:27 2017 -0400
----------------------------------------------------------------------
.../processors/standard/AttributesToJSON.java | 49 ++++++++++++++++----
.../standard/TestAttributesToJSON.java | 34 ++++++++++++++
2 files changed, 73 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/e62417ea/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
index cfa4cfe..6e00619 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
@@ -46,6 +46,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
+import java.util.regex.Pattern;
import java.util.HashSet;
import java.util.Map;
import java.util.Collections;
@@ -79,6 +80,19 @@ public class AttributesToJSON extends AbstractProcessor {
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
+
+ public static final PropertyDescriptor ATTRIBUTES_REGEX = new PropertyDescriptor.Builder()
+ .name("attributes-to-json-regex")
+ .displayName("Attributes Regular Expression")
+ .description("Regular expression that will be evaluated against the flow file attributes to select "
+ + "the matching attributes. This property can be used in combination with the attributes "
+ + "list property.")
+ .required(false)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true))
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
.name("Destination")
.description("Control if JSON value is written as a new flowfile attribute '" + JSON_ATTRIBUTE_NAME + "' " +
@@ -120,11 +134,13 @@ public class AttributesToJSON extends AbstractProcessor {
private volatile Set<String> attributes;
private volatile Boolean nullValueForEmptyString;
private volatile boolean destinationContent;
+ private volatile Pattern pattern;
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(ATTRIBUTES_LIST);
+ properties.add(ATTRIBUTES_REGEX);
properties.add(DESTINATION);
properties.add(INCLUDE_CORE_ATTRIBUTES);
properties.add(NULL_VALUE_FOR_EMPTY_STRING);
@@ -153,17 +169,27 @@ public class AttributesToJSON extends AbstractProcessor {
* @return
* Map of values that are feed to a Jackson ObjectMapper
*/
- protected Map<String, String> buildAttributesMapForFlowFile(FlowFile ff, Set<String> attributes, Set<String> attributesToRemove, boolean nullValForEmptyString) {
+ protected Map<String, String> buildAttributesMapForFlowFile(FlowFile ff, Set<String> attributes, Set<String> attributesToRemove,
+ boolean nullValForEmptyString, Pattern attPattern) {
Map<String, String> result;
//If list of attributes specified get only those attributes. Otherwise write them all
- if (attributes != null) {
- result = new HashMap<>(attributes.size());
- for (String attribute : attributes) {
- String val = ff.getAttribute(attribute);
- if (val != null || nullValForEmptyString) {
- result.put(attribute, val);
- } else {
- result.put(attribute, "");
+ if (attributes != null || attPattern != null) {
+ result = new HashMap<>();
+ if(attributes != null) {
+ for (String attribute : attributes) {
+ String val = ff.getAttribute(attribute);
+ if (val != null || nullValForEmptyString) {
+ result.put(attribute, val);
+ } else {
+ result.put(attribute, "");
+ }
+ }
+ }
+ if(attPattern != null) {
+ for (Map.Entry<String, String> e : ff.getAttributes().entrySet()) {
+ if(attPattern.matcher(e.getKey()).matches()) {
+ result.put(e.getKey(), e.getValue());
+ }
}
}
} else {
@@ -204,6 +230,9 @@ public class AttributesToJSON extends AbstractProcessor {
attributes = buildAtrs(context.getProperty(ATTRIBUTES_LIST).getValue(), attributesToRemove);
nullValueForEmptyString = context.getProperty(NULL_VALUE_FOR_EMPTY_STRING).asBoolean();
destinationContent = DESTINATION_CONTENT.equals(context.getProperty(DESTINATION).getValue());
+ if(context.getProperty(ATTRIBUTES_REGEX).isSet()) {
+ pattern = Pattern.compile(context.getProperty(ATTRIBUTES_REGEX).evaluateAttributeExpressions().getValue());
+ }
}
@Override
@@ -213,7 +242,7 @@ public class AttributesToJSON extends AbstractProcessor {
return;
}
- final Map<String, String> atrList = buildAttributesMapForFlowFile(original, attributes, attributesToRemove, nullValueForEmptyString);
+ final Map<String, String> atrList = buildAttributesMapForFlowFile(original, attributes, attributesToRemove, nullValueForEmptyString, pattern);
try {
if (destinationContent) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/e62417ea/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java
index 5c8df9b..84f5c7d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java
@@ -388,4 +388,38 @@ public class TestAttributesToJSON {
Set<String> coreAttributes = Arrays.stream(CoreAttributes.values()).map(CoreAttributes::key).collect(Collectors.toSet());
val.keySet().forEach(k -> assertTrue(coreAttributes.contains(k)));
}
+
+ @Test
+ public void testAttributesRegex() throws IOException {
+ final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToJSON());
+ testRunner.setVariable("regex", "delimited\\.header\\.column\\.[0-9]+");
+ testRunner.setProperty(AttributesToJSON.ATTRIBUTES_REGEX, "${regex}");
+ testRunner.setProperty(AttributesToJSON.ATTRIBUTES_LIST, "test, test1");
+
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("delimited.header.column.1", "Registry");
+ attributes.put("delimited.header.column.2", "Assignment");
+ attributes.put("delimited.header.column.3", "Organization Name");
+ attributes.put("delimited.header.column.4", "Organization Address");
+ attributes.put("delimited.footer.column.1", "not included");
+ attributes.put("test", "test");
+ attributes.put("test1", "test1");
+ testRunner.enqueue("".getBytes(), attributes);
+
+ testRunner.run();
+
+ testRunner.assertTransferCount(AttributesToJSON.REL_FAILURE, 0);
+ testRunner.assertTransferCount(AttributesToJSON.REL_SUCCESS, 1);
+
+ MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS).get(0);
+
+ Map<String, String> val = new ObjectMapper().readValue(flowFile.getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME), HashMap.class);
+ assertTrue(val.keySet().contains("delimited.header.column.1"));
+ assertTrue(val.keySet().contains("delimited.header.column.2"));
+ assertTrue(val.keySet().contains("delimited.header.column.3"));
+ assertTrue(val.keySet().contains("delimited.header.column.4"));
+ assertTrue(!val.keySet().contains("delimited.footer.column.1"));
+ assertTrue(val.keySet().contains("test"));
+ assertTrue(val.keySet().contains("test1"));
+ }
}