You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2021/04/27 11:03:33 UTC

[nifi] branch main updated: NIFI-8474: Added new Replacement Strategy for variable substitution in ReplaceText.

This is an automated email from the ASF dual-hosted git repository.

pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/main by this push:
     new 525bfe0  NIFI-8474: Added new Replacement Strategy for variable substitution in ReplaceText.
525bfe0 is described below

commit 525bfe00b3858d9836c736d5c60acce3c1e13239
Author: Peter Turcsanyi <tu...@apache.org>
AuthorDate: Mon Apr 26 21:30:17 2021 +0200

    NIFI-8474: Added new Replacement Strategy for variable substitution in ReplaceText.
    
    Signed-off-by: Pierre Villard <pi...@gmail.com>
    
    This closes #5032.
---
 .../nifi/processors/standard/ReplaceText.java      | 46 +++++++++++++++++++++-
 .../nifi/processors/standard/TestReplaceText.java  | 44 +++++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
index fb0cbb8..7d5eae4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
@@ -17,6 +17,7 @@
 package org.apache.nifi.processors.standard;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.text.StringSubstitutor;
 import org.apache.nifi.annotation.behavior.EventDriven;
 import org.apache.nifi.annotation.behavior.InputRequirement;
 import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -95,6 +96,7 @@ public class ReplaceText extends AbstractProcessor {
     public static final String regexReplaceValue = "Regex Replace";
     public static final String literalReplaceValue = "Literal Replace";
     public static final String alwaysReplace = "Always Replace";
+    public static final String SUBSTITUTE_VARIABLES_VALUE = "Substitute Variables";
     private static final Pattern unescapedBackReferencePattern = Pattern.compile("[^\\\\]\\$(\\d+)");
     private static final String DEFAULT_REGEX = "(?s)(^.*$)";
     private static final String DEFAULT_REPLACEMENT_VALUE = "$1";
@@ -120,6 +122,9 @@ public class ReplaceText extends AbstractProcessor {
     static final AllowableValue ALWAYS_REPLACE = new AllowableValue(alwaysReplace, alwaysReplace,
         "Always replaces the entire line or the entire contents of the FlowFile (depending on the value of the <Evaluation Mode> property) and does not bother searching "
             + "for any value. When this strategy is chosen, the <Search Value> property is ignored.");
+    static final AllowableValue SUBSTITUTE_VARIABLES = new AllowableValue(SUBSTITUTE_VARIABLES_VALUE, SUBSTITUTE_VARIABLES_VALUE,
+            "Substitute variable references (specified in ${var} form) using FlowFile attributes for looking up the replacement value by variable name. "
+                    + "When this strategy is chosen, both the <Search Value> and <Replacement Value> properties are ignored.");
 
     public static final PropertyDescriptor SEARCH_VALUE = new PropertyDescriptor.Builder()
         .name("Regular Expression")
@@ -163,7 +168,7 @@ public class ReplaceText extends AbstractProcessor {
     public static final PropertyDescriptor REPLACEMENT_STRATEGY = new PropertyDescriptor.Builder()
         .name("Replacement Strategy")
         .description("The strategy for how and what to replace within the FlowFile's text content.")
-        .allowableValues(PREPEND, APPEND, REGEX_REPLACE, LITERAL_REPLACE, ALWAYS_REPLACE)
+        .allowableValues(PREPEND, APPEND, REGEX_REPLACE, LITERAL_REPLACE, ALWAYS_REPLACE, SUBSTITUTE_VARIABLES)
         .defaultValue(REGEX_REPLACE.getValue())
         .required(true)
         .build();
@@ -246,6 +251,7 @@ public class ReplaceText extends AbstractProcessor {
             case appendValue:
             case prependValue:
             case alwaysReplace:
+            case SUBSTITUTE_VARIABLES_VALUE:
             default:
                 // nothing to check, search value is not used
                 break;
@@ -286,6 +292,9 @@ public class ReplaceText extends AbstractProcessor {
             case alwaysReplace:
                 replacementStrategyExecutor = new AlwaysReplace();
                 break;
+            case SUBSTITUTE_VARIABLES_VALUE:
+                replacementStrategyExecutor = new SubstituteVariablesReplace();
+                break;
             default:
                 throw new AssertionError();
         }
@@ -652,6 +661,41 @@ public class ReplaceText extends AbstractProcessor {
         }
     }
 
+    private static class SubstituteVariablesReplace implements ReplacementStrategyExecutor {
+        @Override
+        public FlowFile replace(FlowFile flowFile, final ProcessSession session, final ProcessContext context, final String evaluateMode, final Charset charset, final int maxBufferSize) {
+            final Map<String, String> flowFileAttributes = flowFile.getAttributes();
+
+            if (evaluateMode.equalsIgnoreCase(ENTIRE_TEXT)) {
+                final int flowFileSize = (int) flowFile.getSize();
+                final int bufferSize = Math.min(maxBufferSize, flowFileSize);
+                final byte[] buffer = new byte[bufferSize];
+
+                flowFile = session.write(flowFile, new StreamCallback() {
+                    @Override
+                    public void process(final InputStream in, final OutputStream out) throws IOException {
+                        StreamUtils.fillBuffer(in, buffer, false);
+                        final String originalContent = new String(buffer, 0, flowFileSize, charset);
+                        final String substitutedContent = StringSubstitutor.replace(originalContent, flowFileAttributes);
+                        out.write(substitutedContent.getBytes(charset));
+                    }
+                });
+            } else {
+                flowFile = session.write(flowFile, new StreamReplaceCallback(charset, maxBufferSize, context.getProperty(LINE_BY_LINE_EVALUATION_MODE).getValue(),
+                        (bw, oneLine) -> {
+                            final String substitutedLine = StringSubstitutor.replace(oneLine, flowFileAttributes);
+                            bw.write(substitutedLine);
+                        }));
+            }
+            return flowFile;
+        }
+
+        @Override
+        public boolean isAllDataBufferedForEntireText() {
+            return true;
+        }
+    }
+
     /**
      * If we have a '$' followed by anything other than a number, then escape
      * it. E.g., '$d' becomes '\$d' so that it can be used as a literal in a
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
index 341b75e..88ed097 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
@@ -1641,6 +1641,50 @@ public class TestReplaceText {
         out.assertContentEquals("WO$1R$2D");
     }
 
+    @Test
+    public void testSubstituteVariablesWithEvaluationModeEntireText() {
+        testSubstituteVariables("Line1: ${var1}\nLine2: ${var2}", "Line1: foo\nLine2: bar", ReplaceText.ENTIRE_TEXT, createAttributesMap());
+    }
+
+    @Test
+    public void testSubstituteVariablesWithEvaluationModeLineByLine() {
+        testSubstituteVariables("Line1: ${var1}\nLine2: ${var2}", "Line1: foo\nLine2: bar", ReplaceText.LINE_BY_LINE, createAttributesMap());
+    }
+
+    @Test
+    public void testSubstituteVariablesWhenVariableValueMissing() {
+        testSubstituteVariables("Line1: ${var1}\nLine2: ${var2}", "Line1: ${var1}\nLine2: ${var2}", ReplaceText.ENTIRE_TEXT, Collections.emptyMap());
+    }
+
+    @Test
+    public void testSubstituteVariablesWhenVariableReferenceEscaped() {
+        testSubstituteVariables("Line1: $${var1}\nLine2: $${var2}", "Line1: ${var1}\nLine2: ${var2}", ReplaceText.ENTIRE_TEXT, createAttributesMap());
+    }
+
+    @Test
+    public void testSubstituteVariablesWhenVariableNameEmpty() {
+        testSubstituteVariables("Line1: ${}\nLine2: ${}", "Line1: ${}\nLine2: ${}", ReplaceText.ENTIRE_TEXT, createAttributesMap());
+    }
+
+    private void testSubstituteVariables(String inputContent, String expectedContent, String evaluationMode, Map<String, String> attributesMap) {
+        final TestRunner runner = getRunner();
+        runner.setProperty(ReplaceText.EVALUATION_MODE, evaluationMode);
+        runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.SUBSTITUTE_VARIABLES.getValue());
+        runner.enqueue(inputContent, attributesMap);
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
+        final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
+        out.assertContentEquals(expectedContent);
+    }
+
+    private Map<String, String> createAttributesMap() {
+        final Map<String, String> attributesMap = new HashMap<>();
+        attributesMap.put("var1", "foo");
+        attributesMap.put("var2", "bar");
+        return attributesMap;
+    }
+
     /*
      * A repeated alternation regex such as (A|B)* can lead to StackOverflowError
      * on large input strings.