You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2018/09/11 15:16:33 UTC
nifi git commit: Revert "NIFI-4272 support multiple captures when EL
is present in replacement value"
Repository: nifi
Updated Branches:
refs/heads/master 87cf474e5 -> 0274bd4ff
Revert "NIFI-4272 support multiple captures when EL is present in replacement value"
This reverts commit f7f809c3d3632eea5234b31740984b73de322464.
NIFI-5474, NIFI-4272: When using Regex Replace with ReplaceText, and there are capturing groups, ensure that we populate the 'additionalVariables' map for each match of the regex. This allows Expression Language to reference the back-references properly even when there are multiple matches
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2951
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/0274bd4f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/0274bd4f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/0274bd4f
Branch: refs/heads/master
Commit: 0274bd4ff3f4199838ff1307c9c01d98fcc9150b
Parents: 87cf474
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue Aug 14 15:35:47 2018 -0400
Committer: Matthew Burgess <ma...@apache.org>
Committed: Tue Sep 11 11:07:33 2018 -0400
----------------------------------------------------------------------
.../nifi/processors/standard/ReplaceText.java | 166 +++++++------------
.../processors/standard/TestReplaceText.java | 94 ++++-------
2 files changed, 93 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/0274bd4f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
index e424108..2a9a6fe 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java
@@ -80,9 +80,7 @@ import java.util.regex.Pattern;
@SystemResourceConsideration(resource = SystemResource.MEMORY)
public class ReplaceText extends AbstractProcessor {
- private static Pattern QUOTED_GROUP_REF_PATTERN = Pattern.compile("\\$\\{\\s*?'\\$\\d+?'.+?\\}");
- private static Pattern DOUBLE_QUOTED_GROUP_REF_PATTERN = Pattern.compile("\\$\\{\\s*?\"\\$\\d+?\".+?\\}");
- private static Pattern LITERAL_QUOTED_PATTERN = Pattern.compile("literal\\(('.*?')\\)",Pattern.DOTALL);
+ private static Pattern REPLACEMENT_NORMALIZATION_PATTERN = Pattern.compile("(\\$\\D)");
// Constants
public static final String LINE_BY_LINE = "Line-by-Line";
@@ -295,9 +293,7 @@ public class ReplaceText extends AbstractProcessor {
final StopWatch stopWatch = new StopWatch(true);
try {
-
flowFile = replacementStrategyExecutor.replace(flowFile, session, context, evaluateMode, charset, maxBufferSize);
-
} catch (StackOverflowError e) {
// Some regular expressions can produce many matches on large input data size using recursive code
// do not log the StackOverflowError stack trace
@@ -314,8 +310,12 @@ public class ReplaceText extends AbstractProcessor {
// If we find a back reference that is not valid, then we will treat it as a literal string. For example, if we have 3 capturing
// groups and the Replacement Value has the value is "I owe $8 to him", then we want to treat the $8 as a literal "$8", rather
- // than attempting to use it as a back reference. We do this even if there are no capture groups.
+ // than attempting to use it as a back reference.
private static String escapeLiteralBackReferences(final String unescaped, final int numCapturingGroups) {
+ if (numCapturingGroups == 0) {
+ return unescaped;
+ }
+
String value = unescaped;
final Matcher backRefMatcher = unescapedBackReferencePattern.matcher(value); // consider unescaped back references
while (backRefMatcher.find()) {
@@ -523,12 +523,8 @@ public class ReplaceText extends AbstractProcessor {
@Override
public FlowFile replace(final FlowFile flowFile, final ProcessSession session, final ProcessContext context, final String evaluateMode, final Charset charset, final int maxBufferSize) {
- final AttributeValueDecorator quotedAttributeDecorator = new AttributeValueDecorator() {
- @Override
- public String decorate(final String attributeValue) {
- return Pattern.quote(attributeValue);
- }
- };
+ final AttributeValueDecorator quotedAttributeDecorator = Pattern::quote;
+
final String searchRegex = context.getProperty(SEARCH_VALUE).evaluateAttributeExpressions(flowFile, quotedAttributeDecorator).getValue();
final Pattern searchPattern = Pattern.compile(searchRegex);
@@ -545,24 +541,29 @@ public class ReplaceText extends AbstractProcessor {
final String contentString = new String(buffer, 0, flowFileSize, charset);
additionalAttrs.clear();
final Matcher matcher = searchPattern.matcher(contentString);
- if (matcher.find()) {
- for (int i = 1; i <= matcher.groupCount(); i++) {
- final String groupValue = matcher.group(i);
- additionalAttrs.put("$" + i, groupValue);
+
+ final PropertyValue replacementValueProperty = context.getProperty(REPLACEMENT_VALUE);
+
+ int matches = 0;
+ final StringBuffer sb = new StringBuffer();
+ while (matcher.find()) {
+ matches++;
+
+ for (int i=0; i <= matcher.groupCount(); i++) {
+ additionalAttrs.put("$" + i, matcher.group(i));
}
- // prepare the string and do the regex replace first
- // then evaluate the EL on the result
- String replacement = context.getProperty(REPLACEMENT_VALUE).getValue();
+ String replacement = replacementValueProperty.evaluateAttributeExpressions(flowFile, additionalAttrs, escapeBackRefDecorator).getValue();
replacement = escapeLiteralBackReferences(replacement, numCapturingGroups);
- replacement = escapeExpressionDollarSigns(replacement);
- replacement = wrapLiterals(replacement);
- replacement = contentString.replaceAll(searchRegex, replacement);
- replacement = escapeForEvaluation(replacement);
+ String replacementFinal = normalizeReplacementString(replacement);
+
+ matcher.appendReplacement(sb, replacementFinal);
+ }
- PropertyValue tempValue = context.newPropertyValue(replacement);
- final String updatedValue = tempValue.evaluateAttributeExpressions(flowFile, additionalAttrs, null).getValue();
+ if (matches > 0) {
+ matcher.appendTail(sb);
+ final String updatedValue = sb.toString();
updatedFlowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream out) throws IOException {
@@ -570,9 +571,9 @@ public class ReplaceText extends AbstractProcessor {
}
});
} else {
- // If no match, just return the original. No need to write out any content.
return flowFile;
}
+
} else {
updatedFlowFile = session.write(flowFile, new StreamCallback() {
@Override
@@ -580,26 +581,39 @@ public class ReplaceText extends AbstractProcessor {
try (NLKBufferedReader br = new NLKBufferedReader(new InputStreamReader(in, charset), maxBufferSize);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String oneLine;
+
+ final StringBuffer sb = new StringBuffer();
+ Matcher matcher = null;
+
while (null != (oneLine = br.readLine())) {
additionalAttrs.clear();
- final Matcher matcher = searchPattern.matcher(oneLine);
- if (matcher.find()) {
- for (int i = 1; i <= matcher.groupCount(); i++) {
- final String groupValue = matcher.group(i);
- additionalAttrs.put("$" + i, groupValue);
+ if (matcher == null) {
+ matcher = searchPattern.matcher(oneLine);
+ } else {
+ matcher.reset(oneLine);
+ }
+
+ int matches = 0;
+ sb.setLength(0);
+
+ while (matcher.find()) {
+ matches++;
+
+ for (int i=0; i <= matcher.groupCount(); i++) {
+ additionalAttrs.put("$" + i, matcher.group(i));
}
- // prepare the string and do the regex replace first
- // then evaluate the EL on the result
- String replacement = context.getProperty(REPLACEMENT_VALUE).getValue();
+ String replacement = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile, additionalAttrs, escapeBackRefDecorator).getValue();
replacement = escapeLiteralBackReferences(replacement, numCapturingGroups);
- replacement = escapeExpressionDollarSigns(replacement);
- replacement = wrapLiterals(replacement);
- replacement = oneLine.replaceAll(searchRegex, replacement);
- replacement = escapeForEvaluation(replacement);
+ String replacementFinal = normalizeReplacementString(replacement);
+
+ matcher.appendReplacement(sb, replacementFinal);
+ }
- PropertyValue tempValue = context.newPropertyValue(replacement);
- final String updatedValue = tempValue.evaluateAttributeExpressions(flowFile, additionalAttrs, null).getValue();
+ if (matches > 0) {
+ matcher.appendTail(sb);
+
+ final String updatedValue = sb.toString();
bw.write(updatedValue);
} else {
// No match. Just write out the line as it was.
@@ -679,76 +693,16 @@ public class ReplaceText extends AbstractProcessor {
}
/**
- * Wraps '$1' with the {@code literal} function for EL evaluation.
- * @param possibleLiteral the {@code String} to evaluate.
- * @return {@code String} with literals wrapped. If no literals or Expression Lanaguage present the passed string
- * is returned.
- */
- private static String wrapLiterals(String possibleLiteral) {
- String replacementFinal = possibleLiteral;
- if (!possibleLiteral.contains("${")) {
- return possibleLiteral;
- }
-
- if (QUOTED_GROUP_REF_PATTERN.matcher(replacementFinal).find()) {
- replacementFinal = replacementFinal.replaceAll("(\\$\\{\\s*?)('\\$\\d+?')(.*\\})", "$1literal($2)$3");
- }
-
- if (DOUBLE_QUOTED_GROUP_REF_PATTERN.matcher(replacementFinal).find()) {
- replacementFinal = replacementFinal.replaceAll("(\\$\\{\\s*?)(\"\\$\\d+?\")(.*\\})", "$1literal($2)$3");
- }
-
- return replacementFinal;
- }
-
- /**
* If we have a '$' followed by anything other than a number, then escape
- * it if it is not already escaped. E.g., '$d' becomes '\$d' so that it can be used as a literal in a
+ * it. E.g., '$d' becomes '\$d' so that it can be used as a literal in a
* regex.
*/
- private static String escapeExpressionDollarSigns(String replacement) {
-
- // are there expressions or group references
- if (replacement.indexOf('$') == -1) {
- return replacement;
+ private static String normalizeReplacementString(String replacement) {
+ String replacementFinal = replacement;
+ if (REPLACEMENT_NORMALIZATION_PATTERN.matcher(replacement).find()) {
+ replacementFinal = Matcher.quoteReplacement(replacement);
}
- StringBuilder sb = new StringBuilder();
- boolean lastWasEscape = false;
- for (int i=0; i<replacement.length(); i++) {
- char c = replacement.charAt(i);
- if (c == '\\' ) {
- lastWasEscape = true;
- } else {
- if ( c == '$') {
- if (!lastWasEscape && !Character.isDigit(replacement.charAt(i+1))) {
- sb.append('\\');
- }
- }
- lastWasEscape = false;
- }
- sb.append(c);
- }
- return sb.toString();
- }
-
- /**
- * Escapes a {@code String} containing literal('') EL values.
- * @param contentString the {@code String}
- * @return the escaped {@code String}. If no literal() is present, then the input {@code String} will be returned
- */
- private static String escapeForEvaluation(String contentString) {
- final Matcher matcher = LITERAL_QUOTED_PATTERN.matcher(contentString);
- String returnString = contentString;
- while(matcher.find()) {
- for (int i = 1; i <= matcher.groupCount(); i ++) {
- String replacement = matcher.group(i)
- .replaceAll("\\n","\\\\n")
- .replaceAll("\\r","\\\\r")
- .replaceAll("\\t","\\\\t");
- returnString = new StringBuilder(returnString).replace(matcher.start(i),matcher.end(i),replacement).toString();
- }
- }
- return returnString;
+ return replacementFinal;
}
private interface ReplacementStrategyExecutor {
http://git-wip-us.apache.org/repos/asf/nifi/blob/0274bd4f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
index 3c432e5..e96eddd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestReplaceText.java
@@ -16,6 +16,14 @@
*/
package org.apache.nifi.processors.standard;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -27,14 +35,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.nifi.util.MockFlowFile;
-import org.apache.nifi.util.TestRunner;
-import org.apache.nifi.util.TestRunners;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
public class TestReplaceText {
@Rule
@@ -66,87 +66,75 @@ public class TestReplaceText {
}
@Test
- public void testSimple() throws IOException {
+ public void testIterativeRegexReplace() {
final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "ell");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "lle");
+ runner.setProperty(ReplaceText.SEARCH_VALUE, "\"([a-z]+?)\":\"(.*?)\"");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "\"${'$1':toUpper()}\":\"$2\"");
- runner.enqueue(Paths.get("src/test/resources/hello.txt"));
+ runner.enqueue("{\"name\":\"Smith\",\"middle\":\"nifi\",\"firstname\":\"John\"}");
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("Hlleo, World!".getBytes("UTF-8"));
+ out.assertContentEquals("{\"NAME\":\"Smith\",\"MIDDLE\":\"nifi\",\"FIRSTNAME\":\"John\"}");
}
@Test
- public void testEscapedEnough$InReplacementCanReturnEscaped$() throws IOException {
+ public void testIterativeRegexReplaceLineByLine() {
final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s)(^.*$)");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "a\\\\\\$b");
+ runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
+ runner.setProperty(ReplaceText.SEARCH_VALUE, "\"([a-z]+?)\":\"(.*?)\"");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "\"${'$1':toUpper()}\":\"$2\"");
- runner.enqueue("a$a,b,c,d");
+ runner.enqueue("{\"name\":\"Smith\",\"middle\":\"nifi\",\"firstname\":\"John\"}");
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("a\\$b".getBytes("UTF-8"));
+ out.assertContentEquals("{\"NAME\":\"Smith\",\"MIDDLE\":\"nifi\",\"FIRSTNAME\":\"John\"}");
}
+
@Test
- public void testWithEscaped$InReplacement() throws IOException {
+ public void testSimple() throws IOException {
final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s)(^.*$)");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "a\\$b");
+ runner.setProperty(ReplaceText.SEARCH_VALUE, "ell");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "lle");
- runner.enqueue("a$a,b,c,d");
+ runner.enqueue(Paths.get("src/test/resources/hello.txt"));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("a$b".getBytes("UTF-8"));
+ out.assertContentEquals("Hlleo, World!".getBytes("UTF-8"));
}
@Test
- public void testWithUnEscaped$InReplacement() throws IOException {
+ public void testWithEscaped$InReplacement() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s:^.*$)");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "a$b");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "a\\$b");
runner.enqueue("a$a,b,c,d");
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("a$b".getBytes("UTF-8"));
+ out.assertContentEquals("a\\$b".getBytes("UTF-8"));
}
@Test
- public void testWithSingleQuotedELInReplacement() throws IOException {
+ public void testWithUnEscaped$InReplacement() throws IOException {
final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "\"([a-z]+)\":\"(\\w+)\"");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "\"${'$1':toUpper()}\":\"$2\"");
- runner.enqueue("{\"name\":\"Smith\",\"middle\":\"nifi\",\"firstname\":\"John\"}");
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
- final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("{\"NAME\":\"Smith\",\"MIDDLE\":\"nifi\",\"FIRSTNAME\":\"John\"}");
-
- }
+ runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s:^.*$)");
+ runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "a$b");
- @Test
- public void testWithDoubleQuotedELInReplacement() throws IOException {
- final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "\"([a-z]+)\":\"(\\w+)\"");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "\"${\"$1\":toUpper()}\":\"$2\"");
- runner.enqueue("{\"name\":\"Smith\",\"middle\":\"nifi\",\"firstname\":\"John\"}");
+ runner.enqueue("a$a,b,c,d");
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("{\"NAME\":\"Smith\",\"MIDDLE\":\"nifi\",\"FIRSTNAME\":\"John\"}");
-
+ out.assertContentEquals("a$b".getBytes("UTF-8"));
}
@Test
@@ -1143,22 +1131,6 @@ public class TestReplaceText {
}
@Test
- public void testRegexWithELAndELSpecialChars() throws Exception {
- final TestRunner runner = getRunner();
- runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s)(^.*$)");
- runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "${'$1':toUpper()}"); // will uppercase group with good Java regex
- runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.REGEX_REPLACE);
- runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.ENTIRE_TEXT);
-
- runner.enqueue("testing\n\t\r123".getBytes());
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
- final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
- out.assertContentEquals("TESTING\n\t\r123");
- }
-
- @Test
public void testRegexNoCaptureDefaultReplacement() throws IOException {
// Test the old Default Regex and new Default Regex with the default replacement. This should fail
// because the regex does not create a capture group.