You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2018/11/14 14:47:28 UTC
nifi git commit: NIFI-4130 Add lookup controller service in
TransformXML to define XSLT from the UI
Repository: nifi
Updated Branches:
refs/heads/master 9e7610ac7 -> 4112af013
NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI
addressed review comments
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #1953
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4112af01
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4112af01
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4112af01
Branch: refs/heads/master
Commit: 4112af013d1b1f49e83f881d85ebe66e097840b5
Parents: 9e7610a
Author: Pierre Villard <pi...@gmail.com>
Authored: Mon Jun 26 17:48:06 2017 +0200
Committer: Matthew Burgess <ma...@apache.org>
Committed: Wed Nov 14 09:45:20 2018 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/TransformXml.java | 132 +++++++++++++++---
.../processors/standard/TestTransformXml.java | 133 ++++++++++++++++++-
2 files changed, 238 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/4112af01/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
index ff15428..1cc57fa 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
@@ -16,21 +16,24 @@
*/
package org.apache.nifi.processors.standard;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
import javax.xml.XMLConstants;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Templates;
@@ -39,6 +42,8 @@ import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
@@ -49,6 +54,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
@@ -56,6 +62,9 @@ import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.lookup.LookupFailureException;
+import org.apache.nifi.lookup.LookupService;
+import org.apache.nifi.lookup.StringLookupService;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
@@ -67,6 +76,10 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StopWatch;
import org.apache.nifi.util.Tuple;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
@EventDriven
@SideEffectFree
@SupportsBatching
@@ -82,12 +95,33 @@ public class TransformXml extends AbstractProcessor {
public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder()
.name("XSLT file name")
- .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.")
- .required(true)
+ .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content."
+ + "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.")
+ .required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
.build();
+ public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder()
+ .name("xslt-controller")
+ .displayName("XSLT Lookup")
+ .description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and "
+ + "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service "
+ + "should not be used to store large XSLT files.")
+ .required(false)
+ .identifiesControllerService(StringLookupService.class)
+ .build();
+
+ public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder()
+ .name("xslt-controller-key")
+ .displayName("XSLT Lookup key")
+ .description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be "
+ + "set when using the XSLT controller property.")
+ .required(false)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
+ .build();
+
public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder()
.name("indent-output")
.displayName("Indent")
@@ -140,10 +174,14 @@ public class TransformXml extends AbstractProcessor {
private Set<Relationship> relationships;
private LoadingCache<String, Templates> cache;
+ private static AtomicReference<LookupService<String>> lookupService = new AtomicReference<LookupService<String>>(null);
+
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(XSLT_FILE_NAME);
+ properties.add(XSLT_CONTROLLER);
+ properties.add(XSLT_CONTROLLER_KEY);
properties.add(INDENT_OUTPUT);
properties.add(SECURE_PROCESSING);
properties.add(CACHE_SIZE);
@@ -167,6 +205,47 @@ public class TransformXml extends AbstractProcessor {
}
@Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+
+ PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME);
+ PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER);
+ PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY);
+
+ if((filename.isSet() && controller.isSet())
+ || (!filename.isSet() && !controller.isSet())) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .subject(this.getClass().getSimpleName())
+ .explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.")
+ .build());
+ }
+
+ if(controller.isSet() && !key.isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .subject(XSLT_CONTROLLER_KEY.getDisplayName())
+ .explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.")
+ .build());
+ }
+
+ if(controller.isSet()) {
+ final LookupService<String> lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class);
+ final Set<String> requiredKeys = lookupService.getRequiredKeys();
+ if (requiredKeys == null || requiredKeys.size() != 1) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .subject(XSLT_CONTROLLER.getDisplayName())
+ .explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " +
+ (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size())))
+ .build());
+ }
+ }
+
+ return results;
+ }
+
+ @Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
@@ -177,9 +256,10 @@ public class TransformXml extends AbstractProcessor {
.build();
}
- private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException {
+ private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException {
final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean();
TransformerFactory factory = TransformerFactory.newInstance();
+ final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet();
if (secureProcessing) {
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
@@ -188,7 +268,17 @@ public class TransformXml extends AbstractProcessor {
factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities", false);
}
- return factory.newTemplates(new StreamSource(path));
+ if(isFilename) {
+ return factory.newTemplates(new StreamSource(path));
+ } else {
+ final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next();
+ final Optional<String> attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path));
+ if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) {
+ return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8))));
+ } else {
+ throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service.");
+ }
+ }
}
@OnScheduled
@@ -198,20 +288,21 @@ public class TransformXml extends AbstractProcessor {
final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);
if (cacheSize > 0) {
- CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
+ CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
if (cacheTTL > 0) {
cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
}
cache = cacheBuilder.build(
- new CacheLoader<String, Templates>() {
- public Templates load(String path) throws TransformerConfigurationException {
- return newTemplates(context, path);
- }
- });
+ new CacheLoader<String, Templates>() {
+ @Override
+ public Templates load(String path) throws TransformerConfigurationException, LookupFailureException {
+ return newTemplates(context, path);
+ }
+ });
} else {
cache = null;
- logger.warn("Stylesheet cache disabled because cache size is set to 0");
+ logger.info("Stylesheet cache disabled because cache size is set to 0");
}
}
@@ -224,10 +315,11 @@ public class TransformXml extends AbstractProcessor {
final ComponentLog logger = getLogger();
final StopWatch stopWatch = new StopWatch(true);
- final String xsltFileName = context.getProperty(XSLT_FILE_NAME)
- .evaluateAttributeExpressions(original)
- .getValue();
+ final String path = context.getProperty(XSLT_FILE_NAME).isSet()
+ ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue()
+ : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue();
final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean();
+ lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class));
try {
FlowFile transformed = session.write(original, new StreamCallback() {
@@ -236,9 +328,9 @@ public class TransformXml extends AbstractProcessor {
try (final InputStream in = new BufferedInputStream(rawIn)) {
final Templates templates;
if (cache != null) {
- templates = cache.get(xsltFileName);
+ templates = cache.get(path);
} else {
- templates = newTemplates(context, xsltFileName);
+ templates = newTemplates(context, path);
}
final Transformer transformer = templates.newTransformer();
@@ -303,4 +395,4 @@ public class TransformXml extends AbstractProcessor {
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/4112af01/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
index 5ced266..3d5f1e0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
@@ -16,23 +16,21 @@
*/
package org.apache.nifi.processors.standard;
-import java.io.IOException;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
+import org.apache.nifi.lookup.SimpleKeyValueLookupService;
+import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
-
import org.junit.Test;
public class TestTransformXml {
@@ -56,8 +54,6 @@ public class TestTransformXml {
runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0);
- final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8);
-
original.assertContentEquals("not xml");
}
@@ -107,7 +103,6 @@ public class TestTransformXml {
runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
- final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);
final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml")));
transformed.assertContentEquals(expectedContent);
@@ -148,4 +143,128 @@ public class TestTransformXml {
transformed.assertContentEquals(expectedContent);
}
+ @Test
+ public void testTransformBothControllerFileNotValid() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+ runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");
+
+ final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+ runner.addControllerService("simple-key-value-lookup-service", service);
+ runner.setProperty(service, "key1", "value1");
+ runner.enableControllerService(service);
+ runner.assertValid(service);
+ runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+ runner.setProperty(TransformXml.CACHE_SIZE, "0");
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testTransformControllerNoKey() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+ final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+ runner.addControllerService("simple-key-value-lookup-service", service);
+ runner.setProperty(service, "key1", "value1");
+ runner.enableControllerService(service);
+ runner.assertValid(service);
+ runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testTransformWithController() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+ final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+ runner.addControllerService("simple-key-value-lookup-service", service);
+ runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+ + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+ + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+ + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+ + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+ + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+ + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+ + "</HTML></xsl:template></xsl:stylesheet>");
+ runner.enableControllerService(service);
+ runner.assertValid(service);
+ runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+ runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+ runner.setProperty("header", "Test for mod");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("xslt", "math");
+ runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
+ final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
+ final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
+
+ transformed.assertContentEquals(expectedContent);
+ }
+
+ @Test
+ public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+ final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+ runner.addControllerService("simple-key-value-lookup-service", service);
+ runner.enableControllerService(service);
+ runner.assertValid(service);
+ runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+ runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+ runner.setProperty("header", "Test for mod");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("xslt", "math");
+ runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
+ }
+
+ @Test
+ public void testTransformWithControllerNoCache() throws IOException, InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+ final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+ runner.addControllerService("simple-key-value-lookup-service", service);
+ runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+ + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+ + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+ + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+ + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+ + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+ + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+ + "</HTML></xsl:template></xsl:stylesheet>");
+ runner.enableControllerService(service);
+ runner.assertValid(service);
+ runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+ runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+ runner.setProperty(TransformXml.CACHE_SIZE, "0");
+ runner.setProperty("header", "Test for mod");
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("xslt", "math");
+ runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
+ final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
+ final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
+
+ transformed.assertContentEquals(expectedContent);
+ }
+
}