You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2019/04/04 15:14:13 UTC
[nifi] branch master updated: NIFI-6186 Resolve handling of module
paths in JythonScriptEngineConfigurator
This is an automated email from the ASF dual-hosted git repository.
mattyb149 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new c93ee5a NIFI-6186 Resolve handling of module paths in JythonScriptEngineConfigurator
c93ee5a is described below
commit c93ee5ad5949858bd3f4eec2067ae30016ea76ac
Author: ambah <am...@users.noreply.github.com>
AuthorDate: Wed Apr 3 18:15:58 2019 -0400
NIFI-6186 Resolve handling of module paths in JythonScriptEngineConfigurator
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #3406
---
.../impl/JythonScriptEngineConfigurator.java | 3 +-
.../nifi/processors/script/TestInvokeJython.java | 55 ++++++++++++++++
.../src/test/resources/jython/test_modules_path.py | 74 ++++++++++++++++++++++
3 files changed, 131 insertions(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
index 458658b..3b8271b 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
@@ -18,6 +18,7 @@ package org.apache.nifi.script.impl;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processors.script.ScriptEngineConfigurator;
+import org.python.core.PyString;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
@@ -47,7 +48,7 @@ public class JythonScriptEngineConfigurator implements ScriptEngineConfigurator
engine.eval("import sys");
if (modulePaths != null) {
for (String modulePath : modulePaths) {
- engine.eval("sys.path.append('" + modulePath + "')");
+ engine.eval("sys.path.append(" + PyString.encode_UnicodeEscape(modulePath, true) + ")");
}
}
}
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
index 5b17e04..345f4af 100755
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.processors.script;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.script.ScriptingComponentUtils;
import org.apache.nifi.util.MockFlowFile;
@@ -31,6 +32,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
public class TestInvokeJython extends BaseScriptTest {
@@ -95,6 +99,57 @@ public class TestInvokeJython extends BaseScriptTest {
}
/**
+ * Test a script that has a Jython processor that reads the system path as controlled by the Module Directory property then stores it in the attributes of the flowfile being routed.
+ * <p>
+ * This tests whether the JythonScriptEngineConfigurator successfully translates the "Module Directory" property into Python system paths, even with strings that contain Python escape sequences
+ *
+ * @throws Exception Any error encountered while testing
+ */
+ @Test
+ public void testUpdateAttributeFromProcessorModulePaths() throws Exception {
+ // Prepare a set of easily identified paths for the Module Directory property
+ final String moduleDirectoryTestPrefix = "test";
+ final String[] testModuleDirectoryValues = { "abc","\\a\\b\\c","\\123","\\d\"e" };
+ final int numTestValues = testModuleDirectoryValues.length;
+ // Prepend each module directory value with a simple prefix and an identifying number so we can identify it later.
+ final List<String> testModuleDirectoryFullValues = IntStream.range(0,numTestValues)
+ .boxed()
+ .map(i -> String.format("%s#%s#%s",moduleDirectoryTestPrefix,i,testModuleDirectoryValues[i]))
+ .collect(Collectors.toList());
+ final String testModuleDirectoryCombined = String.join(",",testModuleDirectoryFullValues);
+
+ // Run the script that captures the system path resulting from the Module Directory property
+ final TestRunner runner = TestRunners.newTestRunner(new InvokeScriptedProcessor());
+
+ runner.setValidateExpressionUsage(false);
+ runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "python");
+ runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "target/test/resources/jython/test_modules_path.py");
+ runner.setProperty(ScriptingComponentUtils.MODULES, testModuleDirectoryCombined);
+
+ final Map<String, String> attributes = new HashMap<>();
+
+ runner.assertValid();
+ runner.enqueue(new byte[0], attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred("success", 1);
+ final List<MockFlowFile> result = runner.getFlowFilesForRelationship("success");
+
+ // verify successful processing of the module paths
+ result.get(0).assertAttributeExists("from-path");
+ final String[] effectivePaths = result.get(0).getAttribute("from-path").split(","); // Extract the comma-delimited paths from the script-produced attribute
+ Assert.assertTrue(effectivePaths.length >= numTestValues); // we should have our test values, plus defaults
+ // Isolate only the paths with our identified prefix
+ final List<String> relevantPaths = Arrays.stream(effectivePaths).filter(path -> path.startsWith(moduleDirectoryTestPrefix)).collect(Collectors.toList());
+ Assert.assertEquals(testModuleDirectoryFullValues.size(), relevantPaths.size());
+ relevantPaths.forEach(path -> {
+ final int resultIx = Integer.valueOf(StringUtils.substringBetween(path,"#")); // extract the index so we can relate it to the sources, despite potential mangling
+ final String expectedValue = testModuleDirectoryFullValues.get(resultIx);
+ Assert.assertEquals(expectedValue, path); // Ensure our path was passed through without mangling
+ });
+ }
+
+ /**
* Tests a script that has a Jython Processor that that reads the first line of text from the flowfiles content and stores the value in an attribute of the outgoing flowfile.
*
* @throws Exception Any error encountered while testing
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py
new file mode 100644
index 0000000..bee52e6
--- /dev/null
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py
@@ -0,0 +1,74 @@
+#! /usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import sys
+import traceback
+from org.apache.nifi.processor import Processor
+from org.apache.nifi.processor import Relationship
+
+class ReadModulesPathAndStoreAsAttribute(Processor) :
+ __rel_success = Relationship.Builder().description("Success").name("success").build()
+
+ def __init__(self) :
+ pass
+
+ def initialize(self, context) :
+ pass
+
+ def getRelationships(self) :
+ return set([self.__rel_success])
+
+ def validate(self, context) :
+ pass
+
+ def getPropertyDescriptors(self) :
+ pass
+
+ def onPropertyModified(self, descriptor, newValue, oldValue) :
+ pass
+
+ def onTrigger(self, context, sessionFactory) :
+ session = sessionFactory.createSession()
+ try :
+ # ensure there is work to do
+ flowfile = session.get()
+ if flowfile is None :
+ return
+
+ # Extract sys.path and encode as a comma-separated string
+ sysPathString = ','.join(sys.path)
+
+ # set an attribute with the captured contents of sys.path
+ flowfile = session.putAttribute(flowfile, "from-path", sysPathString)
+
+ # transfer
+ session.transfer(flowfile, self.__rel_success)
+ session.commit()
+ except :
+ print sys.exc_info()[0]
+ print "Exception in TestReader:"
+ print '-' * 60
+ traceback.print_exc(file=sys.stdout)
+ print '-' * 60
+
+ session.rollback(True)
+ raise
+
+processor = ReadModulesPathAndStoreAsAttribute()
\ No newline at end of file