You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2020/08/07 00:40:36 UTC
[nifi] branch main updated: NIFI-7572: Force
ScriptedTransformRecord to recompile the script when started
This is an automated email from the ASF dual-hosted git repository.
alopresto pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 6488db1 NIFI-7572: Force ScriptedTransformRecord to recompile the script when started
6488db1 is described below
commit 6488db13762fd42ce6c51f574e2e47ac29006ab5
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Thu Aug 6 10:54:41 2020 -0400
NIFI-7572: Force ScriptedTransformRecord to recompile the script when started
This closes #4458.
Signed-off-by: Andy LoPresto <al...@apache.org>
---
.../processors/script/ScriptedTransformRecord.java | 2 +
.../script/TestScriptedTransformRecord.java | 60 ++++++++++++++++++++++
2 files changed, 62 insertions(+)
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java
index d14187e..0adf82c 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java
@@ -183,6 +183,8 @@ public class ScriptedTransformRecord extends AbstractProcessor implements Search
scriptToRun = IOUtils.toString(scriptStream, Charset.defaultCharset());
}
}
+ // Always compile when first run
+ compiledScriptRef.set(null);
}
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedTransformRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedTransformRecord.java
index 3b15cf3..08676c8 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedTransformRecord.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestScriptedTransformRecord.java
@@ -333,6 +333,66 @@ public class TestScriptedTransformRecord {
assertEquals("Unknown Author", outputRecords.get(1).getAsRecord("book", bookSchema).getValue("author"));
}
+ @Test
+ public void testRecompileJythonScript() throws InitializationException {
+ final RecordSchema schema = createSimpleNumberSchema();
+ setup(schema);
+
+ testRunner.setProperty(ScriptedTransformRecord.LANGUAGE, "python");
+ testRunner.setProperty(ScriptingComponentUtils.SCRIPT_BODY, "_ = record");
+
+ final Map<String, Object> num1 = new HashMap<>();
+ num1.put("num", 1);
+ final Map<String, Object> num2 = new HashMap<>();
+ num2.put("num", 2);
+ final Map<String, Object> num3 = new HashMap<>();
+ num3.put("num", 3);
+
+ recordReader.addRecord(new MapRecord(schema, num1));
+ recordReader.addRecord(new MapRecord(schema, num2));
+ recordReader.addRecord(new MapRecord(schema, num3));
+
+ testRunner.enqueue(new byte[0]);
+
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(ScriptedTransformRecord.REL_SUCCESS, 1);
+
+ MockFlowFile out = testRunner.getFlowFilesForRelationship(ScriptedTransformRecord.REL_SUCCESS).get(0);
+ out.assertAttributeEquals("record.count", "3");
+ assertEquals(3, testRunner.getCounterValue("Records Transformed").intValue());
+ assertEquals(0, testRunner.getCounterValue("Records Dropped").intValue());
+
+ List<Record> recordsWritten = recordWriter.getRecordsWritten();
+ assertEquals(3, recordsWritten.size());
+
+ for (int i = 0; i < 3; i++) {
+ assertEquals(i + 1, recordsWritten.get(i).getAsInt("num").intValue());
+ }
+
+ testRunner.clearTransferState();
+ // reset the writer
+ testRunner.removeControllerService(recordWriter);
+ recordWriter = new ArrayListRecordWriter(schema);
+ testRunner.addControllerService("record-writer", recordWriter);
+ testRunner.enableControllerService(recordWriter);
+
+ testRunner.setProperty(ScriptingComponentUtils.SCRIPT_BODY, "record.setValue(\"num\", 5)\n_ = record");
+
+ testRunner.enqueue(new byte[0]);
+ testRunner.run();
+ testRunner.assertAllFlowFilesTransferred(ScriptedTransformRecord.REL_SUCCESS, 1);
+
+ out = testRunner.getFlowFilesForRelationship(ScriptedTransformRecord.REL_SUCCESS).get(0);
+ out.assertAttributeEquals("record.count", "3");
+
+ recordsWritten = recordWriter.getRecordsWritten();
+ assertEquals(3, recordsWritten.size());
+
+ for (int i = 0; i < 3; i++) {
+ assertEquals(5, recordsWritten.get(i).getAsInt("num").intValue());
+ }
+ }
+
private Record createBook(final String author, final String date, final RecordSchema bookSchema, final RecordSchema outerSchema) {
final Map<String, Object> firstBookValues = new HashMap<>();
firstBookValues.put("author", author);