You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2026/09/08 21:23:18 UTC
(nifi) branch NIFI-16322 created (now 40752b9c629)
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a change to branch NIFI-16322
in repository https://gitbox.apache.org/repos/asf/nifi.git
at 40752b9c629 NIFI-16322 Stream GenerateFlowFile content instead of a full-size byte array
This branch includes the following new commits:
new 40752b9c629 NIFI-16322 Stream GenerateFlowFile content instead of a full-size byte array
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(nifi) 01/01: NIFI-16322 Stream GenerateFlowFile content instead of a full-size byte array
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch NIFI-16322
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 40752b9c6292996cd679cfa410bd864163da6d7c
Author: Joseph Witt <jo...@apache.org>
AuthorDate: Tue Sep 8 14:17:55 2026 -0700
NIFI-16322 Stream GenerateFlowFile content instead of a full-size byte array
Co-authored-by: Cursor <cu...@cursor.com>
---
.../nifi/processors/standard/GenerateFlowFile.java | 131 +++++++++++++--------
.../processors/standard/TestGenerateFlowFile.java | 73 +++++++++++-
2 files changed, 152 insertions(+), 52 deletions(-)
diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java
index 3a46949cbd5..f173a97df5b 100644
--- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java
+++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java
@@ -42,6 +42,8 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.scheduling.SchedulingStrategy;
+import java.io.IOException;
+import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
@@ -67,7 +69,9 @@ import java.util.concurrent.atomic.AtomicReference;
@DefaultSchedule(strategy = SchedulingStrategy.TIMER_DRIVEN, period = "1 min")
public class GenerateFlowFile extends AbstractProcessor {
- private final AtomicReference<byte[]> data = new AtomicReference<>();
+ private static final int BUFFER_SIZE = 8192;
+
+ private final AtomicReference<GeneratedData> generatedData = new AtomicReference<>();
public static final String DATA_FORMAT_BINARY = "Binary";
public static final String DATA_FORMAT_TEXT = "Text";
@@ -77,7 +81,7 @@ public class GenerateFlowFile extends AbstractProcessor {
.description("The size of the file that will be used")
.required(true)
.defaultValue("0B")
- .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
+ .addValidator(StandardValidators.createDataSizeBoundsValidator(0, Integer.MAX_VALUE))
.expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
.build();
public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
@@ -96,8 +100,8 @@ public class GenerateFlowFile extends AbstractProcessor {
.build();
public static final PropertyDescriptor UNIQUE_FLOWFILES = new PropertyDescriptor.Builder()
.name("Unique FlowFiles")
- .description("If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles "
- + "will get the same content but this offers much higher throughput")
+ .description("If true, each FlowFile that is generated will be unique. If false, all generated FlowFiles will have the same content. "
+ + "When Unique FlowFiles is false, the first FlowFile in a batch is written and additional FlowFiles clone that content.")
.required(true)
.allowableValues("true", "false")
.defaultValue("false")
@@ -169,10 +173,10 @@ public class GenerateFlowFile extends AbstractProcessor {
@OnScheduled
public void onScheduled(final ProcessContext context) {
- if (context.getProperty(UNIQUE_FLOWFILES).asBoolean()) {
- this.data.set(null);
- } else if (!context.getProperty(CUSTOM_TEXT).isSet()) {
- this.data.set(generateData(context));
+ if (context.getProperty(UNIQUE_FLOWFILES).asBoolean() || context.getProperty(CUSTOM_TEXT).isSet()) {
+ generatedData.set(null);
+ } else {
+ generatedData.set(getGeneratedData(context));
}
}
@@ -191,45 +195,44 @@ public class GenerateFlowFile extends AbstractProcessor {
return results;
}
- private byte[] generateData(final ProcessContext context) {
- final int byteCount = context.getProperty(FILE_SIZE).evaluateAttributeExpressions().asDataSize(DataUnit.B).intValue();
-
- final Random random = new Random();
- final byte[] array = new byte[byteCount];
- if (context.getProperty(DATA_FORMAT).getValue().equals(DATA_FORMAT_BINARY)) {
- random.nextBytes(array);
- } else {
- for (int i = 0; i < array.length; i++) {
- final int index = random.nextInt(TEXT_CHARS.length);
- array[i] = (byte) TEXT_CHARS[index];
- }
- }
-
- return array;
+ private GeneratedData getGeneratedData(final ProcessContext context) {
+ final long byteCount = context.getProperty(FILE_SIZE).evaluateAttributeExpressions().asDataSize(DataUnit.B).longValue();
+ final boolean binary = context.getProperty(DATA_FORMAT).getValue().equals(DATA_FORMAT_BINARY);
+ return new GeneratedData(byteCount, binary, new Random().nextLong());
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
- final byte[] data;
final boolean uniqueData = context.getProperty(UNIQUE_FLOWFILES).asBoolean();
+ final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
+ final Map<String, String> generatedAttributes = getGeneratedAttributes(context);
+
if (uniqueData) {
- data = new byte[0];
- } else {
- if (context.getProperty(CUSTOM_TEXT).isSet()) {
- final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
- data = context.getProperty(CUSTOM_TEXT).evaluateAttributeExpressions().getValue().getBytes(charset);
- } else {
- data = this.data.get();
+ for (int i = 0; i < batchSize; i++) {
+ session.transfer(createGeneratedFlowFile(session, getGeneratedData(context), generatedAttributes), SUCCESS);
}
+ return;
+ }
+
+ final FlowFile first;
+ if (context.getProperty(CUSTOM_TEXT).isSet()) {
+ first = createCustomTextFlowFile(context, session, generatedAttributes);
+ } else {
+ first = createGeneratedFlowFile(session, generatedData.get(), generatedAttributes);
+ }
+
+ for (int i = 1; i < batchSize; i++) {
+ session.transfer(session.clone(first), SUCCESS);
}
+ session.transfer(first, SUCCESS);
+ }
- Map<PropertyDescriptor, String> processorProperties = context.getProperties();
- Map<String, String> generatedAttributes = new HashMap<>();
- for (final Map.Entry<PropertyDescriptor, String> entry : processorProperties.entrySet()) {
- PropertyDescriptor property = entry.getKey();
+ private Map<String, String> getGeneratedAttributes(final ProcessContext context) {
+ final Map<String, String> generatedAttributes = new HashMap<>();
+ for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
+ final PropertyDescriptor property = entry.getKey();
if (property.isDynamic() && property.isExpressionLanguageSupported()) {
- String dynamicValue = context.getProperty(property).evaluateAttributeExpressions().getValue();
- generatedAttributes.put(property.getName(), dynamicValue);
+ generatedAttributes.put(property.getName(), context.getProperty(property).evaluateAttributeExpressions().getValue());
}
}
@@ -237,17 +240,53 @@ public class GenerateFlowFile extends AbstractProcessor {
generatedAttributes.put(CoreAttributes.MIME_TYPE.key(), context.getProperty(MIME_TYPE).getValue());
}
- for (int i = 0; i < context.getProperty(BATCH_SIZE).asInteger(); i++) {
- FlowFile flowFile = session.create();
- final byte[] writtenData = uniqueData ? generateData(context) : data;
- if (writtenData.length > 0) {
- flowFile = session.write(flowFile, out -> out.write(writtenData));
- }
- flowFile = session.putAllAttributes(flowFile, generatedAttributes);
+ return generatedAttributes;
+ }
+
+ private FlowFile createCustomTextFlowFile(final ProcessContext context, final ProcessSession session, final Map<String, String> generatedAttributes) {
+ final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
+ final byte[] customData = context.getProperty(CUSTOM_TEXT).evaluateAttributeExpressions().getValue().getBytes(charset);
+ FlowFile flowFile = session.create();
+ if (customData.length > 0) {
+ flowFile = session.write(flowFile, out -> out.write(customData));
+ }
+ return finishCreatedFlowFile(session, flowFile, generatedAttributes);
+ }
- session.getProvenanceReporter().create(flowFile);
- session.transfer(flowFile, SUCCESS);
+ private FlowFile createGeneratedFlowFile(final ProcessSession session, final GeneratedData data, final Map<String, String> generatedAttributes) {
+ FlowFile flowFile = session.create();
+ if (data.byteCount() > 0) {
+ flowFile = session.write(flowFile, out -> writeGeneratedData(out, data));
}
+ return finishCreatedFlowFile(session, flowFile, generatedAttributes);
+ }
+
+ private FlowFile finishCreatedFlowFile(final ProcessSession session, FlowFile flowFile, final Map<String, String> generatedAttributes) {
+ flowFile = session.putAllAttributes(flowFile, generatedAttributes);
+ session.getProvenanceReporter().create(flowFile);
+ return flowFile;
+ }
+
+ private static void writeGeneratedData(final OutputStream outputStream, final GeneratedData generatedData) throws IOException {
+ final Random random = new Random(generatedData.seed());
+ final byte[] buffer = new byte[(int) Math.min(BUFFER_SIZE, generatedData.byteCount())];
+ long remaining = generatedData.byteCount();
+
+ while (remaining > 0) {
+ final int length = (int) Math.min(buffer.length, remaining);
+ if (generatedData.binary()) {
+ random.nextBytes(buffer);
+ } else {
+ for (int i = 0; i < length; i++) {
+ buffer[i] = (byte) TEXT_CHARS[random.nextInt(TEXT_CHARS.length)];
+ }
+ }
+ outputStream.write(buffer, 0, length);
+ remaining -= length;
+ }
+ }
+
+ private record GeneratedData(long byteCount, boolean binary, long seed) {
}
@Override
diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java
index 89e4a82aa9f..2941b8e6140 100644
--- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java
+++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java
@@ -23,15 +23,22 @@ import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Unit tests for the GenerateFlowFile processor.
*/
public class TestGenerateFlowFile {
+ private static final String STREAMED_FILE_SIZE = "9 KB";
+ private static final long STREAMED_FILE_SIZE_BYTES = 9 * 1024;
+
private TestRunner runner;
@BeforeEach
@@ -41,21 +48,23 @@ public class TestGenerateFlowFile {
@Test
public void testGenerateCustomText() {
- runner.setProperty(GenerateFlowFile.FILE_SIZE, "100MB");
+ runner.setProperty(GenerateFlowFile.FILE_SIZE, "1B");
runner.setProperty(GenerateFlowFile.DATA_FORMAT, GenerateFlowFile.DATA_FORMAT_TEXT);
runner.setProperty(GenerateFlowFile.CUSTOM_TEXT, "This is my custom text!");
+ runner.setProperty(GenerateFlowFile.BATCH_SIZE, "2");
+
runner.run();
- runner.assertTransferCount(GenerateFlowFile.SUCCESS, 1);
- MockFlowFile generatedFlowFile = runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).getFirst();
- generatedFlowFile.assertContentEquals("This is my custom text!");
- generatedFlowFile.assertAttributeNotExists("mime.type");
+ runner.assertTransferCount(GenerateFlowFile.SUCCESS, 2);
+ runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).assertContentEquals("This is my custom text!");
+ runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(1).assertContentEquals("This is my custom text!");
+ runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).assertAttributeNotExists("mime.type");
}
@Test
public void testInvalidCustomText() {
- runner.setProperty(GenerateFlowFile.FILE_SIZE, "100MB");
+ runner.setProperty(GenerateFlowFile.FILE_SIZE, "1B");
runner.setProperty(GenerateFlowFile.DATA_FORMAT, GenerateFlowFile.DATA_FORMAT_BINARY);
runner.setProperty(GenerateFlowFile.CUSTOM_TEXT, "This is my custom text!");
runner.assertNotValid();
@@ -65,6 +74,34 @@ public class TestGenerateFlowFile {
runner.assertNotValid();
}
+ @Test
+ public void testFileSizeLargerThanIntegerMaxIsInvalid() {
+ // Configuration only — do not runner.run() with this size. GitHub runners cannot host a 3 GB FlowFile.
+ runner.setProperty(GenerateFlowFile.FILE_SIZE, "3 GB");
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testGenerateNonUniqueBinaryContentStreamed() {
+ assertStreamedContent(false, GenerateFlowFile.DATA_FORMAT_BINARY);
+ }
+
+ @Test
+ public void testGenerateNonUniqueTextContentStreamed() {
+ assertStreamedContent(false, GenerateFlowFile.DATA_FORMAT_TEXT);
+ }
+
+ @Test
+ public void testGenerateUniqueBinaryContentStreamed() {
+ assertStreamedContent(true, GenerateFlowFile.DATA_FORMAT_BINARY);
+ }
+
+ @Test
+ public void testGenerateUniqueTextContentStreamed() {
+ assertStreamedContent(true, GenerateFlowFile.DATA_FORMAT_TEXT);
+ }
+
@Test
public void testDynamicPropertiesToAttributes() {
runner.setProperty(GenerateFlowFile.FILE_SIZE, "1B");
@@ -136,4 +173,28 @@ public class TestGenerateFlowFile {
assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
}
+ private void assertStreamedContent(final boolean unique, final String dataFormat) {
+ runner.setProperty(GenerateFlowFile.FILE_SIZE, STREAMED_FILE_SIZE);
+ runner.setProperty(GenerateFlowFile.DATA_FORMAT, dataFormat);
+ runner.setProperty(GenerateFlowFile.UNIQUE_FLOWFILES, Boolean.toString(unique));
+ runner.setProperty(GenerateFlowFile.BATCH_SIZE, unique ? "2" : "3");
+
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(GenerateFlowFile.SUCCESS, unique ? 2 : 3);
+ final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS);
+ final MockFlowFile first = flowFiles.getFirst();
+ for (final MockFlowFile flowFile : flowFiles) {
+ assertEquals(STREAMED_FILE_SIZE_BYTES, flowFile.getSize());
+ }
+
+ if (unique) {
+ assertFalse(Arrays.equals(first.getData(), flowFiles.get(1).getData()));
+ } else {
+ for (int i = 1; i < flowFiles.size(); i++) {
+ assertArrayEquals(first.getData(), flowFiles.get(i).getData());
+ }
+ }
+ }
+
}