You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jg...@apache.org on 2022/02/22 23:54:55 UTC
[nifi] branch main updated: NIFI-9691: Added ForkEnrichment, JoinEnrichment processors
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new ce0122b NIFI-9691: Added ForkEnrichment, JoinEnrichment processors
ce0122b is described below
commit ce0122bd2530e37eb57ace919a42bb2eb3cd5c02
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Mon Jan 17 19:54:11 2022 -0500
NIFI-9691: Added ForkEnrichment, JoinEnrichment processors
Signed-off-by: Joe Gresock <jg...@gmail.com>
This closes #5772.
---
.../org/apache/nifi/processor/ProcessSession.java | 9 +-
.../nifi/serialization/record/MapRecord.java | 31 +-
.../org/apache/nifi/util/MockProcessSession.java | 4 +
.../apache/nifi/processor/util/bin/BinFiles.java | 82 +--
.../record/CommaSeparatedRecordReader.java | 18 +-
.../repository/StandardProcessSession.java | 11 +
.../repository/metrics/StandardFlowFileEvent.java | 7 +-
.../repository/BatchingSessionFactory.java | 5 +
.../groovyx/flow/ProcessSessionWrap.java | 5 +
.../nifi-standard-processors/pom.xml | 6 +
.../nifi/processors/standard/ForkEnrichment.java | 101 ++++
.../nifi/processors/standard/JoinEnrichment.java | 562 +++++++++++++++++++++
.../nifi/processors/standard/QueryRecord.java | 319 +-----------
.../standard/calcite/RecordPathFunction.java | 134 +++++
.../standard/calcite/RecordPathFunctions.java | 239 +++++++++
.../standard/enrichment/EnrichmentRole.java | 24 +
.../enrichment/IndexCorrelatedJoinStrategy.java | 140 +++++
.../enrichment/InsertRecordFieldsJoinStrategy.java | 74 +++
.../standard/enrichment/RecordJoinInput.java | 46 ++
.../standard/enrichment/RecordJoinResult.java | 26 +
.../standard/enrichment/RecordJoinStrategy.java | 25 +
.../standard/enrichment/SqlJoinCache.java | 127 +++++
.../enrichment/SqlJoinCalciteParameters.java | 84 +++
.../standard/enrichment/SqlJoinStrategy.java | 102 ++++
.../standard/enrichment/WrapperJoinStrategy.java | 70 +++
.../org/apache/nifi/queryrecord/FlowFileTable.java | 3 +-
.../services/org.apache.nifi.processor.Processor | 2 +
.../additionalDetails.html | 84 +++
.../fork-join-enrichment.png | Bin 0 -> 600998 bytes
.../additionalDetails.html | 524 +++++++++++++++++++
.../fork-join-enrichment.png | Bin 0 -> 600998 bytes
.../processors/standard/TestJoinEnrichment.java | 259 ++++++++++
.../nifi/processors/standard/TestMergeContent.java | 22 +-
.../TestIndexCorrelatedJoinStrategy.java | 67 +++
.../TestInsertRecordFieldsJoinStrategy.java | 177 +++++++
.../enrichment/TestWrapperJoinStrategy.java | 112 ++++
.../TestJoinEnrichment/insert-enrichment.json | 14 +
.../TestJoinEnrichment/insert-original.json | 37 ++
.../left-outer-join-enrichment.csv | 4 +
.../left-outer-join-expected.csv | 5 +
.../left-outer-join-original.csv | 5 +
.../left-outer-join-rename-expected.csv | 5 +
42 files changed, 3204 insertions(+), 367 deletions(-)
diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
index ed9e997..5ec1714 100644
--- a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
+++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessSession.java
@@ -268,13 +268,18 @@ public interface ProcessSession {
* newly added attribute) will be migrated, not the outdated version of the FlowFile that <code>flowFile1</code> points to.
* </p>
*
- * @param newOwner the ProcessSession that is to become the new owner of all FlowFiles
- * that currently belong to {@code this}.
+ * @param newOwner the ProcessSession that is to become the new owner of the given FlowFiles
* @param flowFiles the FlowFiles to migrate
*/
void migrate(ProcessSession newOwner, Collection<FlowFile> flowFiles);
/**
+ * Migrates all FlowFiles to the given new owner as described in {@link #migrate(ProcessSession, Collection)}
+ * @param newOwner the ProcessSession that is to become the new owner of all FlowFiles
+ */
+ void migrate(ProcessSession newOwner);
+
+ /**
* Adjusts counter data for the given counter name and takes care of
* registering the counter if not already present. The adjustment occurs
* only if and when the ProcessSession is committed.
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
index 16ff5c5..4909564 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
@@ -29,6 +29,7 @@ import org.apache.nifi.serialization.record.util.IllegalTypeConversionException;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -297,7 +298,35 @@ public class MapRecord implements Record {
return false;
}
final MapRecord other = (MapRecord) obj;
- return schema.equals(other.schema) && values.equals(other.values);
+ return schema.equals(other.schema) && valuesEqual(values, other.values);
+ }
+
+ private boolean valuesEqual(final Map<String, Object> thisValues, final Map<String, Object> otherValues) {
+ if (thisValues == null || otherValues == null) {
+ return false;
+ }
+
+ if (thisValues.size() != otherValues.size()) {
+ return false;
+ }
+
+ for (final Map.Entry<String, Object> entry : thisValues.entrySet()) {
+ final Object thisValue = entry.getValue();
+ final Object otherValue = otherValues.get(entry.getKey());
+ if (Objects.equals(thisValue, otherValue)) {
+ continue;
+ }
+
+ if (thisValue instanceof Object[] && otherValue instanceof Object[]) {
+ if (!Arrays.equals((Object[]) thisValue, (Object[]) otherValue)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ return true;
}
@Override
diff --git a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
index 163fc78..bb9d40d 100644
--- a/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
+++ b/nifi-mock/src/main/java/org/apache/nifi/util/MockProcessSession.java
@@ -135,6 +135,10 @@ public class MockProcessSession implements ProcessSession {
counterMap.put(name, counter);
}
+ public void migrate(final ProcessSession newOwner) {
+ migrate(newOwner, (Collection) currentVersions.values());
+ }
+
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void migrate(final ProcessSession newOwner, final Collection<FlowFile> flowFiles) {
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-bin-manager/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java b/nifi-nar-bundles/nifi-extension-utils/nifi-bin-manager/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
index df8d4cf..1253faf 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-bin-manager/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-bin-manager/src/main/java/org/apache/nifi/processor/util/bin/BinFiles.java
@@ -21,6 +21,7 @@ import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
@@ -189,7 +190,7 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
}
final int binsMigrated = migrateBins(context, flowFilesBinned == 0);
- final int binsProcessed = processBins(context);
+ final int binsProcessed = processBins(context, sessionFactory);
//If we accomplished nothing then let's yield
if (flowFilesBinned == 0 && binsMigrated == 0 && binsProcessed == 0) {
context.yield();
@@ -206,7 +207,7 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
// if we have created all of the bins that are allowed, go ahead and remove the oldest one. If we don't do
// this, then we will simply wait for it to expire because we can't get any more FlowFiles into the
// bins. So we may as well expire it now.
- if (added == 0 && binManager.getBinCount() >= context.getProperty(MAX_BIN_COUNT).asInteger()) {
+ if (added == 0 && binManager.getBinCount() > context.getProperty(MAX_BIN_COUNT).asInteger()) {
final Bin bin = binManager.removeOldestBin();
if (bin != null) {
added++;
@@ -218,11 +219,15 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
return added;
}
- private int processBins(final ProcessContext context) {
+ protected Queue<Bin> getReadyBins() {
+ return readyBins;
+ }
+
+ protected int processBins(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
final ComponentLog logger = getLogger();
int processedBins = 0;
Bin bin;
- while ((bin = readyBins.poll()) != null) {
+ while (isScheduled() && (bin = readyBins.poll()) != null) {
BinProcessingResult binProcessingResult;
try {
binProcessingResult = this.processBin(bin, context);
@@ -299,39 +304,54 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
@OnScheduled
public final void onScheduled(final ProcessContext context) throws IOException {
- binManager.setMinimumSize(context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue());
+ binManager.setMinimumSize(getMinBytes(context));
+ binManager.setMaximumSize(getMaxBytes(context));
+ binManager.setMaxBinAge(getMaxBinAgeSeconds(context));
+ binManager.setMinimumEntries(getMinEntries(context));
+ binManager.setMaximumEntries(getMaxEntries(context));
+
+ this.setUpBinManager(binManager, context);
+ }
+ protected int getMaxBinAgeSeconds(final PropertyContext context) {
if (context.getProperty(MAX_BIN_AGE).isSet()) {
- binManager.setMaxBinAge(context.getProperty(MAX_BIN_AGE).asTimePeriod(TimeUnit.SECONDS).intValue());
- } else {
- binManager.setMaxBinAge(Integer.MAX_VALUE);
+ return context.getProperty(MAX_BIN_AGE).asTimePeriod(TimeUnit.SECONDS).intValue();
}
+ return Integer.MAX_VALUE;
+ }
+
+ protected long getMinBytes(final PropertyContext context) {
+ return context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue();
+ }
+
+ protected long getMaxBytes(final PropertyContext context) {
if (context.getProperty(MAX_SIZE).isSet()) {
- binManager.setMaximumSize(context.getProperty(MAX_SIZE).asDataSize(DataUnit.B).longValue());
- } else {
- binManager.setMaximumSize(Long.MAX_VALUE);
+ return context.getProperty(MAX_SIZE).asDataSize(DataUnit.B).longValue();
}
+ return Long.MAX_VALUE;
+ }
- binManager.setMinimumEntries(context.getProperty(MIN_ENTRIES).asInteger());
+ protected int getMinEntries(final PropertyContext context) {
+ return context.getProperty(MIN_ENTRIES).asInteger();
+ }
+ protected int getMaxEntries(final PropertyContext context) {
if (context.getProperty(MAX_ENTRIES).isSet()) {
- binManager.setMaximumEntries(context.getProperty(MAX_ENTRIES).asInteger().intValue());
- } else {
- binManager.setMaximumEntries(Integer.MAX_VALUE);
+ return context.getProperty(MAX_ENTRIES).asInteger();
}
- this.setUpBinManager(binManager, context);
+ return Integer.MAX_VALUE;
}
@Override
protected final Collection<ValidationResult> customValidate(final ValidationContext context) {
final List<ValidationResult> problems = new ArrayList<>(super.customValidate(context));
- final long minBytes = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue();
- final Double maxBytes = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B);
+ final long minBytes = getMinBytes(context);
+ final long maxBytes = getMaxBytes(context);
- if (maxBytes != null && maxBytes.longValue() < minBytes) {
+ if (maxBytes < minBytes) {
problems.add(
new ValidationResult.Builder()
.subject(MIN_SIZE.getName())
@@ -342,19 +362,17 @@ public abstract class BinFiles extends AbstractSessionFactoryProcessor {
);
}
- final Long min = context.getProperty(MIN_ENTRIES).asLong();
- final Long max = context.getProperty(MAX_ENTRIES).asLong();
-
- if (min != null && max != null) {
- if (min > max) {
- problems.add(
- new ValidationResult.Builder().subject(MIN_ENTRIES.getName())
- .input(context.getProperty(MIN_ENTRIES).getValue())
- .valid(false)
- .explanation("Min Entries must be less than or equal to Max Entries")
- .build()
- );
- }
+ final int minEntries = getMinEntries(context);
+ final int maxEntries = getMaxEntries(context);
+
+ if (minEntries > maxEntries) {
+ problems.add(
+ new ValidationResult.Builder().subject(MIN_ENTRIES.getName())
+ .input(context.getProperty(MIN_ENTRIES).getValue())
+ .valid(false)
+ .explanation("Min Entries must be less than or equal to Max Entries")
+ .build()
+ );
}
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/CommaSeparatedRecordReader.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/CommaSeparatedRecordReader.java
index 560955c..09ddaaf 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/CommaSeparatedRecordReader.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/CommaSeparatedRecordReader.java
@@ -37,6 +37,7 @@ import java.util.Map;
public class CommaSeparatedRecordReader extends AbstractControllerService implements RecordReaderFactory {
private int failAfterN;
private int recordCount = 0;
+ private boolean useNullForEmptyString = false;
public CommaSeparatedRecordReader() {
this(-1);
@@ -50,6 +51,10 @@ public class CommaSeparatedRecordReader extends AbstractControllerService implem
this.failAfterN = failAfterN;
}
+ public void setUseNullForEmptyString(final boolean useNullForEmptyString) {
+ this.useNullForEmptyString = useNullForEmptyString;
+ }
+
@Override
public RecordReader createRecordReader(Map<String, String> variables, final InputStream in, final long inputLength, final ComponentLog logger) throws IOException, SchemaNotFoundException {
final BufferedReader reader = new BufferedReader(new InputStreamReader(in));
@@ -86,7 +91,18 @@ public class CommaSeparatedRecordReader extends AbstractControllerService implem
int i = 0;
for (final RecordField field : fields) {
final String fieldName = field.getFieldName();
- valueMap.put(fieldName, values[i++].trim());
+ if (i >= values.length) {
+ valueMap.put(fieldName, null);
+ } else {
+ final String trimmed = values[i].trim();
+ if (useNullForEmptyString && trimmed.isEmpty()) {
+ valueMap.put(fieldName, null);
+ } else {
+ valueMap.put(fieldName, trimmed);
+ }
+ }
+
+ i++;
}
return new MapRecord(new SimpleRecordSchema(fields), valueMap);
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 2e36f74..16d810f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -1428,6 +1428,17 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn
}
@Override
+ public void migrate(final ProcessSession newOwner) {
+ final List<FlowFile> allFlowFiles = new ArrayList<>();
+
+ for (final StandardRepositoryRecord repositoryRecord : records.values()) {
+ allFlowFiles.add(repositoryRecord.getCurrent());
+ }
+
+ migrate(newOwner, allFlowFiles);
+ }
+
+ @Override
public void migrate(final ProcessSession newOwner, final Collection<FlowFile> flowFiles) {
verifyTaskActive();
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/metrics/StandardFlowFileEvent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/metrics/StandardFlowFileEvent.java
index a6e12c9..084a86e 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/metrics/StandardFlowFileEvent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/metrics/StandardFlowFileEvent.java
@@ -171,11 +171,8 @@ public final class StandardFlowFileEvent implements FlowFileEvent, Cloneable {
@Override
public long getAverageLineageMillis() {
- if (flowFilesOut == 0 && flowFilesRemoved == 0) {
- return 0L;
- }
-
- return aggregateLineageMillis / (flowFilesOut + flowFilesRemoved);
+ final int flowFileCount = flowFilesOut + flowFilesRemoved;
+ return flowFileCount == 0 ? 0L : aggregateLineageMillis / flowFileCount;
}
public void setAggregateLineageMillis(long lineageMilliseconds) {
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
index 2edc028..c2b2b86 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/BatchingSessionFactory.java
@@ -107,6 +107,11 @@ public class BatchingSessionFactory implements ProcessSessionFactory {
}
@Override
+ public void migrate(final ProcessSession newOwner) {
+ session.migrate(newOwner);
+ }
+
+ @Override
public void adjustCounter(String name, long delta, boolean immediate) {
session.adjustCounter(name, delta, immediate);
}
diff --git a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/src/main/java/org/apache/nifi/processors/groovyx/flow/ProcessSessionWrap.java b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/src/main/java/org/apache/nifi/processors/groovyx/flow/ProcessSessionWrap.java
index 6924b6b..8a12049 100644
--- a/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/src/main/java/org/apache/nifi/processors/groovyx/flow/ProcessSessionWrap.java
+++ b/nifi-nar-bundles/nifi-groovyx-bundle/nifi-groovyx-processors/src/main/java/org/apache/nifi/processors/groovyx/flow/ProcessSessionWrap.java
@@ -947,6 +947,11 @@ public abstract class ProcessSessionWrap implements ProcessSession {
}
@Override
+ public void migrate(final ProcessSession newOwner) {
+ session.migrate(newOwner);
+ }
+
+ @Override
public void migrate(ProcessSession newOwner, Collection<FlowFile> flowFiles) {
flowFiles = unwrap(flowFiles);
session.migrate(newOwner, flowFiles);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 708f026..ec3fd81 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -644,6 +644,12 @@
<exclude>src/test/resources/TestValidateRecord/int-maps-schema.avsc</exclude>
<exclude>src/test/resources/TestValidateRecord/int-maps-data.json</exclude>
<exclude>src/test/resources/TestValidateRecord/array-and-map-with-null-element.avro</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/insert-enrichment.json</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/insert-original.json</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/left-outer-join-enrichment.csv</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/left-outer-join-expected.csv</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/left-outer-join-original.csv</exclude>
+ <exclude>src/test/resources/TestJoinEnrichment/left-outer-join-rename-expected.csv</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ForkEnrichment.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ForkEnrichment.java
new file mode 100644
index 0000000..bca9208
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ForkEnrichment.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processors.standard.enrichment.EnrichmentRole;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+@SupportsBatching
+@SideEffectFree
+@SeeAlso(JoinEnrichment.class)
+@WritesAttributes({
+ @WritesAttribute(attribute = "enrichment.group.id", description = "The Group ID to use in order to correlate the 'original' FlowFile with the 'enrichment' FlowFile."),
+ @WritesAttribute(attribute = "enrichment.role", description = "The role to use for enrichment. This will either be ORIGINAL or ENRICHMENT.")
+})
+@CapabilityDescription("Used in conjunction with the JoinEnrichment processor, this processor is responsible for adding the attributes that are necessary for the JoinEnrichment processor to perform" +
+ " its function. Each incoming FlowFile will be cloned. The original FlowFile will have appropriate attributes added and then be transferred to the 'original' relationship. The clone will have" +
+ " appropriate attributes added and then be routed to the 'enrichment' relationship. See the documentation for the JoinEnrichment processor (and especially its Additional Details) for more" +
+ " information on how these Processors work together and how to perform enrichment tasks in NiFi by using these Processors.")
+@Tags({"fork", "join", "enrich", "record"})
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+public class ForkEnrichment extends AbstractProcessor {
+ static final String ENRICHMENT_ROLE = "enrichment.role";
+ static final String ENRICHMENT_GROUP_ID = "enrichment.group.id";
+
+ static final Relationship REL_ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("The incoming FlowFile will be routed to this relationship, after adding appropriate attributes.")
+ .build();
+ static final Relationship REL_ENRICHMENT = new Relationship.Builder()
+ .name("enrichment")
+ .description("A clone of the incoming FlowFile will be routed to this relationship, after adding appropriate attributes.")
+ .build();
+
+ private static final Set<Relationship> relationships = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ REL_ORIGINAL,
+ REL_ENRICHMENT
+ )));
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile original = session.get();
+ if (original == null) {
+ return;
+ }
+
+ final String groupId = UUID.randomUUID().toString();
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put(ENRICHMENT_GROUP_ID, groupId);
+ attributes.put(ENRICHMENT_ROLE, EnrichmentRole.ORIGINAL.name());
+ original = session.putAllAttributes(original, attributes);
+
+ attributes.put(ENRICHMENT_ROLE, EnrichmentRole.ENRICHMENT.name());
+ FlowFile enrichment = session.clone(original);
+ enrichment = session.putAllAttributes(enrichment, attributes);
+
+ session.transfer(original, REL_ORIGINAL);
+ session.transfer(enrichment, REL_ENRICHMENT);
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java
new file mode 100644
index 0000000..73dc768
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoinEnrichment.java
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SystemResource;
+import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.annotation.lifecycle.OnStopped;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.context.PropertyContext;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processor.util.bin.Bin;
+import org.apache.nifi.processor.util.bin.BinFiles;
+import org.apache.nifi.processor.util.bin.BinManager;
+import org.apache.nifi.processor.util.bin.BinProcessingResult;
+import org.apache.nifi.processor.util.bin.EvictionReason;
+import org.apache.nifi.processors.standard.enrichment.EnrichmentRole;
+import org.apache.nifi.processors.standard.enrichment.InsertRecordFieldsJoinStrategy;
+import org.apache.nifi.processors.standard.enrichment.RecordJoinInput;
+import org.apache.nifi.processors.standard.enrichment.RecordJoinResult;
+import org.apache.nifi.processors.standard.enrichment.RecordJoinStrategy;
+import org.apache.nifi.processors.standard.enrichment.SqlJoinCache;
+import org.apache.nifi.processors.standard.enrichment.SqlJoinStrategy;
+import org.apache.nifi.processors.standard.enrichment.WrapperJoinStrategy;
+import org.apache.nifi.record.path.validation.RecordPathValidator;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+import org.apache.nifi.serialization.record.util.DataTypeUtils;
+import org.apache.nifi.util.db.JdbcProperties;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@TriggerWhenEmpty
+@SideEffectFree
+@Tags({"fork", "join", "enrichment", "record", "sql", "wrap", "recordpath", "merge", "combine", "streams"})
+@CapabilityDescription("Joins together Records from two different FlowFiles where one FlowFile, the 'original' contains arbitrary records and the second FlowFile, the 'enrichment' contains " +
+ "additional data that should be used to enrich the first. See Additional Details for more information on how to configure this processor and the different use cases that it aims to accomplish.")
+@SeeAlso(ForkEnrichment.class)
+@WritesAttributes({
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Record Writer"),
+ @WritesAttribute(attribute = "record.count", description = "The number of records in the FlowFile")
+})
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "This Processor will load into heap all FlowFiles that are on its incoming queues. While it loads the FlowFiles " +
+ "themselves, and not their content, the FlowFile attributes can be very memory intensive. Additionally, if the Join Strategy is set to SQL, the SQL engine may require buffering the entire " +
+ "contents of the enrichment FlowFile for each concurrent task. See Processor's Additional Details for more details and for steps on how to mitigate these concerns.")
+public class JoinEnrichment extends BinFiles {
+ static final String GROUP_ID_ATTRIBUTE = "enrichment.group.id";
+ static final String ENRICHMENT_ROLE_ATTRIBUTE = "enrichment.role";
+ static final String RECORD_COUNT_ATTRIBUTE = "record.count";
+
+ static final AllowableValue JOIN_WRAPPER = new AllowableValue("Wrapper", "Wrapper", "The output is a Record that contains two fields: (1) 'original', containing the Record from the original " +
+ "FlowFile and (2) 'enrichment' containing the corresponding Record from the enrichment FlowFile. Records will be correlated based on their index in the FlowFile. If one FlowFile has more " +
+ "Records than the other, a null value will be used.");
+ static final AllowableValue JOIN_SQL = new AllowableValue("SQL", "SQL", "The output is derived by evaluating a SQL SELECT statement that allows for two tables: 'original' and 'enrichment'. This" +
+ " allows for SQL JOIN statements to be used in order to correlate the Records of the two FlowFiles, so the index in which the Record is encountered in the FlowFile does not matter.");
+ static final AllowableValue JOIN_INSERT_ENRICHMENT_FIELDS = new AllowableValue("Insert Enrichment Fields", "Insert Enrichment Fields",
+ "The enrichment is joined together with the original FlowFile by placing all fields of the enrichment Record into the corresponding Record from the original FlowFile. " +
+ "Records will be correlated based on their index in the FlowFile.");
+
+ static final PropertyDescriptor ORIGINAL_RECORD_READER = new PropertyDescriptor.Builder()
+ .name("Original Record Reader")
+ .displayName("Original Record Reader")
+ .description("The Record Reader for reading the 'original' FlowFile")
+ .required(true)
+ .identifiesControllerService(RecordReaderFactory.class)
+ .build();
+ static final PropertyDescriptor ENRICHMENT_RECORD_READER = new PropertyDescriptor.Builder()
+ .name("Enrichment Record Reader")
+ .displayName("Enrichment Record Reader")
+ .description("The Record Reader for reading the 'enrichment' FlowFile")
+ .required(true)
+ .identifiesControllerService(RecordReaderFactory.class)
+ .build();
+ static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder()
+ .name("Record Writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for writing the results. If the Record Writer is configured to inherit the schema from the Record, the schema that it will inherit will be the result " +
+ "of merging both the 'original' record schema and the 'enrichment' record schema.")
+ .required(true)
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .build();
+
+ static final PropertyDescriptor JOIN_STRATEGY = new PropertyDescriptor.Builder()
+ .name("Join Strategy")
+ .displayName("Join Strategy")
+ .description("Specifies how to join the two FlowFiles into a single FlowFile")
+ .required(true)
+ .allowableValues(JOIN_WRAPPER, JOIN_SQL, JOIN_INSERT_ENRICHMENT_FIELDS)
+ .defaultValue(JOIN_WRAPPER.getValue())
+ .build();
+ static final PropertyDescriptor SQL = new PropertyDescriptor.Builder()
+ .name("SQL")
+ .displayName("SQL")
+ .description("The SQL SELECT statement to evaluate. Expression Language may be provided, but doing so may result in poorer performance. Because this Processor is dealing with two FlowFiles " +
+ "at a time, it's also important to understand how attributes will be referenced. If both FlowFiles have an attribute with the same name but different values, the Expression Language " +
+ "will resolve to the value provided by the 'enrichment' FlowFile.")
+ .required(true)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(JOIN_STRATEGY, JOIN_SQL)
+ .defaultValue("SELECT original.*, enrichment.* \nFROM original \nLEFT OUTER JOIN enrichment \nON original.id = enrichment.id")
+ .build();
+ static final PropertyDescriptor DEFAULT_PRECISION = new PropertyDescriptor.Builder()
+ .fromPropertyDescriptor(JdbcProperties.DEFAULT_PRECISION)
+ .required(false)
+ .dependsOn(JOIN_STRATEGY, JOIN_SQL)
+ .build();
+ static final PropertyDescriptor DEFAULT_SCALE = new PropertyDescriptor.Builder()
+ .fromPropertyDescriptor(JdbcProperties.DEFAULT_SCALE)
+ .required(false)
+ .dependsOn(JOIN_STRATEGY, JOIN_SQL)
+ .build();
+ static final PropertyDescriptor INSERTION_RECORD_PATH = new PropertyDescriptor.Builder()
+ .name("Insertion Record Path")
+ .displayName("Insertion Record Path")
+ .description("Specifies where in the 'original' Record the 'enrichment' Record's fields should be inserted. Note that if the RecordPath " +
+ "does not point to any existing field in the original Record, the enrichment will not be inserted.")
+ .required(true)
+ .addValidator(new RecordPathValidator())
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(JOIN_STRATEGY, JOIN_INSERT_ENRICHMENT_FIELDS)
+ .build();
+
+ static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
+ .name("Timeout")
+ .displayName("Timeout")
+ .description("Specifies the maximum amount of time to wait for the second FlowFile once the first arrives at the processor, after which point the first " +
+ "FlowFile will be routed to the 'timeout' relationship.")
+ .required(true)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .defaultValue("10 min")
+ .build();
+ static final PropertyDescriptor MAX_BIN_COUNT = new PropertyDescriptor.Builder()
+ .fromPropertyDescriptor(BinFiles.MAX_BIN_COUNT)
+ .defaultValue("10000")
+ .build();
+
+ private static final List<PropertyDescriptor> properties = Collections.unmodifiableList(Arrays.asList(
+ ORIGINAL_RECORD_READER,
+ ENRICHMENT_RECORD_READER,
+ RECORD_WRITER,
+ JOIN_STRATEGY,
+ SQL,
+ DEFAULT_PRECISION,
+ DEFAULT_SCALE,
+ INSERTION_RECORD_PATH,
+ MAX_BIN_COUNT,
+ TIMEOUT
+ ));
+
+ // Relationships
+ static final Relationship REL_JOINED = new Relationship.Builder()
+ .name("joined")
+ .description("The resultant FlowFile with Records joined together from both the original and enrichment FlowFiles will be routed to this relationship")
+ .build();
+ static final Relationship REL_ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Both of the incoming FlowFiles ('original' and 'enrichment') will be routed to this Relationship. I.e., this is the 'original' version of both of these FlowFiles.")
+ .autoTerminateDefault(true)
+ .build();
+ static final Relationship REL_TIMEOUT = new Relationship.Builder()
+ .name("timeout")
+ .description("If one of the incoming FlowFiles (i.e., the 'original' FlowFile or the 'enrichment' FlowFile) arrives to this Processor but the other does not arrive within the configured " +
+ "Timeout period, the FlowFile that did arrive is routed to this relationship.")
+ .build();
+ static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("If both the 'original' and 'enrichment' FlowFiles arrive at the processor but there was a failure in joining the records, both of those FlowFiles will be routed to this " +
+ "relationship.")
+ .build();
+
+ private static final Set<Relationship> relationships = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ REL_JOINED,
+ REL_ORIGINAL,
+ REL_TIMEOUT,
+ REL_FAILURE
+ )));
+
+ private final SqlJoinCache sqlJoinCache = new SqlJoinCache(getLogger());
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return relationships;
+ }
+
+ @OnScheduled
+ public void registerCalciteDriver() {
+ // This is necessary in order to create a Calcite Connection for the SQL based Join strategy. We could put this in the SqlJoinStrategy but do it here to avoid doing this multiple times.
+ try {
+ DriverManager.registerDriver(new org.apache.calcite.jdbc.Driver());
+ } catch (final SQLException e) {
+ throw new ProcessException("Failed to load Calcite JDBC Driver", e);
+ }
+ }
+
+ @OnStopped
+ public synchronized void cleanup() throws Exception {
+ sqlJoinCache.close();
+ }
+
+ // No-op
+ @Override
+ protected FlowFile preprocessFlowFile(final ProcessContext context, final ProcessSession session, final FlowFile flowFile) {
+ return flowFile;
+ }
+
+ @Override
+ protected String getGroupId(final ProcessContext context, final FlowFile flowFile, final ProcessSession session) {
+ return flowFile.getAttribute(GROUP_ID_ATTRIBUTE);
+ }
+
+ // No-op
+ @Override
+ protected void setUpBinManager(final BinManager binManager, final ProcessContext context) {
+ }
+
+ // We expect to always have exactly 2 entries ('original' FlowFile and 'enrichment' FlowFile).
+ @Override
+ protected int getMinEntries(final PropertyContext context) {
+ return 2;
+ }
+
+ // We expect to always have exactly 2 entries ('original' FlowFile and 'enrichment' FlowFile).
+ @Override
+ protected int getMaxEntries(final PropertyContext context) {
+ return 2;
+ }
+
+ // We will not constrain bin sizes on byte counts.
+ @Override
+ protected long getMinBytes(final PropertyContext context) {
+ return 0L;
+ }
+
+ // We will not constrain bin sizes on byte counts.
+ @Override
+ protected long getMaxBytes(final PropertyContext context) {
+ return Long.MAX_VALUE;
+ }
+
+ @Override
+ protected int getMaxBinAgeSeconds(final PropertyContext context) {
+ return context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue();
+ }
+
+ @Override
+ protected BinProcessingResult processBin(final Bin unmodifiableBin, final ProcessContext context) throws ProcessException {
+ final ProcessSession session = unmodifiableBin.getSession();
+ final List<FlowFile> flowFiles = unmodifiableBin.getContents();
+
+ // We should receive either 1 FlowFile (in case of timeout) or 2 FlowFiles (max bin entries). Check for timeouts.
+ if (flowFiles.size() != 2) {
+ if (unmodifiableBin.getEvictionReason() == EvictionReason.BIN_MANAGER_FULL) {
+ session.transfer(flowFiles);
+ session.commitAsync();
+ return new BinProcessingResult(true);
+ }
+
+ session.transfer(flowFiles, REL_TIMEOUT);
+
+ final FlowFile flowFile = flowFiles.get(0);
+ final EnrichmentRole role = getEnrichmentRole(flowFile);
+ final String missingType = (role == null) ? "other" : getOtherEnrichmentRole(role).name();
+ getLogger().warn("Timed out waiting for the {} FlowFile to match {}; routing to {}", missingType, flowFiles.get(0), REL_TIMEOUT.getName());
+ session.commitAsync();
+
+ return new BinProcessingResult(true);
+ }
+
+ // Ensure that we receive both an 'original' and an 'enrichment' flowfile. This ensures that our attributes are properly aligned.
+ final FlowFile original = getFlowFileWithRole(flowFiles, EnrichmentRole.ORIGINAL);
+ final FlowFile enrichment = getFlowFileWithRole(flowFiles, EnrichmentRole.ENRICHMENT);
+
+ if (original == null || enrichment == null) {
+ getLogger().error("Received two FlowFiles {} but could not find both an 'original' and an 'enrichment' FlowFile. The FlowFiles do not appear to have the proper attributes necessary for " +
+ "use with this Processor. Routing to {}", flowFiles, REL_FAILURE);
+ return transferFailure(flowFiles, session);
+ }
+
+ // Combine FlowFile attributes into a single Map, preferring the 'enrichment' FlowFile's attributes over the 'original' attributes in case of conflicts.
+ final Map<String, String> combinedAttributes = new HashMap<>(original.getAttributes());
+ combinedAttributes.putAll(enrichment.getAttributes());
+
+ // Create the Record Readers/Writers and necessary schemas
+ final RecordReaderFactory originalRecordReaderFactory = context.getProperty(ORIGINAL_RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordReaderFactory enrichmentRecordReaderFactory = context.getProperty(ENRICHMENT_RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+
+ final RecordSchema originalSchema;
+ try {
+ originalSchema = getReaderSchema(originalRecordReaderFactory, original, session);
+ } catch (final Exception e) {
+ getLogger().error("Failed to determine Record Schema from 'original' FlowFile {}; routing to failure", original, e);
+ return transferFailure(flowFiles, session);
+ }
+
+ final RecordSchema enrichmentSchema;
+ try {
+ enrichmentSchema = getReaderSchema(enrichmentRecordReaderFactory, enrichment, session);
+ } catch (final Exception e) {
+ getLogger().error("Failed to determine Record Schema from 'enrichment' FlowFile {}; routing to failure", original, e);
+ return transferFailure(flowFiles, session);
+ }
+
+ // Merge the 'original' and 'enrichment' schemas into a combined schema.
+ final RecordSchema combinedSchema = DataTypeUtils.merge(originalSchema, enrichmentSchema);
+
+ // Determine the RecordWriter schema
+ final RecordSchema writerSchema;
+ try {
+ writerSchema = writerFactory.getSchema(combinedAttributes, combinedSchema);
+ } catch (final Exception e) {
+ getLogger().error("Failed to determine Record Schema for the Record Writer from 'original' FlowFile {} and 'enrichment' FLowFile {}; routing to failure", original, enrichment, e);
+ return transferFailure(flowFiles, session);
+ }
+
+ final RecordJoinInput originalInput = new RecordJoinInput(original, originalRecordReaderFactory, originalSchema);
+ final RecordJoinInput enrichmentInput = new RecordJoinInput(enrichment, enrichmentRecordReaderFactory, enrichmentSchema);
+
+ // Get the appropriate strategy for joining the FlowFiles
+ final RecordJoinStrategy joinStrategy = getJoinStrategy(context, combinedAttributes);
+
+ // Create a RecordSet that will be used to produce the Records we need.
+ final WriteResult writeResult;
+ final String mimeType;
+ FlowFile output;
+ try (final RecordJoinResult result = joinStrategy.join(originalInput, enrichmentInput, session, writerSchema)) {
+ // Create output FlowFile
+ output = session.create(flowFiles);
+
+ // Create the RecordSetWriter and write the RecordSet. This will allow us to iterate over the Records in the RecordSet in a 'pull'-like manner, in much the same
+ // way that the Java Stream class works.
+ try (final OutputStream out = session.write(output)) {
+ final RecordSet resultRecordSet = result.getRecordSet();
+ final RecordSchema rsSchema = resultRecordSet.getSchema();
+
+ try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), rsSchema, out, combinedAttributes)) {
+ writeResult = writer.write(resultRecordSet);
+ mimeType = writer.getMimeType();
+ }
+ }
+ } catch (final Exception e) {
+ getLogger().error("Failed to join 'original' FlowFile {} and 'enrichment' FlowFile {}; routing to failure", original, enrichment, e);
+ return transferFailure(flowFiles, session);
+ }
+
+ // Create attributes for outbound FlowFile
+ final Map<String, String> writeAttributes = new HashMap<>(writeResult.getAttributes());
+ final int joinedCount = writeResult.getRecordCount();
+ writeAttributes.put(RECORD_COUNT_ATTRIBUTE, String.valueOf(joinedCount));
+ writeAttributes.put(CoreAttributes.MIME_TYPE.key(), mimeType);
+ session.putAllAttributes(output, writeAttributes);
+
+ session.adjustCounter("Records Written", joinedCount, false);
+ session.transfer(output, REL_JOINED);
+
+ // We do not transfer original FlowFiles or commit the session - let parent class handle that, so return a BinProcessingResult with a 'false' for isCommitted.
+ return new BinProcessingResult(false);
+ }
+
+ private RecordSchema getReaderSchema(final RecordReaderFactory recordReaderFactory, final FlowFile flowFile, final ProcessSession session)
+ throws IOException, MalformedRecordException, SchemaNotFoundException {
+
+ try (final InputStream rawIn = session.read(flowFile)) {
+ final Map<String, String> enrichmentAttributes = flowFile.getAttributes();
+ final RecordReader reader = recordReaderFactory.createRecordReader(enrichmentAttributes, rawIn, flowFile.getSize(), getLogger());
+ final RecordSchema schema = reader.getSchema();
+ return schema;
+ }
+ }
+
+ private BinProcessingResult transferFailure(final List<FlowFile> flowFiles, final ProcessSession session) {
+ session.transfer(flowFiles, REL_FAILURE);
+ session.commitAsync();
+ return new BinProcessingResult(true);
+ }
+
+ private RecordJoinStrategy getJoinStrategy(final ProcessContext context, final Map<String, String> attributes) {
+ final String strategyName = context.getProperty(JOIN_STRATEGY).getValue();
+ if (strategyName.equalsIgnoreCase(JOIN_SQL.getValue())) {
+ final String sql = context.getProperty(SQL).getValue();
+ final int defaultPrecision = context.getProperty(DEFAULT_PRECISION).evaluateAttributeExpressions(attributes).asInteger();
+ final int defaultScale = context.getProperty(DEFAULT_SCALE).evaluateAttributeExpressions(attributes).asInteger();
+ return new SqlJoinStrategy(sqlJoinCache, sql, getLogger(), defaultPrecision, defaultScale);
+ } else if (strategyName.equalsIgnoreCase(JOIN_WRAPPER.getValue())) {
+ return new WrapperJoinStrategy(getLogger());
+ } else if (strategyName.equalsIgnoreCase(JOIN_INSERT_ENRICHMENT_FIELDS.getValue())) {
+ final String recordPath = context.getProperty(INSERTION_RECORD_PATH).evaluateAttributeExpressions(attributes).getValue();
+ return new InsertRecordFieldsJoinStrategy(getLogger(), recordPath);
+ }
+
+ throw new ProcessException("Invalid Join Strategy: " + strategyName);
+ }
+
+ private FlowFile getFlowFileWithRole(final Collection<FlowFile> flowFiles, final EnrichmentRole desiredRole) {
+ for (final FlowFile flowFile : flowFiles) {
+ final EnrichmentRole role = getEnrichmentRole(flowFile);
+ if (role == desiredRole) {
+ return flowFile;
+ }
+ }
+
+ return null;
+ }
+
+ private EnrichmentRole getEnrichmentRole(final FlowFile flowFile) {
+ final String role = flowFile.getAttribute(ENRICHMENT_ROLE_ATTRIBUTE);
+ if (role == null) {
+ return EnrichmentRole.UNKNOWN;
+ }
+
+ try {
+ return EnrichmentRole.valueOf(role);
+ } catch (final Exception e) {
+ return EnrichmentRole.UNKNOWN;
+ }
+ }
+
+ private EnrichmentRole getOtherEnrichmentRole(final EnrichmentRole role) {
+ if (role == null) {
+ return null;
+ }
+
+ switch (role) {
+ case ENRICHMENT:
+ return EnrichmentRole.ORIGINAL;
+ case ORIGINAL:
+ return EnrichmentRole.ENRICHMENT;
+ case UNKNOWN:
+ return EnrichmentRole.UNKNOWN;
+ }
+
+ return null;
+ }
+
+ @Override
+ protected int processBins(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
+ // If there is only a single ready bin, process in the same way that our parent class does.
+ // However, because this Processor uses tiny bins, if we have many ready we can significantly benefit
+ // by creating a new ProcessSession and batching together the processing of many bins into one.
+ final Queue<Bin> readyBins = getReadyBins();
+ if (readyBins.size() <= 1) {
+ return super.processBins(context, sessionFactory);
+ }
+
+ final ComponentLog logger = getLogger();
+ int binsProcessed = 0;
+ long bytesProcessed = 0L;
+
+ final ProcessSession batchSession = sessionFactory.createSession();
+
+ Bin bin;
+ while (isScheduled() && (bin = readyBins.poll()) != null) {
+ long binBytes = 0L;
+ for (final FlowFile flowFile : bin.getContents()) {
+ binBytes += flowFile.getSize();
+ }
+
+ BinProcessingResult binProcessingResult;
+ try {
+ binProcessingResult = this.processBin(bin, context);
+ } catch (final ProcessException e) {
+ logger.error("Failed to process bundle of {} files due to {}", new Object[] {bin.getContents().size(), e});
+
+ final ProcessSession binSession = bin.getSession();
+ for (final FlowFile flowFile : bin.getContents()) {
+ binSession.transfer(flowFile, REL_FAILURE);
+ }
+ binSession.commitAsync();
+ continue;
+ } catch (final Exception e) {
+ logger.error("Failed to process bundle of {} files due to {}; rolling back sessions", new Object[] {bin.getContents().size(), e});
+
+ bin.getSession().rollback();
+ continue;
+ }
+
+ // If this bin's session has been committed, move on.
+ if (!binProcessingResult.isCommitted()) {
+ final ProcessSession binSession = bin.getSession();
+ bin.getContents().forEach(ff -> binSession.putAllAttributes(ff, binProcessingResult.getAttributes()));
+ binSession.transfer(bin.getContents(), REL_ORIGINAL);
+
+ // Migrate FlowFiles to our batch session. Then commit the bin session to free up any resources
+ binSession.migrate(batchSession);
+ binSession.commitAsync();
+ }
+
+ binsProcessed++;
+ bytesProcessed += binBytes;
+
+ // Every 100 bins or 5 MB of input, commit the session
+ if (binsProcessed % 100 == 0 || bytesProcessed > 5_000_000L) {
+ batchSession.commitAsync();
+ break;
+ }
+ }
+
+ batchSession.commitAsync();
+ return binsProcessed;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
index cb921e2..aec971a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
@@ -23,7 +23,6 @@ import org.apache.calcite.config.CalciteConnectionProperty;
import org.apache.calcite.config.Lex;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.impl.ScalarFunctionImpl;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParser.Config;
import org.apache.nifi.annotation.behavior.DynamicProperty;
@@ -52,23 +51,14 @@ import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
+import org.apache.nifi.processors.standard.calcite.RecordPathFunctions;
import org.apache.nifi.queryrecord.FlowFileTable;
-import org.apache.nifi.record.path.FieldValue;
-import org.apache.nifi.record.path.RecordPath;
-import org.apache.nifi.record.path.RecordPathResult;
-import org.apache.nifi.record.path.StandardFieldValue;
-import org.apache.nifi.record.path.util.RecordPathCache;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
-import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.WriteResult;
-import org.apache.nifi.serialization.record.MapRecord;
-import org.apache.nifi.serialization.record.Record;
-import org.apache.nifi.serialization.record.RecordField;
-import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.ResultSetRecordSet;
import org.apache.nifi.util.StopWatch;
@@ -85,22 +75,17 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Function;
import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import static org.apache.nifi.util.db.JdbcProperties.DEFAULT_PRECISION;
import static org.apache.nifi.util.db.JdbcProperties.DEFAULT_SCALE;
@@ -428,7 +413,7 @@ public class QueryRecord extends AbstractProcessor {
final RecordReaderFactory recordReaderFactory) {
final CalciteConnection connection = createConnection();
- final SchemaPlus rootSchema = createRootSchema(connection);
+ final SchemaPlus rootSchema = RecordPathFunctions.createRootSchema(connection);
final FlowFileTable flowFileTable = new FlowFileTable(session, flowFile, schema, recordReaderFactory, getLogger());
rootSchema.add("FLOWFILE", flowFileTable);
@@ -503,18 +488,7 @@ public class QueryRecord extends AbstractProcessor {
};
}
- private SchemaPlus createRootSchema(final CalciteConnection calciteConnection) {
- final SchemaPlus rootSchema = calciteConnection.getRootSchema();
- rootSchema.add("RPATH", ScalarFunctionImpl.create(ObjectRecordPath.class, "eval"));
- rootSchema.add("RPATH_STRING", ScalarFunctionImpl.create(StringRecordPath.class, "eval"));
- rootSchema.add("RPATH_INT", ScalarFunctionImpl.create(IntegerRecordPath.class, "eval"));
- rootSchema.add("RPATH_LONG", ScalarFunctionImpl.create(LongRecordPath.class, "eval"));
- rootSchema.add("RPATH_DATE", ScalarFunctionImpl.create(DateRecordPath.class, "eval"));
- rootSchema.add("RPATH_DOUBLE", ScalarFunctionImpl.create(DoubleRecordPath.class, "eval"));
- rootSchema.add("RPATH_FLOAT", ScalarFunctionImpl.create(FloatRecordPath.class, "eval"));
-
- return rootSchema;
- }
+
private void closeQuietly(final AutoCloseable... closeables) {
if (closeables == null) {
@@ -601,291 +575,4 @@ public class QueryRecord extends AbstractProcessor {
}
}
-
- // ------------------------------------------------------------
- // User-Defined Functions for Calcite
- // ------------------------------------------------------------
-
-
- public static class ObjectRecordPath extends RecordPathFunction {
- private static final RecordField ROOT_RECORD_FIELD = new RecordField("root", RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()));
- private static final RecordSchema ROOT_RECORD_SCHEMA = new SimpleRecordSchema(Collections.singletonList(ROOT_RECORD_FIELD));
- private static final RecordField PARENT_RECORD_FIELD = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(ROOT_RECORD_SCHEMA));
-
-
- public Object eval(Object record, String recordPath) {
- if (record == null) {
- return null;
- }
- if (record instanceof Record) {
- return eval((Record) record, recordPath);
- }
- if (record instanceof Record[]) {
- return eval((Record[]) record, recordPath);
- }
- if (record instanceof Iterable) {
- return eval((Iterable<Record>) record, recordPath);
- }
- if (record instanceof Map) {
- return eval((Map<?, ?>) record, recordPath);
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against given argument because the argument is of type " + record.getClass() + " instead of Record");
- }
-
- private Object eval(final Map<?, ?> map, final String recordPath) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final Record record = new MapRecord(ROOT_RECORD_SCHEMA, Collections.singletonMap("root", map));
- final FieldValue parentFieldValue = new StandardFieldValue(record, PARENT_RECORD_FIELD, null);
- final FieldValue fieldValue = new StandardFieldValue(map, ROOT_RECORD_FIELD, parentFieldValue);
- final RecordPathResult result = compiled.evaluate(record, fieldValue);
-
- final List<FieldValue> selectedFields = result.getSelectedFields().collect(Collectors.toList());
- return evalResults(selectedFields);
- }
-
- private Object eval(final Record record, final String recordPath) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
- final RecordPathResult result = compiled.evaluate(record);
-
- final List<FieldValue> selectedFields = result.getSelectedFields().collect(Collectors.toList());
- return evalResults(selectedFields);
- }
-
- private Object eval(final Iterable<Record> records, final String recordPath) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final List<FieldValue> selectedFields = new ArrayList<>();
- for (final Record record : records) {
- final RecordPathResult result = compiled.evaluate(record);
- result.getSelectedFields().forEach(selectedFields::add);
- }
-
- return evalResults(selectedFields);
- }
-
- private Object eval(final Record[] records, final String recordPath) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final List<FieldValue> selectedFields = new ArrayList<>();
- for (final Record record : records) {
- final RecordPathResult result = compiled.evaluate(record);
- result.getSelectedFields().forEach(selectedFields::add);
- }
-
- return evalResults(selectedFields);
- }
-
- private Object evalResults(final List<FieldValue> selectedFields) {
- if (selectedFields.isEmpty()) {
- return null;
- }
-
- if (selectedFields.size() == 1) {
- return selectedFields.get(0).getValue();
- }
-
- return selectedFields.stream()
- .map(FieldValue::getValue)
- .toArray();
- }
-
- }
-
- public static class StringRecordPath extends RecordPathFunction {
- public String eval(Object record, String recordPath) {
- return eval(record, recordPath, Object::toString);
- }
- }
-
- public static class IntegerRecordPath extends RecordPathFunction {
- public Integer eval(Object record, String recordPath) {
- return eval(record, recordPath, val -> {
- if (val instanceof Number) {
- return ((Number) val).intValue();
- }
- if (val instanceof String) {
- return Integer.parseInt((String) val);
- }
- if (val instanceof Date) {
- return (int) ((Date) val).getTime();
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Integer against " + record
- + " because the value returned is of type " + val.getClass());
- });
- }
- }
-
- public static class LongRecordPath extends RecordPathFunction {
- public Long eval(Object record, String recordPath) {
- return eval(record, recordPath, val -> {
- if (val instanceof Number) {
- return ((Number) val).longValue();
- }
- if (val instanceof String) {
- return Long.parseLong((String) val);
- }
- if (val instanceof Date) {
- return ((Date) val).getTime();
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Long against " + record
- + " because the value returned is of type " + val.getClass());
- });
- }
- }
-
- public static class FloatRecordPath extends RecordPathFunction {
- public Float eval(Object record, String recordPath) {
- return eval(record, recordPath, val -> {
- if (val instanceof Number) {
- return ((Number) val).floatValue();
- }
- if (val instanceof String) {
- return Float.parseFloat((String) val);
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Float against " + record
- + " because the value returned is of type " + val.getClass());
- });
- }
- }
-
- public static class DoubleRecordPath extends RecordPathFunction {
- public Double eval(Object record, String recordPath) {
- return eval(record, recordPath, val -> {
- if (val instanceof Number) {
- return ((Number) val).doubleValue();
- }
- if (val instanceof String) {
- return Double.parseDouble((String) val);
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Double against " + record
- + " because the value returned is of type " + val.getClass());
- });
- }
- }
-
- public static class DateRecordPath extends RecordPathFunction {
- // Interestingly, Calcite throws an Exception if the schema indicates a DATE type and we return a java.util.Date. Calcite requires that a Long be returned instead.
- public Long eval(Object record, String recordPath) {
- return eval(record, recordPath, val -> {
- if (val instanceof Number) {
- return ((Number) val).longValue();
- }
- if (val instanceof String) {
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Date against " + record
- + " because the value returned is of type String. To parse a String value as a Date, please use the toDate function. For example, " +
- "SELECT RPATH_DATE( record, 'toDate( /event/timestamp, \"yyyy-MM-dd\" )' ) AS eventDate FROM FLOWFILE");
- }
- if (val instanceof Date) {
- return ((Date) val).getTime();
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Date against " + record
- + " because the value returned is of type " + val.getClass());
- });
- }
- }
-
-
- public static class RecordPathFunction {
- private static final RecordField ROOT_RECORD_FIELD = new RecordField("root", RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()));
- private static final RecordSchema ROOT_RECORD_SCHEMA = new SimpleRecordSchema(Collections.singletonList(ROOT_RECORD_FIELD));
- private static final RecordField PARENT_RECORD_FIELD = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(ROOT_RECORD_SCHEMA));
-
- protected static final RecordPathCache RECORD_PATH_CACHE = new RecordPathCache(100);
-
- protected <T> T eval(final Object record, final String recordPath, final Function<Object, T> transform) {
- if (record == null) {
- return null;
- }
-
- try {
- if (record instanceof Record) {
- return eval((Record) record, recordPath, transform);
- } else if (record instanceof Record[]) {
- return eval((Record[]) record, recordPath, transform);
- } else if (record instanceof Iterable) {
- return eval((Iterable<Record>) record, recordPath, transform);
- } else if (record instanceof Map) {
- return eval((Map<?, ?>) record, recordPath, transform);
- }
- } catch (IllegalArgumentException e) {
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against " + record, e);
- }
-
- throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against given argument because the argument is of type " + record.getClass() + " instead of Record");
- }
-
- private <T> T eval(final Map<?, ?> map, final String recordPath, final Function<Object, T> transform) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final Record record = new MapRecord(ROOT_RECORD_SCHEMA, Collections.singletonMap("root", map));
- final FieldValue parentFieldValue = new StandardFieldValue(record, PARENT_RECORD_FIELD, null);
- final FieldValue fieldValue = new StandardFieldValue(map, ROOT_RECORD_FIELD, parentFieldValue);
- final RecordPathResult result = compiled.evaluate(record, fieldValue);
-
- return evalResults(result.getSelectedFields(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
- }
-
-
- private <T> T eval(final Record record, final String recordPath, final Function<Object, T> transform) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
- final RecordPathResult result = compiled.evaluate((Record) record);
-
- return evalResults(result.getSelectedFields(), transform,
- () -> "RecordPath " + recordPath + " evaluated against " + record + " resulted in more than one return value. The RecordPath must be further constrained.");
- }
-
- private <T> T eval(final Record[] records, final String recordPath, final Function<Object, T> transform) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final List<FieldValue> selectedFields = new ArrayList<>();
- for (final Record record : records) {
- final RecordPathResult result = compiled.evaluate(record);
- result.getSelectedFields().forEach(selectedFields::add);
- }
-
- if (selectedFields.isEmpty()) {
- return null;
- }
-
- return evalResults(selectedFields.stream(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
- }
-
- private <T> T eval(final Iterable<Record> records, final String recordPath, final Function<Object, T> transform) {
- final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
-
- final List<FieldValue> selectedFields = new ArrayList<>();
- for (final Record record : records) {
- final RecordPathResult result = compiled.evaluate(record);
- result.getSelectedFields().forEach(selectedFields::add);
- }
-
- if (selectedFields.isEmpty()) {
- return null;
- }
-
- return evalResults(selectedFields.stream(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
- }
-
-
- private <T> T evalResults(final Stream<FieldValue> fields, final Function<Object, T> transform, final Supplier<String> multipleReturnValueErrorSupplier) {
- return fields.map(FieldValue::getValue)
- .filter(Objects::nonNull)
- .map(transform)
- .reduce((a, b) -> {
- // Only allow a single value
- throw new RuntimeException(multipleReturnValueErrorSupplier.get());
- })
- .orElse(null);
-
- }
- }
-
-
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunction.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunction.java
new file mode 100644
index 0000000..e0d84b3
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunction.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.calcite;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.record.path.util.RecordPathCache;
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+public class RecordPathFunction {
+ private static final RecordField ROOT_RECORD_FIELD = new RecordField("root", RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()));
+ private static final RecordSchema ROOT_RECORD_SCHEMA = new SimpleRecordSchema(Collections.singletonList(ROOT_RECORD_FIELD));
+ private static final RecordField PARENT_RECORD_FIELD = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(ROOT_RECORD_SCHEMA));
+
+ protected static final RecordPathCache RECORD_PATH_CACHE = new RecordPathCache(100);
+
+ protected <T> T eval(final Object record, final String recordPath, final Function<Object, T> transform) {
+ if (record == null) {
+ return null;
+ }
+
+ try {
+ if (record instanceof Record) {
+ return eval((Record) record, recordPath, transform);
+ } else if (record instanceof Record[]) {
+ return eval((Record[]) record, recordPath, transform);
+ } else if (record instanceof Iterable) {
+ return eval((Iterable<Record>) record, recordPath, transform);
+ } else if (record instanceof Map) {
+ return eval((Map<?, ?>) record, recordPath, transform);
+ }
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against " + record, e);
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against given argument because the argument is of type " + record.getClass() + " instead of Record");
+ }
+
+ private <T> T eval(final Map<?, ?> map, final String recordPath, final Function<Object, T> transform) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final Record record = new MapRecord(ROOT_RECORD_SCHEMA, Collections.singletonMap("root", map));
+ final FieldValue parentFieldValue = new StandardFieldValue(record, PARENT_RECORD_FIELD, null);
+ final FieldValue fieldValue = new StandardFieldValue(map, ROOT_RECORD_FIELD, parentFieldValue);
+ final RecordPathResult result = compiled.evaluate(record, fieldValue);
+
+ return evalResults(result.getSelectedFields(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
+ }
+
+
+ private <T> T eval(final Record record, final String recordPath, final Function<Object, T> transform) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+ final RecordPathResult result = compiled.evaluate((Record) record);
+
+ return evalResults(result.getSelectedFields(), transform,
+ () -> "RecordPath " + recordPath + " evaluated against " + record + " resulted in more than one return value. The RecordPath must be further constrained.");
+ }
+
+ private <T> T eval(final Record[] records, final String recordPath, final Function<Object, T> transform) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final List<FieldValue> selectedFields = new ArrayList<>();
+ for (final Record record : records) {
+ final RecordPathResult result = compiled.evaluate(record);
+ result.getSelectedFields().forEach(selectedFields::add);
+ }
+
+ if (selectedFields.isEmpty()) {
+ return null;
+ }
+
+ return evalResults(selectedFields.stream(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
+ }
+
+ private <T> T eval(final Iterable<Record> records, final String recordPath, final Function<Object, T> transform) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final List<FieldValue> selectedFields = new ArrayList<>();
+ for (final Record record : records) {
+ final RecordPathResult result = compiled.evaluate(record);
+ result.getSelectedFields().forEach(selectedFields::add);
+ }
+
+ if (selectedFields.isEmpty()) {
+ return null;
+ }
+
+ return evalResults(selectedFields.stream(), transform, () -> "RecordPath " + recordPath + " resulted in more than one return value. The RecordPath must be further constrained.");
+ }
+
+
+ private <T> T evalResults(final Stream<FieldValue> fields, final Function<Object, T> transform, final Supplier<String> multipleReturnValueErrorSupplier) {
+ return fields.map(FieldValue::getValue)
+ .filter(Objects::nonNull)
+ .map(transform)
+ .reduce((a, b) -> {
+ // Only allow a single value
+ throw new RuntimeException(multipleReturnValueErrorSupplier.get());
+ })
+ .orElse(null);
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java
new file mode 100644
index 0000000..30ad088
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/calcite/RecordPathFunctions.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.calcite;
+
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.ScalarFunctionImpl;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class RecordPathFunctions {
+ public static SchemaPlus createRootSchema(final CalciteConnection calciteConnection) {
+ final SchemaPlus rootSchema = calciteConnection.getRootSchema();
+ rootSchema.add("RPATH", ScalarFunctionImpl.create(ObjectRecordPath.class, "eval"));
+ rootSchema.add("RPATH_STRING", ScalarFunctionImpl.create(StringRecordPath.class, "eval"));
+ rootSchema.add("RPATH_INT", ScalarFunctionImpl.create(IntegerRecordPath.class, "eval"));
+ rootSchema.add("RPATH_LONG", ScalarFunctionImpl.create(LongRecordPath.class, "eval"));
+ rootSchema.add("RPATH_DATE", ScalarFunctionImpl.create(DateRecordPath.class, "eval"));
+ rootSchema.add("RPATH_DOUBLE", ScalarFunctionImpl.create(DoubleRecordPath.class, "eval"));
+ rootSchema.add("RPATH_FLOAT", ScalarFunctionImpl.create(FloatRecordPath.class, "eval"));
+
+ return rootSchema;
+ }
+
+ public static class ObjectRecordPath extends RecordPathFunction {
+ private static final RecordField ROOT_RECORD_FIELD = new RecordField("root", RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()));
+ private static final RecordSchema ROOT_RECORD_SCHEMA = new SimpleRecordSchema(Collections.singletonList(ROOT_RECORD_FIELD));
+ private static final RecordField PARENT_RECORD_FIELD = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(ROOT_RECORD_SCHEMA));
+
+
+ public Object eval(Object record, String recordPath) {
+ if (record == null) {
+ return null;
+ }
+ if (record instanceof Record) {
+ return eval((Record) record, recordPath);
+ }
+ if (record instanceof Record[]) {
+ return eval((Record[]) record, recordPath);
+ }
+ if (record instanceof Iterable) {
+ return eval((Iterable<Record>) record, recordPath);
+ }
+ if (record instanceof Map) {
+ return eval((Map<?, ?>) record, recordPath);
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " against given argument because the argument is of type " + record.getClass() + " instead of Record");
+ }
+
+ private Object eval(final Map<?, ?> map, final String recordPath) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final Record record = new MapRecord(ROOT_RECORD_SCHEMA, Collections.singletonMap("root", map));
+ final FieldValue parentFieldValue = new StandardFieldValue(record, PARENT_RECORD_FIELD, null);
+ final FieldValue fieldValue = new StandardFieldValue(map, ROOT_RECORD_FIELD, parentFieldValue);
+ final RecordPathResult result = compiled.evaluate(record, fieldValue);
+
+ final List<FieldValue> selectedFields = result.getSelectedFields().collect(Collectors.toList());
+ return evalResults(selectedFields);
+ }
+
+ private Object eval(final Record record, final String recordPath) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+ final RecordPathResult result = compiled.evaluate(record);
+
+ final List<FieldValue> selectedFields = result.getSelectedFields().collect(Collectors.toList());
+ return evalResults(selectedFields);
+ }
+
+ private Object eval(final Iterable<Record> records, final String recordPath) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final List<FieldValue> selectedFields = new ArrayList<>();
+ for (final Record record : records) {
+ final RecordPathResult result = compiled.evaluate(record);
+ result.getSelectedFields().forEach(selectedFields::add);
+ }
+
+ return evalResults(selectedFields);
+ }
+
+ private Object eval(final Record[] records, final String recordPath) {
+ final RecordPath compiled = RECORD_PATH_CACHE.getCompiled(recordPath);
+
+ final List<FieldValue> selectedFields = new ArrayList<>();
+ for (final Record record : records) {
+ final RecordPathResult result = compiled.evaluate(record);
+ result.getSelectedFields().forEach(selectedFields::add);
+ }
+
+ return evalResults(selectedFields);
+ }
+
+ private Object evalResults(final List<FieldValue> selectedFields) {
+ if (selectedFields.isEmpty()) {
+ return null;
+ }
+
+ if (selectedFields.size() == 1) {
+ return selectedFields.get(0).getValue();
+ }
+
+ return selectedFields.stream()
+ .map(FieldValue::getValue)
+ .toArray();
+ }
+
+ }
+
+ public static class StringRecordPath extends RecordPathFunction {
+ public String eval(Object record, String recordPath) {
+ return eval(record, recordPath, Object::toString);
+ }
+ }
+
+ public static class IntegerRecordPath extends RecordPathFunction {
+ public Integer eval(Object record, String recordPath) {
+ return eval(record, recordPath, val -> {
+ if (val instanceof Number) {
+ return ((Number) val).intValue();
+ }
+ if (val instanceof String) {
+ return Integer.parseInt((String) val);
+ }
+ if (val instanceof Date) {
+ return (int) ((Date) val).getTime();
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Integer against " + record
+ + " because the value returned is of type " + val.getClass());
+ });
+ }
+ }
+
+ public static class LongRecordPath extends RecordPathFunction {
+ public Long eval(Object record, String recordPath) {
+ return eval(record, recordPath, val -> {
+ if (val instanceof Number) {
+ return ((Number) val).longValue();
+ }
+ if (val instanceof String) {
+ return Long.parseLong((String) val);
+ }
+ if (val instanceof Date) {
+ return ((Date) val).getTime();
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Long against " + record
+ + " because the value returned is of type " + val.getClass());
+ });
+ }
+ }
+
+ public static class FloatRecordPath extends RecordPathFunction {
+ public Float eval(Object record, String recordPath) {
+ return eval(record, recordPath, val -> {
+ if (val instanceof Number) {
+ return ((Number) val).floatValue();
+ }
+ if (val instanceof String) {
+ return Float.parseFloat((String) val);
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Float against " + record
+ + " because the value returned is of type " + val.getClass());
+ });
+ }
+ }
+
+ public static class DoubleRecordPath extends RecordPathFunction {
+ public Double eval(Object record, String recordPath) {
+ return eval(record, recordPath, val -> {
+ if (val instanceof Number) {
+ return ((Number) val).doubleValue();
+ }
+ if (val instanceof String) {
+ return Double.parseDouble((String) val);
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Double against " + record
+ + " because the value returned is of type " + val.getClass());
+ });
+ }
+ }
+
+ public static class DateRecordPath extends RecordPathFunction {
+ // Interestingly, Calcite throws an Exception if the schema indicates a DATE type and we return a java.util.Date. Calcite requires that a Long be returned instead.
+ public Long eval(Object record, String recordPath) {
+ return eval(record, recordPath, val -> {
+ if (val instanceof Number) {
+ return ((Number) val).longValue();
+ }
+ if (val instanceof String) {
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Date against " + record
+ + " because the value returned is of type String. To parse a String value as a Date, please use the toDate function. For example, " +
+ "SELECT RPATH_DATE( record, 'toDate( /event/timestamp, \"yyyy-MM-dd\" )' ) AS eventDate FROM FLOWFILE");
+ }
+ if (val instanceof Date) {
+ return ((Date) val).getTime();
+ }
+
+ throw new RuntimeException("Cannot evaluate RecordPath " + recordPath + " as Date against " + record
+ + " because the value returned is of type " + val.getClass());
+ });
+ }
+ }
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/EnrichmentRole.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/EnrichmentRole.java
new file mode 100644
index 0000000..08cef25
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/EnrichmentRole.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+public enum EnrichmentRole {
+ ENRICHMENT,
+ ORIGINAL,
+ UNKNOWN
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/IndexCorrelatedJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/IndexCorrelatedJoinStrategy.java
new file mode 100644
index 0000000..78c4b91
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/IndexCorrelatedJoinStrategy.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public abstract class IndexCorrelatedJoinStrategy implements RecordJoinStrategy {
+ private final ComponentLog logger;
+
+ public IndexCorrelatedJoinStrategy(final ComponentLog logger) {
+ this.logger = logger;
+ }
+
+ protected ComponentLog getLogger() {
+ return logger;
+ }
+
+ @Override
+ public RecordJoinResult join(final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput, final ProcessSession session, final RecordSchema writerSchema) throws Exception {
+ final FlowFile originalFlowFile = originalInput.getFlowFile();
+ final FlowFile enrichmentFlowFile = enrichmentInput.getFlowFile();
+
+ InputStream originalIn = null;
+ RecordReader originalRecordReader = null;
+ InputStream enrichmentIn = null;
+ RecordReader enrichmentRecordReader = null;
+
+ try {
+ originalIn = session.read(originalFlowFile);
+ originalRecordReader = originalInput.getRecordReaderFactory().createRecordReader(originalFlowFile, originalIn, logger);
+
+ enrichmentIn = session.read(enrichmentFlowFile);
+ enrichmentRecordReader = enrichmentInput.getRecordReaderFactory().createRecordReader(enrichmentFlowFile, enrichmentIn, logger);
+
+ final Record firstOriginalRecord = originalRecordReader.nextRecord();
+ final Record firstEnrichmentRecord = enrichmentRecordReader.nextRecord();
+
+ final RecordSchema resultSchema = createResultSchema(firstOriginalRecord, firstEnrichmentRecord);
+
+ final InputStream finalOriginalIn = originalIn;
+ final RecordReader finalOriginalRecordReader = originalRecordReader;
+ final InputStream finalEnrichmentIn = enrichmentIn;
+ final RecordReader finalEnrichmentRecordReader = enrichmentRecordReader;
+
+ final RecordSet recordSet = new RecordSet() {
+ private boolean usedFirstRecords = false;
+
+ @Override
+ public RecordSchema getSchema() {
+ return resultSchema;
+ }
+
+ @Override
+ public Record next() throws IOException {
+ if (!usedFirstRecords) {
+ usedFirstRecords = true;
+ final Record combined = combineRecords(firstOriginalRecord, firstEnrichmentRecord, resultSchema);
+ return combined;
+ }
+
+ try {
+ final Record originalRecord = finalOriginalRecordReader.nextRecord();
+ final Record enrichmentRecord = finalEnrichmentRecordReader.nextRecord();
+
+ if (originalRecord == null && enrichmentRecord == null) {
+ return null;
+ }
+
+ final Record combined = combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ return combined;
+ } catch (final MalformedRecordException e) {
+ throw new IOException("Failed to read record", e);
+ }
+ }
+ };
+
+ return new RecordJoinResult() {
+ @Override
+ public RecordSet getRecordSet() {
+ return recordSet;
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(finalOriginalRecordReader, finalOriginalIn, finalEnrichmentRecordReader, finalEnrichmentIn);
+ }
+ };
+ } catch (final Throwable t) {
+ closeQuietly(originalRecordReader, originalIn, enrichmentRecordReader, enrichmentIn);
+ throw t;
+ }
+ }
+
+
+ private void closeQuietly(final AutoCloseable... closeables) {
+ for (final AutoCloseable closeable : closeables) {
+ closeQuietly(closeable);
+ }
+ }
+
+ private void closeQuietly(final AutoCloseable closeable) {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (final Exception e) {
+ logger.warn("Failed to close {}", closeable, e);
+ }
+ }
+ }
+
+ protected abstract Record combineRecords(Record originalRecord, Record enrichmentRecord, RecordSchema resultSchema);
+
+ protected abstract RecordSchema createResultSchema(Record firstOriginalRecord, Record firstEnrichmentRecord);
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/InsertRecordFieldsJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/InsertRecordFieldsJoinStrategy.java
new file mode 100644
index 0000000..7f09bf9
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/InsertRecordFieldsJoinStrategy.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.record.path.util.RecordPathCache;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class InsertRecordFieldsJoinStrategy extends IndexCorrelatedJoinStrategy {
+ private static final RecordPathCache recordPathCache = new RecordPathCache(100);
+ private final RecordPath recordPath;
+
+ public InsertRecordFieldsJoinStrategy(final ComponentLog logger, final String insertionRecordPath) {
+ super(logger);
+ recordPath = recordPathCache.getCompiled(insertionRecordPath);
+ }
+
+ @Override
+ protected Record combineRecords(final Record originalRecord, final Record enrichmentRecord, final RecordSchema resultSchema) {
+ if (originalRecord == null) {
+ return null;
+ }
+
+ if (enrichmentRecord == null) {
+ return originalRecord;
+ }
+
+ final RecordPathResult result = recordPath.evaluate(originalRecord);
+ final List<FieldValue> fieldValues = result.getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ final Object value = fieldValue.getValue();
+ if (!(value instanceof Record)) {
+ getLogger().debug("Did not find a Record after evaluating RecordPath {} but found {} instead", recordPath.getPath(), value);
+ continue;
+ }
+
+ final Record parentRecord = (Record) value;
+ enrichmentRecord.toMap().forEach(parentRecord::setValue);
+
+ parentRecord.incorporateInactiveFields();
+ }
+
+ return originalRecord;
+ }
+
+ @Override
+ protected RecordSchema createResultSchema(final Record firstOriginalRecord, final Record firstEnrichmentRecord) {
+ final Record combined = combineRecords(firstOriginalRecord, firstEnrichmentRecord, firstOriginalRecord.getSchema());
+ combined.incorporateInactiveFields();
+ return combined.getSchema();
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinInput.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinInput.java
new file mode 100644
index 0000000..0875642
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinInput.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+public class RecordJoinInput {
+ private final FlowFile flowFile;
+ private final RecordReaderFactory recordReaderFactory;
+ private final RecordSchema recordSchema;
+
+ public RecordJoinInput(final FlowFile flowFile, final RecordReaderFactory recordReaderFactory, final RecordSchema recordSchema) {
+ this.flowFile = flowFile;
+ this.recordReaderFactory = recordReaderFactory;
+ this.recordSchema = recordSchema;
+ }
+
+ public FlowFile getFlowFile() {
+ return flowFile;
+ }
+
+ public RecordReaderFactory getRecordReaderFactory() {
+ return recordReaderFactory;
+ }
+
+ public RecordSchema getRecordSchema() {
+ return recordSchema;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinResult.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinResult.java
new file mode 100644
index 0000000..d578ab8
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinResult.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.serialization.record.RecordSet;
+
+public interface RecordJoinResult extends AutoCloseable {
+
+ RecordSet getRecordSet();
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinStrategy.java
new file mode 100644
index 0000000..50b90a4
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/RecordJoinStrategy.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+public interface RecordJoinStrategy {
+ RecordJoinResult join(RecordJoinInput originalInput, RecordJoinInput enrichmentInput, ProcessSession session, RecordSchema outputSchema) throws Exception;
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java
new file mode 100644
index 0000000..a53f21a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCache.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import org.apache.calcite.config.CalciteConnectionProperty;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processors.standard.calcite.RecordPathFunctions;
+import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.util.Tuple;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+public class SqlJoinCache implements AutoCloseable {
+ private final ComponentLog logger;
+ private final Cache<Tuple<String, RecordSchema>, BlockingQueue<SqlJoinCalciteParameters>> calciteParameterQueues = Caffeine.newBuilder()
+ .maximumSize(25)
+ .removalListener(this::onCacheEviction)
+ .build();
+
+
+ public SqlJoinCache(final ComponentLog logger) {
+ this.logger = logger;
+ }
+
+ public SqlJoinCalciteParameters getCalciteParameters(final String sql, final ProcessSession session, final RecordSchema schema, final RecordJoinInput originalInput,
+ final RecordJoinInput enrichmentInput) throws SQLException {
+ final Tuple<String, RecordSchema> tuple = new Tuple<>(sql, schema);
+ final BlockingQueue<SqlJoinCalciteParameters> queue = calciteParameterQueues.get(tuple, key -> new LinkedBlockingQueue<>());
+
+ final SqlJoinCalciteParameters cachedStmt = queue.poll();
+ if (cachedStmt != null) {
+ return cachedStmt;
+ }
+
+ return createCalciteParameters(sql, session, originalInput, enrichmentInput, queue);
+ }
+
+ public void returnCalciteParameters(final String sql, final RecordSchema schema, final SqlJoinCalciteParameters parameters) {
+ final Tuple<String, RecordSchema> tuple = new Tuple<>(sql, schema);
+ final BlockingQueue<SqlJoinCalciteParameters> queue = calciteParameterQueues.getIfPresent(tuple);
+
+ if (queue == null || !queue.offer(parameters)) {
+ parameters.close();
+ }
+ }
+
+ private SqlJoinCalciteParameters createCalciteParameters(final String sql, final ProcessSession session, final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput,
+ final BlockingQueue<SqlJoinCalciteParameters> parameterQueue) throws SQLException {
+ final CalciteConnection connection = createCalciteConnection();
+
+ final SchemaPlus rootSchema = RecordPathFunctions.createRootSchema(connection);
+
+ final FlowFileTable originalTable = new FlowFileTable(session, originalInput.getFlowFile(), originalInput.getRecordSchema(), originalInput.getRecordReaderFactory(), logger);
+ rootSchema.add("ORIGINAL", originalTable);
+
+ final FlowFileTable enrichmentTable = new FlowFileTable(session, enrichmentInput.getFlowFile(), enrichmentInput.getRecordSchema(), enrichmentInput.getRecordReaderFactory(), logger);
+ rootSchema.add("ENRICHMENT", enrichmentTable);
+
+ rootSchema.setCacheEnabled(false);
+
+ final PreparedStatement preparedStatement = connection.prepareStatement(sql);
+ return new SqlJoinCalciteParameters(sql, connection, preparedStatement, originalTable, enrichmentTable);
+ }
+
+ private CalciteConnection createCalciteConnection() {
+ final Properties properties = new Properties();
+ properties.put(CalciteConnectionProperty.LEX.camelName(), Lex.MYSQL_ANSI.name());
+
+ try {
+ final Connection connection = DriverManager.getConnection("jdbc:calcite:", properties);
+ final CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ return calciteConnection;
+ } catch (final Exception e) {
+ throw new ProcessException(e);
+ }
+ }
+
+ private void onCacheEviction(final Tuple<String, RecordSchema> key, final BlockingQueue<SqlJoinCalciteParameters> queue, final RemovalCause cause) {
+ clearQueue(queue);
+ }
+
+ private void clearQueue(final BlockingQueue<SqlJoinCalciteParameters> parameterQueue) {
+ SqlJoinCalciteParameters parameters;
+ while ((parameters = parameterQueue.poll()) != null) {
+ parameters.close();
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ for (final BlockingQueue<SqlJoinCalciteParameters> statementQueue : calciteParameterQueues.asMap().values()) {
+ clearQueue(statementQueue);
+ }
+
+ calciteParameterQueues.invalidateAll();
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java
new file mode 100644
index 0000000..b845c3d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinCalciteParameters.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.nifi.queryrecord.FlowFileTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.PreparedStatement;
+
+public class SqlJoinCalciteParameters implements AutoCloseable {
+ private static final Logger logger = LoggerFactory.getLogger(SqlJoinCalciteParameters.class);
+
+ private final String sql;
+ private final CalciteConnection connection;
+ private final PreparedStatement preparedStatement;
+ private final FlowFileTable originalTable;
+ private final FlowFileTable enrichmentTable;
+
+ public SqlJoinCalciteParameters(final String sql, final CalciteConnection connection, final PreparedStatement preparedStatement,
+ final FlowFileTable originalTable, final FlowFileTable enrichmentTable) {
+ this.sql = sql;
+ this.connection = connection;
+ this.preparedStatement = preparedStatement;
+ this.originalTable = originalTable;
+ this.enrichmentTable = enrichmentTable;
+ }
+
+ public String getSql() {
+ return sql;
+ }
+
+ public CalciteConnection getConnection() {
+ return connection;
+ }
+
+ public PreparedStatement getPreparedStatement() {
+ return preparedStatement;
+ }
+
+ public FlowFileTable getOriginalTable() {
+ return originalTable;
+ }
+
+ public FlowFileTable getEnrichmentTable() {
+ return enrichmentTable;
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(preparedStatement, "Calcite Prepared Statement");
+ closeQuietly(connection, "Calcite Connection");
+ closeQuietly(originalTable, "Calcite 'Original' Table");
+ closeQuietly(enrichmentTable, "Calcite 'Enrichment' Table");
+ }
+
+ private void closeQuietly(final AutoCloseable closeable, final String description) {
+ if (closeable == null) {
+ return;
+ }
+
+ try {
+ closeable.close();
+ } catch (final Exception e) {
+ logger.warn("Failed to close {}", description, e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java
new file mode 100644
index 0000000..1db67f7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/SqlJoinStrategy.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.queryrecord.FlowFileTable;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+import org.apache.nifi.serialization.record.ResultSetRecordSet;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SqlJoinStrategy implements RecordJoinStrategy {
+
+ private final SqlJoinCache cache;
+ private final ComponentLog logger;
+ private final String sql;
+ private final int defaultPrecision;
+ private final int defaultScale;
+
+ public SqlJoinStrategy(final SqlJoinCache cache, final String sql, final ComponentLog logger, final int defaultPrecision, final int defaultScale) {
+ this.cache = cache;
+ this.sql = sql;
+ this.logger = logger;
+ this.defaultPrecision = defaultPrecision;
+ this.defaultScale = defaultScale;
+ }
+
+ @Override
+ public RecordJoinResult join(final RecordJoinInput originalInput, final RecordJoinInput enrichmentInput, final ProcessSession session, final RecordSchema outputSchema) throws SQLException {
+ final SqlJoinCalciteParameters calciteParameters = cache.getCalciteParameters(sql, session, outputSchema, originalInput, enrichmentInput);
+
+ final FlowFileTable originalTable = calciteParameters.getOriginalTable();
+ final FlowFileTable enrichmentTable = calciteParameters.getEnrichmentTable();
+
+ originalTable.setFlowFile(session, originalInput.getFlowFile());
+ enrichmentTable.setFlowFile(session, enrichmentInput.getFlowFile());
+
+ final PreparedStatement stmt = calciteParameters.getPreparedStatement();
+
+ final ResultSet rs;
+ try {
+ rs = stmt.executeQuery();
+ } catch (final Throwable t) {
+ originalTable.close();
+ enrichmentTable.close();
+
+ throw t;
+ }
+
+ final RecordSet recordSet = new ResultSetRecordSet(rs, outputSchema, defaultPrecision, defaultScale, true);
+
+ // Create a RecordJoinResult that allows us to return our RecordSet and also close all resources when they are no longer needed
+ return new RecordJoinResult() {
+ @Override
+ public void close() {
+ closeQuietly(originalTable, enrichmentTable);
+ cache.returnCalciteParameters(sql, outputSchema, calciteParameters);
+ }
+
+ @Override
+ public RecordSet getRecordSet() {
+ return recordSet;
+ }
+ };
+ }
+
+ private void closeQuietly(final AutoCloseable... closeables) {
+ for (final AutoCloseable closeable : closeables) {
+ closeQuietly(closeable);
+ }
+ }
+
+ private void closeQuietly(final AutoCloseable closeable) {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (final Exception e) {
+ logger.warn("Failed to close {}", closeable, e);
+ }
+ }
+ }
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/WrapperJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/WrapperJoinStrategy.java
new file mode 100644
index 0000000..dc23a9e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/enrichment/WrapperJoinStrategy.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class WrapperJoinStrategy extends IndexCorrelatedJoinStrategy {
+ private static final String ORIGINAL_FIELD_NAME = "original";
+ private static final String ENRICHMENT_FIELD_NAME = "enrichment";
+
+ public WrapperJoinStrategy(final ComponentLog logger) {
+ super(logger);
+ }
+
+ @Override
+ protected Record combineRecords(final Record originalRecord, final Record enrichmentRecord, final RecordSchema resultSchema) {
+ if (originalRecord == null && enrichmentRecord == null) {
+ return null;
+ }
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put(ORIGINAL_FIELD_NAME, originalRecord);
+ values.put(ENRICHMENT_FIELD_NAME, enrichmentRecord);
+
+ final Record wrapped = new MapRecord(resultSchema, values);
+ return wrapped;
+ }
+
+ protected RecordSchema createResultSchema(final Record firstOriginalRecord, final Record firstEnrichmentRecord) {
+ final List<RecordField> fields = new ArrayList<>();
+
+ if (firstOriginalRecord != null) {
+ fields.add(new RecordField(ORIGINAL_FIELD_NAME, RecordFieldType.RECORD.getRecordDataType(firstOriginalRecord.getSchema())));
+ }
+
+ if (firstEnrichmentRecord != null) {
+ fields.add(new RecordField(ENRICHMENT_FIELD_NAME, RecordFieldType.RECORD.getRecordDataType(firstEnrichmentRecord.getSchema())));
+ }
+
+ final RecordSchema recordSetSchema = new SimpleRecordSchema(fields);
+ return recordSetSchema;
+ }
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java
index 20c2dbf..a2debbd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/queryrecord/FlowFileTable.java
@@ -45,6 +45,7 @@ import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.type.ArrayDataType;
import org.apache.nifi.serialization.record.type.ChoiceDataType;
+import java.io.Closeable;
import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -54,7 +55,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-public class FlowFileTable extends AbstractTable implements QueryableTable, TranslatableTable {
+public class FlowFileTable extends AbstractTable implements QueryableTable, TranslatableTable, Closeable {
private final RecordReaderFactory recordReaderFactory;
private final ComponentLog logger;
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index 6a767c1..19b5653 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -45,6 +45,7 @@ org.apache.nifi.processors.standard.FetchFTP
org.apache.nifi.processors.standard.FetchSFTP
org.apache.nifi.processors.standard.FlattenJson
org.apache.nifi.processors.standard.ForkRecord
+org.apache.nifi.processors.standard.ForkEnrichment
org.apache.nifi.processors.standard.GenerateFlowFile
org.apache.nifi.processors.standard.GenerateTableFetch
org.apache.nifi.processors.standard.GetFile
@@ -60,6 +61,7 @@ org.apache.nifi.processors.standard.HashContent
org.apache.nifi.processors.standard.IdentifyMimeType
org.apache.nifi.processors.standard.InvokeHTTP
org.apache.nifi.processors.standard.JoltTransformJSON
+org.apache.nifi.processors.standard.JoinEnrichment
org.apache.nifi.processors.standard.ListDatabaseTables
org.apache.nifi.processors.standard.ListenFTP
org.apache.nifi.processors.standard.ListenHTTP
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/additionalDetails.html
new file mode 100644
index 0000000..85f665d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/additionalDetails.html
@@ -0,0 +1,84 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>ForkEnrichment</title>
+
+ <link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+
+<h3>Introduction</h3>
+
+<p>
+The ForkEnrichment processor is designed to be used in conjunction with the <a href="../org.apache.nifi.processors.standard.JoinEnrichment/index.html">JoinEnrichment Processor</a>.
+Used together, they provide a powerful mechanism for transforming data into a separate request payload for gathering enrichment data, gathering that enrichment data, optionally transforming
+the enrichment data, and finally joining together the original payload with the enrichment data.
+</p>
+
+
+
+<h3>Typical Dataflow</h3>
+
+<p>
+ A typical dataflow for accomplishing this may look something like this:
+</p>
+
+<img src="fork-join-enrichment.png" style="height: 50%; width: 50%" />
+
+<p>
+ Here, we have a ForkEnrichment processor that is responsible for taking in a FlowFile and producing two copies of it: one to the "original" relationship and the other to the "enrichment"
+ relationship. Each copy will have its own set of attributes added to it.
+</p>
+
+<p>
+ Next, we have the "original" FlowFile being routed to the JoinEnrichment processor, while the "enrichment" FlowFile is routed in a different direction. Each of these FlowFiles will have an
+ attribute named "enrichment.group.id" with the same value. The JoinEnrichment processor then uses this information to correlate the two FlowFiles. The "enrichment.role" attribute will also
+ be added to each FlowFile but with a different value. The FlowFile routed to "original" will have an enrichment.role of ORIGINAL while the FlowFile routed to "enrichment" will have an
+ enrichment.role of ENRICHMENT.
+</p>
+
+<p>
+ The Processors that make up the "enrichment" path will vary from use case to use case. In this example, we use
+ <a href="../org.apache.nifi.processors.standard.JoltTransformJSON/index.html">JoltTransformJSON</a> processor in order to transform our payload from the original payload into a payload that is
+ expected by our web service. We then use the <a href="../org.apache.nifi.processors.standard.InvokeHTTP/index.html">InvokeHTTP</a> processor in order to gather
+ enrichment data that is relevant to our use case. Other common processors to use in this path include
+ <a href="../org.apache.nifi.processors.standard.QueryRecord/index.html">QueryRecord</a>, <a href="../org.apache.nifi.processors.standard.UpdateRecord/index.html">UpdateRecord</a>,
+ <a href="../org.apache.nifi.processors.standard.ReplaceText/index.html">ReplaceText</a>, JoltTransformRecord, and ScriptedTransformRecord.
+ It is also be a common use case to transform the response from the web service that is invoked via InvokeHTTP using one or more of these processors.
+</p>
+
+<p>
+ After the enrichment data has been gathered, it does us little good unless we are able to somehow combine our enrichment data back with our original payload.
+ To achieve this, we use the JoinEnrichment processor. It is responsible for combining records from both the "original" FlowFile and the "enrichment" FlowFile.
+</p>
+
+<p>
+ The JoinEnrichment Processor is configured with a separate RecordReader for the "original" FlowFile and for the "enrichment" FlowFile. This means that the original data and the
+ enrichment data can have entirely different schemas and can even be in different data formats. For example, our original payload may be CSV data, while our enrichment data is a JSON
+ payload. Because we make use of RecordReaders, this is entirely okay. The Processor also requires a RecordWriter to use for writing out the enriched payload (i.e., the payload that contains
+ the join of both the "original" and the "enrichment" data).
+</p>
+
+<p>
+ For details on how to join the original payload with the enrichment data, see the Additional Details of the
+ <a href="../org.apache.nifi.processors.standard.JoinEnrichment/index.html">JoinEnrichment Processor</a> documentation.
+</p>
+
+</body>
+</html>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/fork-join-enrichment.png b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/fork-join-enrichment.png
new file mode 100644
index 0000000..ab54aec
Binary files /dev/null and b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.ForkEnrichment/fork-join-enrichment.png differ
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/additionalDetails.html
new file mode 100644
index 0000000..842d6ff
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/additionalDetails.html
@@ -0,0 +1,524 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>JoinEnrichment</title>
+
+ <link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+
+<h3>Introduction</h3>
+
+<p>
+ The JoinEnrichment processor is designed to be used in conjunction with the <a href="../org.apache.nifi.processors.standard.ForkEnrichment/index.html">ForkEnrichment Processor</a>.
+ Used together, they provide a powerful mechanism for transforming data into a separate request payload for gathering enrichment data, gathering that enrichment data, optionally transforming
+ the enrichment data, and finally joining together the original payload with the enrichment data.
+</p>
+
+
+
+<h3>Typical Dataflow</h3>
+
+<p>
+ A typical dataflow for accomplishing this may look something like this:
+</p>
+
+<img src="fork-join-enrichment.png" style="height: 50%; width: 50%" />
+
+<p>
+ Here, we have a ForkEnrichment processor that is responsible for taking in a FlowFile and producing two copies of it: one to the "original" relationship and the other to the "enrichment"
+ relationship. Each copy will have its own set of attributes added to it.
+</p>
+
+<p>
+ Next, we have the "original" FlowFile being routed to the JoinEnrichment processor, while the "enrichment" FlowFile is routed in a different direction. Each of these FlowFiles will have an
+ attribute named "enrichment.group.id" with the same value. The JoinEnrichment processor then uses this information to correlate the two FlowFiles. The "enrichment.role" attribute will also
+ be added to each FlowFile but with a different value. The FlowFile routed to "original" will have an enrichment.role of ORIGINAL while the FlowFile routed to "enrichment" will have an
+ enrichment.role of ENRICHMENT.
+</p>
+
+<p>
+ The Processors that make up the "enrichment" path will vary from use case to use case. In this example, we use
+ <a href="../org.apache.nifi.processors.standard.JoltTransformJSON/index.html">JoltTransformJSON</a> processor in order to transform our payload from the original payload into a payload that is
+ expected by our web service. We then use the <a href="../org.apache.nifi.processors.standard.InvokeHTTP/index.html">InvokeHTTP</a> processor in order to gather
+ enrichment data that is relevant to our use case. Other common processors to use in this path include
+ <a href="../org.apache.nifi.processors.standard.QueryRecord/index.html">QueryRecord</a>, <a href="../org.apache.nifi.processors.standard.UpdateRecord/index.html">UpdateRecord</a>,
+ <a href="../org.apache.nifi.processors.standard.ReplaceText/index.html">ReplaceText</a>, JoltTransformRecord, and ScriptedTransformRecord.
+ It is also be a common use case to transform the response from the web service that is invoked via InvokeHTTP using one or more of these processors.
+</p>
+
+<p>
+ After the enrichment data has been gathered, it does us little good unless we are able to somehow combine our enrichment data back with our original payload.
+ To achieve this, we use the JoinEnrichment processor. It is responsible for combining records from both the "original" FlowFile and the "enrichment" FlowFile.
+</p>
+
+<p>
+ The JoinEnrichment Processor is configured with a separate RecordReader for the "original" FlowFile and for the "enrichment" FlowFile. This means that the original data and the
+ enrichment data can have entirely different schemas and can even be in different data formats. For example, our original payload may be CSV data, while our enrichment data is a JSON
+ payload. Because we make use of RecordReaders, this is entirely okay. The Processor also requires a RecordWriter to use for writing out the enriched payload (i.e., the payload that contains
+ the join of both the "original" and the "enrichment" data).
+</p>
+
+<p>
+ The JoinEnrichment Processor offers different strategies for how to combine the original records with the enrichment data. Each of these is explained here in some detail.
+</p>
+
+<h3>Wrapper</h3>
+The Wrapper strategy is the default. Each record in the original payload is expected to have a corresponding record in the enrichment payload. The output record will be a record with two
+fields: <code>original</code> and <code>enrichment</code>. Each of these will contain the <i>n</i>th record from the corresponding FlowFile. For example, if the original FlowFile has the following
+content:
+
+<pre><code>
+id, name, age
+28021, John Doe, 55
+832, Jane Doe, 22
+29201, Jake Doe, 23
+555, Joseph Doe, 2
+</code></pre>
+
+And our enrichment FlowFile has the following content:
+
+<pre><code>
+id, email
+28021, john.doe@nifi.apache.org
+832, jane.doe@nifi.apache.org
+29201, jake.doe@nifi.apache.org
+</code></pre>
+
+This strategy would produce output the looks like this (assuming a JSON Writer):
+<pre><code>
+[
+ {
+ "original": {
+ "id": 28021,
+ "name": "John Doe",
+ "age": 55
+ },
+ "enrichment": {
+ "id": 28021,
+ "email": "john.doe@nifi.apache.org"
+ }
+ }, {
+ "original": {
+ "id": 832,
+ "name": "Jane Doe",
+ "age": 22
+ },
+ "enrichment": {
+ "id": 832,
+ "email": "jane.doe@nifi.apache.org"
+ }
+ }, {
+ "original": {
+ "id": 29201,
+ "name": "Jake Doe",
+ "age": 23
+ },
+ "enrichment": {
+ "id": 29201,
+ "email": "jake.doe@nifi.apache.org"
+ }
+ }, {
+ "original": {
+ "id": 555,
+ "name": "Joseph Doe",
+ "age": 2
+ },
+ "enrichment": null
+ }
+]
+</code></pre>
+
+<p>
+ With this strategy, the first record of the original FlowFile is coupled together with the first record of the enrichment FlowFile. The second record of the original FlowFile is coupled
+ together with the second record of the enrichment FlowFile, and so on. If one of the FlowFiles has more records than the other, a <code>null</code> value will be used.
+</p>
+
+
+
+<h3>Insert Enrichment Fields</h3>
+
+<p>
+ The "Insert Enrichment Fields" strategy inserts all of the fields of the "enrichment" record into the original record. The records are correlated by their index in the FlowFile. That is,
+ the first record in the "enrichment" FlowFile is inserted into the first record in the "original" FlowFile. The second record of the "enrichment" FlowFile is inserted into the second
+ record of the "original" FlowFile and so on.
+</p>
+
+<p>
+ When this strategy is selected, the "Record Path" property is required. The Record Path is evaluated against the "original" record. Consider, for example, the following content for the
+ "original" FlowFile:
+</p>
+
+<pre><code>
+[{
+ "purchase": {
+ "customer": {
+ "loyaltyId": 48202,
+ "firstName": "John",
+ "lastName": "Doe"
+ },
+ "total": 48.28,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 24.14,
+ "quantity": 2
+ }
+ ]
+ }
+}, {
+ "purchase": {
+ "customer": {
+ "loyaltyId": 5512,
+ "firstName": "Jane",
+ "lastName": "Doe"
+ },
+ "total": 121.44,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 28.15,
+ "quantity": 4
+ }, {
+ "itemDescription": "inkpen",
+ "price": 4.42,
+ "quantity": 2
+ }
+ ]
+ }
+}]
+</code></pre>
+
+Joined using the following enrichment content:
+<pre><code>
+[
+ {
+ "customerDetails": {
+ "id": 48202,
+ "phone": "555-555-5555",
+ "email": "john.doe@nifi.apache.org"
+ }
+ }, {
+ "customerDetails": {
+ "id": 5512,
+ "phone": "555-555-5511",
+ "email": "jane.doe@nifi.apache.org"
+ }
+ }
+]
+</code></pre>
+
+<p>
+ Let us then consider that a Record Path is used with a value of "/purchase/customer".
+ This would yield the following results:
+</p>
+
+<pre><code>
+[{
+ "purchase": {
+ "customer": {
+ "loyaltyId": 48202,
+ "firstName": "John",
+ "lastName": "Doe",
+ "customerDetails": {
+ "id": 48202,
+ "phone": "555-555-5555",
+ "email": "john.doe@nifi.apache.org"
+ }
+ },
+ "total": 48.28,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 24.14,
+ "quantity": 2
+ }
+ ]
+ }
+}, {
+ "purchase": {
+ "customer": {
+ "loyaltyId": 5512,
+ "firstName": "Jane",
+ "lastName": "Doe",
+ "customerDetails": {
+ "id": 5512,
+ "phone": "555-555-5511",
+ "email": "jane.doe@nifi.apache.org"
+ }
+ },
+ "total": 121.44,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 28.15,
+ "quantity": 4
+ }, {
+ "itemDescription": "inkpen",
+ "price": 4.42,
+ "quantity": 2
+ }
+ ]
+ }
+}]
+</code></pre>
+
+
+
+<h3>SQL</h3>
+
+<p>
+The SQL strategy provides an important capability that differs from the others, in that it allows for correlating the records in the "original" FlowFile and the records in the "enrichment" FlowFile
+in ways other than index based. That is, the SQL-based strategy doesn't necessarily correlate the first record of the original FlowFile with the first record of the enrichment FlowFile. Instead, it
+allows the records to be correlated using standard SQL JOIN expressions.
+</p>
+
+<p>
+A common use case for this is to create a payload to query some web service. The response contains identifiers with additional information for enrichment, but the order of the records in the
+enrichment may not correspond to the order of the records in the original.
+</p>
+
+<p>
+As an example, consider the following original payload, in CSV:
+</p>
+
+<pre><code>
+id, name, age
+28021, John Doe, 55
+832, Jane Doe, 22
+29201, Jake Doe, 23
+555, Joseph Doe, 2
+</code></pre>
+
+<p>
+Additionally, consider the following payload for the enrichment data:
+</p>
+
+<pre><code>
+customer_id, customer_email, customer_name, customer_since
+555, joseph.doe@nifi.apache.org, Joe Doe, 08/Dec/14
+832, jane.doe@nifi.apache.org, Mrs. Doe, 14/Nov/14
+28021, john.doe@nifi.apache.org, John Doe, 22/Jan/22
+</code></pre>
+
+<p>
+When making use of the SQL strategy, we must provide a SQL SELECT statement to combine both our original data and our enrichment data into a single FlowFile. To do this, we treat our original
+FlowFile as its own table with the name "original" while we treat the enrichment data as its own table with the name "enrichment".
+</p>
+
+<p>
+Given this, we might combine all of the data using a simple query such as:
+</p>
+<pre><code>
+SELECT o.*, e.*
+FROM original o
+JOIN enrichment e
+ON o.id = e.customer_id
+</code></pre>
+
+<p>
+And this would provide the following output:
+</p>
+
+<pre><code>
+id, name, age, customer_id, customer_email, customer_name, customer_since
+28021, John Doe, 55, 28021, john.doe@nifi.apache.org, John Doe, 22/Jan/22
+832, Jane Doe, 22, 832, jane.doe@nifi.apache.org, Mrs. Doe, 14/Nov/14
+555, Joseph Doe, 2, 555, joseph.doe@nifi.apache.org, Joe Doe, 08/Dec/14
+</code></pre>
+
+<p>
+Note that in this case, the record for Jake Doe was removed because we used a JOIN, rather than an OUTER JOIN. We could instead use a LEFT OUTER JOIN to ensure that we retain all records from the
+original FlowFile and simply provide null values for any missing records in the enrichment:
+</p>
+
+<pre><code>
+SELECT o.*, e.*
+FROM original o
+LEFT OUTER JOIN enrichment e
+ON o.id = e.customer_id
+</code></pre>
+
+<p>
+Which would produce the following output:
+</p>
+
+<pre><code>
+id, name, age, customer_id, customer_email, customer_name, customer_since
+28021, John Doe, 55, 28021, john.doe@nifi.apache.org, John Doe, 22/Jan/22
+832, Jane Doe, 22, 832, jane.doe@nifi.apache.org, Mrs. Doe, 14/Nov/14
+29201, Jake Doe, 23,,,,
+555, Joseph Doe, 2, 555, joseph.doe@nifi.apache.org, Joe Doe, 08/Dec/14
+</code></pre>
+
+<p>
+But SQL is far more expressive than this, allowing us to perform far more powerful expressions. In this case, we probably don't want both the "id" and "customer_id" fields, or the "name" and
+"customer_name" fields. Let's consider, though, that the enrichment provides the customer's preferred name instead of their legal name. We might want to drop the customer_since column, as it
+doesn't make sense for our use case. We might then change our SQL to the following:
+</p>
+
+<pre><code>
+SELECT o.id, o.name, e.customer_name AS preferred_name, o.age, e.customer_email AS email
+FROM original o
+LEFT OUTER JOIN enrichment e
+ON o.id = e.customer_id
+</code></pre>
+
+<p>
+And this will produce a more convenient output:
+</p>
+
+<pre><code>
+id, name, preferred_name, age, email
+28021, John Doe, John Doe, 55, john.doe@nifi.apache.org
+832, Jane Doe, Mrs. Doe, 22, jane.doe@nifi.apache.org
+29201, Jake Doe,, 23,
+555, Joseph Doe, Joe Doe, 2, joseph.doe@nifi.apache.org
+</code></pre>
+
+<p>
+So we can see tremendous power from the SQL strategy. However, there is a very important consideration that must be taken into account when using the SQL strategy.
+</p>
+
+<p>
+<b>WARNING:</b> while the SQL strategy provides us great power, it may require significant amounts of heap. Depending on the query, the SQL engine may require buffering the contents of the entire
+"enrichment" FlowFile in memory, in Java's heap. Additionally, if the Processor is scheduled with multiple concurrent tasks, each of the tasks made hold the entire contents of the enrichment
+FlowFile in memory. This can lead to heap exhaustion and cause stability problems or OutOfMemoryErrors to occur.
+</p>
+
+<p>
+There are a couple of options that will help to mitigate these concerns.
+</p>
+<ol>
+ <li>
+ Split into smaller chunks. It is generally ill-advised to split Record-oriented data into many tiny FlowFiles, as NiFi tends to perform best with larger FlowFiles. The sweet spot for NiFi
+ tends to be around 300 KB to 3 MB in size. So we do not want to break a large FlowFile with 100,000 records into 100,000 FlowFiles each with 1 record. It may be advantageous, though, before
+ the ForkEnrichment processor to break that FlowFile into 100 FlowFiles, each 1,000 records; or 10 FlowFiles, each 10,000 records. This typically results in a smaller amount of enrichment
+ data so that we don't need to hold as much in memory.
+ </li>
+ <li>
+ Before the JoinEnrichment processor, trim the enrichment data to remove any fields that are not desirable. In the example above, we may have used QueryRecord, UpdateRecord,
+ JoltTransformRecord, or updated our schema in order to remove the "customer_since" field from the enrichment dataset. Because we didn't make use of the field, we could easily remove it
+ before the JoinEnrichment in order to reduce the size of the enrichment FlowFile and thereby reduce the amount of data held in memory.
+ </li>
+</ol>
+
+<p>
+It is also worth noting that the SQL strategy may result in reordering the records within the FlowFile, so it may be necessary to use an ORDER BY clause, etc. if the ordering is important.
+</p>
+
+
+
+
+<h3>Additional Memory Considerations</h3>
+
+<p>
+In addition to the warning above about using the SQL Join Strategy, there is another consideration to keep in mind in order to limit the amount of information that this Processor must keep in
+memory. While the Processor does not store the contents of all FlowFiles in memory, it does hold all FlowFiles' attributes in memory. As a result, the following points should be kept in mind when
+using this Processor.
+</p>
+
+<ol>
+ <li>
+ Avoid large attributes. FlowFile attributes should not be used to hold FlowFile content. Attributes are intended to be small. Generally, on the order of 100-200 characters. If
+ there are any large attributes, it is recommended that they be removed by using the
+ <a href="../org.apache.nifi.processors.attributes.UpdateAttributes/index.html">UpdateAttribute Processor</a> before the ForkEnrichment processor.
+ </li>
+ <li>
+ Avoid large numbers of attributes. While it is important to avoid creating large FlowFile attributes, it is just as important to avoid creating large numbers of attributes. Keeping 30
+ small attributes on a FlowFile is perfectly fine. Storing 300 attributes, on the other hand, may occupy a significant amount of heap.
+ </li>
+ <li>
+ Limit backpressure. The JoinEnrichment Processor will pull into its own memory all of the incoming FlowFiles. As a result, it will be helpful to avoid providing a huge number of FlowFiles
+ to the Processor at any given time. This can be done by setting the backpressure limits to a smaller value. For example, in our example above, the ForkEnrichment Processor is connected
+ directly to the JoinEnrichment Processor. We may want to limit the backpressure on this connection to 500 or 1,000 instead of the default 10,000. Doing so will limit the number of FlowFiles
+ that are allowed to be loaded into the JoinEnrichment Processor at one time.
+ </li>
+</ol>
+
+
+<h3>More Complex Joining Strategies</h3>
+<p>
+This Processor offers several strategies that can be used for correlating data together and joining records from two different FlowFiles into a single FlowFile. However, there are times
+when users may require more powerful capabilities than what is offered. We might, for example, want to use the information in an enrichment record to determine whether or not to null out a value in
+the corresponding original records.
+</p>
+
+<p>
+For such uses cases, the recommended approach is to make use of the Wrapper strategy or the SQL strategy in order to combine the original and enrichment FlowFiles into a single FlowFile. Then,
+connect the "joined" relationship of this Processor to the most appropriate processor for further processing the data. For example, consider that we use the Wrapper strategy to produce output that
+looks like this:
+</p>
+
+<pre><code>
+{
+ "original": {
+ "id": 482028,
+ "name": "John Doe",
+ "ssn": "555-55-5555",
+ "phone": "555-555-5555",
+ "email": "john.doe@nifi.apache.org"
+ },
+ "enrichment": {
+ "country": "UK",
+ "allowsPII": false
+ }
+}
+</code></pre>
+
+<p>
+We might then use the TransformRecord processor with a JSON RecordReader and a JSON RecordSetWriter to transform this. Using Groovy, our transformation may look something like this:
+</p>
+
+<pre><code>
+import org.apache.nifi.serialization.record.Record
+
+Record original = (Record) record.getValue("original")
+Record enrichment = (Record) record.getValue("enrichment")
+
+if (Boolean.TRUE != enrichment?.getAsBoolean("allowsPII")) {
+ original.setValue("ssn", null)
+ original.setValue("phone", null)
+ original.setValue("email", null)
+}
+
+return original
+</code></pre>
+
+<p>
+Which will produce for us the following output:
+</p>
+
+<pre><code>
+{
+ "id" : 482028,
+ "name" : "John Doe",
+ "ssn" : null,
+ "phone" : null,
+ "email" : null
+}
+</code></pre>
+
+<p>
+In this way, we have used information from the enrichment record to optionally transform the original record. We then return the original record, dropping the enrichment record all together. In
+this way, we open up an infinite number of possibilities for transforming our original payload based on the content of the enrichment data that we have fetched based on that data.
+</p>
+
+</body>
+</html>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/fork-join-enrichment.png b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/fork-join-enrichment.png
new file mode 100644
index 0000000..ab54aec
Binary files /dev/null and b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoinEnrichment/fork-join-enrichment.png differ
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoinEnrichment.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoinEnrichment.java
new file mode 100644
index 0000000..1a30332
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoinEnrichment.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.mock.MockComponentLogger;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.record.ArrayListRecordWriter;
+import org.apache.nifi.serialization.record.CommaSeparatedRecordReader;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestJoinEnrichment {
+ private static final File EXAMPLES_DIR = new File("src/test/resources/TestJoinEnrichment");
+
+
+ @Test
+ public void testSimpleSqlJoin() throws InitializationException {
+ final TestRunner runner = TestRunners.newTestRunner(new JoinEnrichment());
+
+ final ArrayListRecordWriter writer = setupCsvServices(runner);
+ runner.setProperty(JoinEnrichment.JOIN_STRATEGY, JoinEnrichment.JOIN_SQL);
+ runner.setProperty(JoinEnrichment.SQL, "SELECT original.id, enrichment.name FROM original JOIN enrichment ON original.id = enrichment.id");
+
+ final Map<String, String> originalAttributes = new HashMap<>();
+ originalAttributes.put("enrichment.group.id", "abc");
+ originalAttributes.put("enrichment.role", "ORIGINAL");
+ runner.enqueue("id\n5", originalAttributes);
+
+ final Map<String, String> enrichmentAttributes = new HashMap<>();
+ enrichmentAttributes.put("enrichment.group.id", "abc");
+ enrichmentAttributes.put("enrichment.role", "ENRICHMENT");
+ runner.enqueue("id,name\n5,John Doe", enrichmentAttributes);
+
+ runner.run();
+
+ runner.assertTransferCount(JoinEnrichment.REL_JOINED, 1);
+ runner.assertTransferCount(JoinEnrichment.REL_ORIGINAL, 2);
+
+ final List<Record> written = writer.getRecordsWritten();
+ assertEquals(1, written.size());
+
+ final Record outRecord = written.get(0);
+ assertEquals(5, outRecord.getAsInt("id"));
+ assertEquals("John Doe", outRecord.getValue("name"));
+
+ final RecordSchema schema = outRecord.getSchema();
+ assertEquals(RecordFieldType.STRING, schema.getField("id").get().getDataType().getFieldType());
+ assertEquals(RecordFieldType.STRING, schema.getField("name").get().getDataType().getFieldType());
+ }
+
+ // Tests that the LEFT OUTER JOIN example in the Additional Details works as expected
+ @Test
+ public void testLeftOuterJoin() throws InitializationException, IOException, SchemaNotFoundException, MalformedRecordException {
+ final TestRunner runner = TestRunners.newTestRunner(new JoinEnrichment());
+
+ final ArrayListRecordWriter writer = setupCsvServices(runner);
+ runner.setProperty(JoinEnrichment.JOIN_STRATEGY, JoinEnrichment.JOIN_SQL);
+ runner.setProperty(JoinEnrichment.SQL, "SELECT o.*, e.* FROM original o LEFT OUTER JOIN enrichment e ON o.id = e.customer_id");
+
+ final Map<String, String> originalAttributes = new HashMap<>();
+ originalAttributes.put("enrichment.group.id", "abc");
+ originalAttributes.put("enrichment.role", "ORIGINAL");
+ runner.enqueue(new File(EXAMPLES_DIR, "left-outer-join-original.csv").toPath(), originalAttributes);
+
+ final Map<String, String> enrichmentAttributes = new HashMap<>();
+ enrichmentAttributes.put("enrichment.group.id", "abc");
+ enrichmentAttributes.put("enrichment.role", "ENRICHMENT");
+ runner.enqueue(new File(EXAMPLES_DIR, "left-outer-join-enrichment.csv").toPath(), enrichmentAttributes);
+
+ runner.run();
+
+ runner.assertTransferCount(JoinEnrichment.REL_JOINED, 1);
+ runner.assertTransferCount(JoinEnrichment.REL_ORIGINAL, 2);
+
+ final List<Record> expected = readCsvRecords(new File(EXAMPLES_DIR, "left-outer-join-expected.csv"));
+ assertEquals(new HashSet<>(expected), new HashSet<>(writer.getRecordsWritten()));
+ }
+
+ // Tests that the LEFT OUTER JOIN example in the Additional Details that renames fields works as expected
+ @Test
+ public void testLeftOuterJoinRenameFields() throws InitializationException, IOException, SchemaNotFoundException, MalformedRecordException {
+ final TestRunner runner = TestRunners.newTestRunner(new JoinEnrichment());
+
+ final ArrayListRecordWriter writer = setupCsvServices(runner);
+ runner.setProperty(JoinEnrichment.JOIN_STRATEGY, JoinEnrichment.JOIN_SQL);
+ runner.setProperty(JoinEnrichment.SQL,
+ "SELECT o.id, o.name, e.customer_name AS preferred_name, o.age, e.customer_email AS email FROM original o LEFT OUTER JOIN enrichment e ON o.id = e.customer_id");
+
+ final Map<String, String> originalAttributes = new HashMap<>();
+ originalAttributes.put("enrichment.group.id", "abc");
+ originalAttributes.put("enrichment.role", "ORIGINAL");
+ runner.enqueue(new File(EXAMPLES_DIR, "left-outer-join-original.csv").toPath(), originalAttributes);
+
+ final Map<String, String> enrichmentAttributes = new HashMap<>();
+ enrichmentAttributes.put("enrichment.group.id", "abc");
+ enrichmentAttributes.put("enrichment.role", "ENRICHMENT");
+ runner.enqueue(new File(EXAMPLES_DIR, "left-outer-join-enrichment.csv").toPath(), enrichmentAttributes);
+
+ runner.run();
+
+ runner.assertTransferCount(JoinEnrichment.REL_JOINED, 1);
+ runner.assertTransferCount(JoinEnrichment.REL_ORIGINAL, 2);
+
+ final List<Record> expected = readCsvRecords(new File(EXAMPLES_DIR, "left-outer-join-rename-expected.csv"));
+ assertEquals(new HashSet<>(expected), new HashSet<>(writer.getRecordsWritten()));
+ }
+
+
+ // Tests that the Insert Enrichment Record Fields example in the Additional Details produces expected output
+ @Test
+ public void testInsertEnrichmentFields() throws InitializationException, IOException, SchemaNotFoundException, MalformedRecordException {
+ final TestRunner runner = TestRunners.newTestRunner(new JoinEnrichment());
+
+ final ArrayListRecordWriter writer = setupJsonServices(runner);
+ runner.setProperty(JoinEnrichment.JOIN_STRATEGY, JoinEnrichment.JOIN_INSERT_ENRICHMENT_FIELDS);
+ runner.setProperty(JoinEnrichment.INSERTION_RECORD_PATH, "/purchase/customer");
+
+ final Map<String, String> originalAttributes = new HashMap<>();
+ originalAttributes.put("enrichment.group.id", "abc");
+ originalAttributes.put("enrichment.role", "ORIGINAL");
+ runner.enqueue(new File(EXAMPLES_DIR, "insert-original.json").toPath(), originalAttributes);
+
+ final Map<String, String> enrichmentAttributes = new HashMap<>();
+ enrichmentAttributes.put("enrichment.group.id", "abc");
+ enrichmentAttributes.put("enrichment.role", "ENRICHMENT");
+ runner.enqueue(new File(EXAMPLES_DIR, "insert-enrichment.json").toPath(), enrichmentAttributes);
+
+ runner.run();
+
+ runner.assertTransferCount(JoinEnrichment.REL_JOINED, 1);
+ runner.assertTransferCount(JoinEnrichment.REL_ORIGINAL, 2);
+
+ final List<Record> written = writer.getRecordsWritten();
+ assertEquals(2, written.size());
+
+ final RecordPath recordPath = RecordPath.compile("/purchase/customer/customerDetails");
+
+ final List<Object> firstCustomerDetailsList = recordPath.evaluate(written.get(0)).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList());
+ assertEquals(1, firstCustomerDetailsList.size());
+ final Record customerDetails = (Record) firstCustomerDetailsList.get(0);
+ assertEquals(48202, customerDetails.getValue("id"));
+ assertEquals("555-555-5555", customerDetails.getValue("phone"));
+ assertEquals("john.doe@nifi.apache.org", customerDetails.getValue("email"));
+
+ final List<Object> secondCustomerDetailsList = recordPath.evaluate(written.get(1)).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList());
+ assertEquals(1, secondCustomerDetailsList.size());
+ final Record secondCustomerDetails = (Record) secondCustomerDetailsList.get(0);
+ assertEquals(5512, secondCustomerDetails.getValue("id"));
+ assertEquals("555-555-5511", secondCustomerDetails.getValue("phone"));
+ assertEquals("jane.doe@nifi.apache.org", secondCustomerDetails.getValue("email"));
+ }
+
+
+ private List<Record> readCsvRecords(final File file) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ final CommaSeparatedRecordReader reader = new CommaSeparatedRecordReader();
+ reader.setUseNullForEmptyString(true);
+
+ return readRecords(reader, file);
+ }
+
+ private List<Record> readRecords(final RecordReaderFactory readerFactory, final File file) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ final List<Record> records = new ArrayList<>();
+ try (final InputStream rawIn = new FileInputStream(file);
+ final InputStream in = new BufferedInputStream(rawIn)) {
+
+ final RecordReader recordReader = readerFactory.createRecordReader(Collections.emptyMap(), in, file.length(), new MockComponentLogger());
+ Record record;
+ while ((record = recordReader.nextRecord()) != null) {
+ records.add(record);
+ }
+ }
+
+ return records;
+ }
+
+ private ArrayListRecordWriter setupCsvServices(final TestRunner runner) throws InitializationException {
+ final CommaSeparatedRecordReader originalReader = new CommaSeparatedRecordReader();
+ originalReader.setUseNullForEmptyString(true);
+
+ final CommaSeparatedRecordReader enrichmentReader = new CommaSeparatedRecordReader();
+ enrichmentReader.setUseNullForEmptyString(true);
+
+ final ArrayListRecordWriter writer = new ArrayListRecordWriter(null);
+
+ runner.addControllerService("originalReader", originalReader);
+ runner.enableControllerService(originalReader);
+ runner.addControllerService("enrichmentReader", enrichmentReader);
+ runner.enableControllerService(enrichmentReader);
+ runner.addControllerService("writer", writer);
+ runner.enableControllerService(writer);
+
+ runner.setProperty(JoinEnrichment.ORIGINAL_RECORD_READER, "originalReader");
+ runner.setProperty(JoinEnrichment.ENRICHMENT_RECORD_READER, "enrichmentReader");
+ runner.setProperty(JoinEnrichment.RECORD_WRITER, "writer");
+
+ return writer;
+ }
+
+ private ArrayListRecordWriter setupJsonServices(final TestRunner runner) throws InitializationException {
+ final JsonTreeReader originalReader = new JsonTreeReader();
+ final JsonTreeReader enrichmentReader = new JsonTreeReader();
+
+ final ArrayListRecordWriter writer = new ArrayListRecordWriter(null);
+
+ runner.addControllerService("originalReader", originalReader);
+ runner.enableControllerService(originalReader);
+ runner.addControllerService("enrichmentReader", enrichmentReader);
+ runner.enableControllerService(enrichmentReader);
+ runner.addControllerService("writer", writer);
+ runner.enableControllerService(writer);
+
+ runner.setProperty(JoinEnrichment.ORIGINAL_RECORD_READER, "originalReader");
+ runner.setProperty(JoinEnrichment.ENRICHMENT_RECORD_READER, "enrichmentReader");
+ runner.setProperty(JoinEnrichment.RECORD_WRITER, "writer");
+
+ return writer;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
index 54a97f2..0489ad2 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestMergeContent.java
@@ -47,7 +47,6 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
@@ -501,14 +500,14 @@ public class TestMergeContent {
}
@Test
- public void testSimpleBinaryConcatSingleBin() throws IOException, InterruptedException {
+ public void testSimpleBinaryConcatSingleBin() {
final TestRunner runner = TestRunners.newTestRunner(new MergeContent());
runner.setProperty(MergeContent.MAX_BIN_AGE, "1 sec");
runner.setProperty(MergeContent.MERGE_FORMAT, MergeContent.MERGE_FORMAT_CONCAT);
runner.setProperty(MergeContent.MAX_BIN_COUNT, "1");
createFlowFiles(runner);
- runner.run();
+ runner.run(2);
runner.assertQueueEmpty();
runner.assertTransferCount(MergeContent.REL_MERGED, 1);
@@ -516,7 +515,7 @@ public class TestMergeContent {
runner.assertTransferCount(MergeContent.REL_ORIGINAL, 3);
final MockFlowFile bundle = runner.getFlowFilesForRelationship(MergeContent.REL_MERGED).get(0);
- bundle.assertContentEquals("Hello, World!".getBytes("UTF-8"));
+ bundle.assertContentEquals("Hello, World!");
bundle.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/plain-text");
}
@@ -667,7 +666,7 @@ public class TestMergeContent {
}
@Test
- public void testOldestBinIsExpired() throws IOException, InterruptedException {
+ public void testOldestBinIsExpired() {
final TestRunner runner = TestRunners.newTestRunner(new MergeContent());
runner.setProperty(MergeContent.MAX_BIN_AGE, "1 day");
runner.setProperty(MergeContent.MAX_BIN_COUNT, "50");
@@ -705,6 +704,11 @@ public class TestMergeContent {
// will be in a separate bin.
runner.run(1, false, true);
runner.enqueue(new byte[0], attrs);
+
+ // Add one more FlowFile, with a unique correlation id so that it creates the 51st bin.
+ // This should trigger the oldest bin to be evicted.
+ attrs.put("correlationId", "abc");
+ runner.enqueue(new byte[0], attrs);
runner.run();
runner.assertTransferCount(MergeContent.REL_MERGED, 1);
@@ -1262,15 +1266,15 @@ public class TestMergeContent {
assertEquals(2, runner.getQueueSize().getObjectCount());
}
- private void createFlowFiles(final TestRunner testRunner) throws UnsupportedEncodingException {
+ private void createFlowFiles(final TestRunner testRunner) {
final Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/plain-text");
// add 'fragment.index' attribute to ensure non-defragment mode operates correctly even when index is present
attributes.put(MergeContent.FRAGMENT_INDEX_ATTRIBUTE, "1");
- testRunner.enqueue("Hello".getBytes("UTF-8"), attributes);
- testRunner.enqueue(", ".getBytes("UTF-8"), attributes);
- testRunner.enqueue("World!".getBytes("UTF-8"), attributes);
+ testRunner.enqueue("Hello", attributes);
+ testRunner.enqueue(", ", attributes);
+ testRunner.enqueue("World!", attributes);
}
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestIndexCorrelatedJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestIndexCorrelatedJoinStrategy.java
new file mode 100644
index 0000000..4719bd5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestIndexCorrelatedJoinStrategy.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public abstract class TestIndexCorrelatedJoinStrategy {
+ protected RecordSchema getOriginalSchema() {
+ final List<RecordField> originalFields = Collections.singletonList(new RecordField("id", RecordFieldType.INT.getDataType()));
+ return new SimpleRecordSchema(originalFields);
+ }
+
+ protected RecordSchema getEnrichmentSchema() {
+ final List<RecordField> enrichmentFields = Arrays.asList(
+ new RecordField("name", RecordFieldType.STRING.getDataType()),
+ new RecordField("number", RecordFieldType.INT.getDataType())
+ );
+
+ return new SimpleRecordSchema(enrichmentFields);
+ }
+
+ protected Record createOriginalRecord(final int id) {
+ final RecordSchema originalSchema = getOriginalSchema();
+
+ final Map<String, Object> originalValues = new HashMap<>();
+ originalValues.put("id", id);
+ return new MapRecord(originalSchema, originalValues);
+ }
+
+ protected Record createEnrichmentRecord(final int id, final String name, final int number) {
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+
+ final Map<String, Object> enrichmentValues = new HashMap<>();
+ enrichmentValues.put("id", id);
+ enrichmentValues.put("name", name);
+ enrichmentValues.put("number", number);
+
+ final Record enrichmentRecord = new MapRecord(enrichmentSchema, enrichmentValues);
+ return enrichmentRecord;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestInsertRecordFieldsJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestInsertRecordFieldsJoinStrategy.java
new file mode 100644
index 0000000..446fbab
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestInsertRecordFieldsJoinStrategy.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.mock.MockComponentLogger;
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.type.RecordDataType;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestInsertRecordFieldsJoinStrategy extends TestIndexCorrelatedJoinStrategy {
+
+ @Test
+ public void testSimpleInsertAtRoot() {
+ final RecordSchema originalSchema = getOriginalSchema();
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final InsertRecordFieldsJoinStrategy strategy = new InsertRecordFieldsJoinStrategy(new MockComponentLogger(), "/");
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(3, resultFields.size());
+
+ final List<RecordField> combinedFields = new ArrayList<>();
+ combinedFields.add(originalSchema.getFields().get(0));
+ combinedFields.addAll(enrichmentSchema.getFields());
+
+ // Wrap each List of fields in a hashset to combine because the order of the fields is not guaranteed when incorporating schemas.
+ assertEquals(new HashSet<>(combinedFields), new HashSet<>(resultFields));
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(555, combined.getAsInt("id"));
+ assertEquals("John Doe", combined.getValue("name"));
+ assertEquals(100, combined.getValue("number"));
+ }
+
+ @Test
+ public void testNullEnrichment() {
+ final RecordSchema originalSchema = getOriginalSchema();
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final InsertRecordFieldsJoinStrategy strategy = new InsertRecordFieldsJoinStrategy(new MockComponentLogger(), "/");
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(3, resultFields.size());
+
+ final List<RecordField> combinedFields = new ArrayList<>();
+ combinedFields.add(originalSchema.getFields().get(0));
+ combinedFields.addAll(enrichmentSchema.getFields());
+
+ // Wrap each List of fields in a hashset to combine because the order of the fields is not guaranteed when incorporating schemas.
+ assertEquals(new HashSet<>(combinedFields), new HashSet<>(resultFields));
+
+ final Record combined = strategy.combineRecords(createOriginalRecord(555), null, resultSchema);
+ assertEquals(555, combined.getAsInt("id"));
+ assertNull(combined.getValue("name"));
+ assertNull(combined.getValue("number"));
+ }
+
+ @Test
+ public void testNullOriginalNullEnrichment() {
+ final RecordSchema originalSchema = getOriginalSchema();
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final InsertRecordFieldsJoinStrategy strategy = new InsertRecordFieldsJoinStrategy(new MockComponentLogger(), "/");
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(3, resultFields.size());
+
+ final List<RecordField> combinedFields = new ArrayList<>();
+ combinedFields.add(originalSchema.getFields().get(0));
+ combinedFields.addAll(enrichmentSchema.getFields());
+
+ // Wrap each List of fields in a hashset to combine because the order of the fields is not guaranteed when incorporating schemas.
+ assertEquals(new HashSet<>(combinedFields), new HashSet<>(resultFields));
+
+ final Record combined = strategy.combineRecords(null, null, resultSchema);
+ assertNull(combined);
+ }
+
+ @Test
+ public void testRecordPathPointsToNonExistentField() {
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final InsertRecordFieldsJoinStrategy strategy = new InsertRecordFieldsJoinStrategy(new MockComponentLogger(), "/abc");
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(555, combined.getAsInt("id"));
+ assertArrayEquals(new Integer[] {555}, combined.getValues());
+ }
+
+ @Test
+ public void testRecordPathPointingToChildRecord() {
+ final RecordSchema simpleOriginalSchema = getOriginalSchema();
+ final List<RecordField> complexSchemaFields = new ArrayList<>();
+ complexSchemaFields.addAll(simpleOriginalSchema.getFields());
+ // Add a field named 'xyz' that is a record with no fields.
+ final RecordSchema emptySchema = new SimpleRecordSchema(Collections.emptyList());
+ complexSchemaFields.add(new RecordField("xyz", RecordFieldType.RECORD.getRecordDataType(emptySchema)));
+
+ final RecordSchema complexOriginalSchema = new SimpleRecordSchema(complexSchemaFields);
+ final Map<String, Object> originalValues = new HashMap<>();
+ originalValues.put("id", 555);
+ originalValues.put("xyz", new MapRecord(emptySchema, new HashMap<>()));
+ final Record originalRecord = new MapRecord(complexOriginalSchema, originalValues);
+
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final InsertRecordFieldsJoinStrategy strategy = new InsertRecordFieldsJoinStrategy(new MockComponentLogger(), "/xyz");
+
+ // Check that we update the schema properly
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(2, resultFields.size());
+ assertTrue(resultFields.contains(simpleOriginalSchema.getFields().get(0)));
+
+ final DataType xyzDataType = resultSchema.getDataType("xyz").get();
+ final RecordSchema xyzSchema = ((RecordDataType) xyzDataType).getChildSchema();
+ assertEquals(RecordFieldType.INT, xyzSchema.getDataType("id").get().getFieldType());
+ assertEquals(RecordFieldType.STRING, xyzSchema.getDataType("name").get().getFieldType());
+ assertEquals(RecordFieldType.INT, xyzSchema.getDataType("number").get().getFieldType());
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(555, combined.getAsInt("id"));
+
+ final Object xyzValue = combined.getValue("xyz");
+ assertTrue(xyzValue instanceof Record);
+
+ final Record xyzRecord = (Record) xyzValue;
+ assertEquals("John Doe", xyzRecord.getValue("name"));
+ assertEquals(100, xyzRecord.getValue("number"));
+ assertEquals(555, xyzRecord.getValue("id"));
+ }
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestWrapperJoinStrategy.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestWrapperJoinStrategy.java
new file mode 100644
index 0000000..7a5e37d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/enrichment/TestWrapperJoinStrategy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard.enrichment;
+
+import org.apache.nifi.mock.MockComponentLogger;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.type.RecordDataType;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class TestWrapperJoinStrategy extends TestIndexCorrelatedJoinStrategy {
+ @Test
+ public void testSimpleJoin() {
+ final RecordSchema originalSchema = getOriginalSchema();
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final WrapperJoinStrategy strategy = new WrapperJoinStrategy(new MockComponentLogger());
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(2, resultFields.size());
+
+ final RecordDataType originalDataType = (RecordDataType) resultSchema.getDataType("original").get();
+ assertEquals(originalSchema, originalDataType.getChildSchema());
+
+ final RecordDataType enrichmentDataType = (RecordDataType) resultSchema.getDataType("enrichment").get();
+ assertEquals(enrichmentSchema, enrichmentDataType.getChildSchema());
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(originalRecord, combined.getValue("original"));
+ assertEquals(enrichmentRecord, combined.getValue("enrichment"));
+ }
+
+ @Test
+ public void testNullEnrichment() {
+ final RecordSchema originalSchema = getOriginalSchema();
+ final Record originalRecord = createOriginalRecord(555);
+ final Record enrichmentRecord = null;
+
+ final WrapperJoinStrategy strategy = new WrapperJoinStrategy(new MockComponentLogger());
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(1, resultFields.size());
+
+ final RecordDataType originalDataType = (RecordDataType) resultSchema.getDataType("original").get();
+ assertEquals(originalSchema, originalDataType.getChildSchema());
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(originalRecord, combined.getValue("original"));
+ assertEquals(enrichmentRecord, combined.getValue("enrichment"));
+ }
+
+ @Test
+ public void testNullOriginal() {
+ final RecordSchema enrichmentSchema = getEnrichmentSchema();
+ final Record originalRecord = null;
+ final Record enrichmentRecord = createEnrichmentRecord(555, "John Doe", 100);
+
+ final WrapperJoinStrategy strategy = new WrapperJoinStrategy(new MockComponentLogger());
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(1, resultFields.size());
+
+ final RecordDataType enrichmentDataType = (RecordDataType) resultSchema.getDataType("enrichment").get();
+ assertEquals(enrichmentSchema, enrichmentDataType.getChildSchema());
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertEquals(originalRecord, combined.getValue("original"));
+ assertEquals(enrichmentRecord, combined.getValue("enrichment"));
+ }
+
+ @Test
+ public void testCombineNullOriginalNullEnrichment() {
+ final Record originalRecord = null;
+ final Record enrichmentRecord = null;
+
+ final WrapperJoinStrategy strategy = new WrapperJoinStrategy(new MockComponentLogger());
+
+ final RecordSchema resultSchema = strategy.createResultSchema(originalRecord, enrichmentRecord);
+ final List<RecordField> resultFields = resultSchema.getFields();
+ assertEquals(0, resultFields.size());
+
+ final Record combined = strategy.combineRecords(originalRecord, enrichmentRecord, resultSchema);
+ assertNull(combined);
+ }
+
+}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-enrichment.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-enrichment.json
new file mode 100644
index 0000000..25deb89
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-enrichment.json
@@ -0,0 +1,14 @@
+[
+ {
+ "customerDetails": {
+ "id": 48202,
+ "phone": "555-555-5555",
+ "email": "john.doe@nifi.apache.org"
+ }
+ }, {
+ "customerDetails": {
+ "id": 5512,
+ "phone": "555-555-5511",
+ "email": "jane.doe@nifi.apache.org"
+ }
+}]
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-original.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-original.json
new file mode 100644
index 0000000..4f5f228
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/insert-original.json
@@ -0,0 +1,37 @@
+[{
+ "purchase": {
+ "customer": {
+ "loyaltyId": 48202,
+ "firstName": "John",
+ "lastName": "Doe"
+ },
+ "total": 48.28,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 24.14,
+ "quantity": 2
+ }
+ ]
+ }
+}, {
+ "purchase": {
+ "customer": {
+ "loyaltyId": 5512,
+ "firstName": "Jane",
+ "lastName": "Doe"
+ },
+ "total": 121.44,
+ "items": [
+ {
+ "itemDescription": "book",
+ "price": 28.15,
+ "quantity": 4
+ }, {
+ "itemDescription": "inkpen",
+ "price": 4.42,
+ "quantity": 2
+ }
+ ]
+ }
+}]
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-enrichment.csv b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-enrichment.csv
new file mode 100644
index 0000000..2557538
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-enrichment.csv
@@ -0,0 +1,4 @@
+customer_id, customer_email, customer_name, customer_since
+555, joseph.doe@nifi.apache.org, Joe Doe, 08/Dec/14
+832, jane.doe@nifi.apache.org, Mrs. Doe, 14/Nov/14
+28021, john.doe@nifi.apache.org, John Doe, 22/Jan/22
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-expected.csv b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-expected.csv
new file mode 100644
index 0000000..50b5a66
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-expected.csv
@@ -0,0 +1,5 @@
+id, name, age, customer_id, customer_email, customer_name, customer_since
+28021, John Doe, 55, 28021, john.doe@nifi.apache.org, John Doe, 22/Jan/22
+832, Jane Doe, 22, 832, jane.doe@nifi.apache.org, Mrs. Doe, 14/Nov/14
+29201, Jake Doe, 23,,,,
+555, Joseph Doe, 2, 555, joseph.doe@nifi.apache.org, Joe Doe, 08/Dec/14
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-original.csv b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-original.csv
new file mode 100644
index 0000000..4f86e2b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-original.csv
@@ -0,0 +1,5 @@
+id, name, age
+28021, John Doe, 55
+832, Jane Doe, 22
+29201, Jake Doe, 23
+555, Joseph Doe, 2
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-rename-expected.csv b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-rename-expected.csv
new file mode 100644
index 0000000..00c44a4
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoinEnrichment/left-outer-join-rename-expected.csv
@@ -0,0 +1,5 @@
+id, name, preferred_name, age, email
+28021, John Doe, John Doe, 55, john.doe@nifi.apache.org
+832, Jane Doe, Mrs. Doe, 22, jane.doe@nifi.apache.org
+29201, Jake Doe,, 23,
+555, Joseph Doe, Joe Doe, 2, joseph.doe@nifi.apache.org
\ No newline at end of file