You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/05/12 16:49:09 UTC
[1/5] nifi git commit: NIFI-3838: Initial implementation of
RecordPath and UpdateRecord processor
Repository: nifi
Updated Branches:
refs/heads/master 1c58e78ce -> b1901d5fe
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-service-api/src/main/java/org/apache/nifi/serialization/RecordSetWriterFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-service-api/src/main/java/org/apache/nifi/serialization/RecordSetWriterFactory.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-service-api/src/main/java/org/apache/nifi/serialization/RecordSetWriterFactory.java
index e23ad20..9ea6e64 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-service-api/src/main/java/org/apache/nifi/serialization/RecordSetWriterFactory.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-service-api/src/main/java/org/apache/nifi/serialization/RecordSetWriterFactory.java
@@ -24,6 +24,7 @@ import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.record.RecordSchema;
/**
* <p>
@@ -45,19 +46,29 @@ public interface RecordSetWriterFactory extends ControllerService {
/**
* <p>
- * Creates a new RecordSetWriter that is capable of writing record contents to an OutputStream. Note that the
- * FlowFile and InputStream that are given may well be different than the FlowFile that the writer is intended
- * to write to. The given FlowFile and InputStream are intended to be used for determining the schema that should
- * be used when writing records.
+ * Returns the Schema that will be used for writing Records. Note that the FlowFile and InputStream that are given
+ * may well be different than the FlowFile that the writer will write to. The given FlowFile and InputStream are
+ * intended to be used for determining the schema that should be used when writing records.
+ * </p>
+ *
+ * @param flowFile the FlowFile from which the schema should be determined.
+ * @param content the contents of the FlowFile from which to determine the schema
+ * @return the Schema that should be used for writing Records
+ * @throws SchemaNotFoundException if unable to find the schema
+ */
+ RecordSchema getSchema(FlowFile flowFile, InputStream content) throws SchemaNotFoundException, IOException;
+
+ /**
+ * <p>
+ * Creates a new RecordSetWriter that is capable of writing record contents to an OutputStream.
* </p>
*
* @param logger the logger to use when logging information. This is passed in, rather than using the logger of the Controller Service
* because it allows messages to be logged for the component that is calling this Controller Service.
- * @param schemaFlowFile the FlowFile from which the schema should be determined.
- * @param schemaFlowFileContent the contents of the FlowFile from which to determine the schema
+ * @param schema the schema that will be used for writing records
+ *
* @return a RecordSetWriter that can write record sets to an OutputStream
- * @throws SchemaNotFoundException if unable to find the schema
* @throws IOException if unable to read from the given InputStream
*/
- RecordSetWriter createWriter(ComponentLog logger, FlowFile schemaFlowFile, InputStream schemaFlowFileContent) throws SchemaNotFoundException, IOException;
+ RecordSetWriter createWriter(ComponentLog logger, RecordSchema schema) throws SchemaNotFoundException, IOException;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroReaderWithExplicitSchema.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroReaderWithExplicitSchema.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroReaderWithExplicitSchema.java
index daabdcb..ce49443 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroReaderWithExplicitSchema.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroReaderWithExplicitSchema.java
@@ -33,7 +33,6 @@ import org.apache.nifi.serialization.record.RecordSchema;
public class AvroReaderWithExplicitSchema extends AvroRecordReader {
private final InputStream in;
- private final Schema avroSchema;
private final RecordSchema recordSchema;
private final DatumReader<GenericRecord> datumReader;
private final BinaryDecoder decoder;
@@ -43,7 +42,6 @@ public class AvroReaderWithExplicitSchema extends AvroRecordReader {
this.in = in;
this.recordSchema = recordSchema;
- this.avroSchema = avroSchema;
datumReader = new GenericDatumReader<GenericRecord>(avroSchema);
decoder = DecoderFactory.get().binaryDecoder(in, null);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroRecordSetWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroRecordSetWriter.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroRecordSetWriter.java
index a4476f9..1360add 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroRecordSetWriter.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/avro/AvroRecordSetWriter.java
@@ -18,7 +18,6 @@
package org.apache.nifi.avro;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedHashMap;
@@ -32,7 +31,6 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.schema.access.SchemaField;
@@ -59,12 +57,10 @@ public class AvroRecordSetWriter extends SchemaRegistryRecordSetWriter implement
"The FlowFile will have the Avro schema embedded into the content, as is typical with Avro");
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream in) throws IOException {
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema recordSchema) throws IOException {
final String strategyValue = getConfigurationContext().getProperty(getSchemaWriteStrategyDescriptor()).getValue();
try {
- final RecordSchema recordSchema = getSchema(flowFile, in);
-
final Schema avroSchema;
try {
if (recordSchema.getSchemaFormat().isPresent() & recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
index 45b5bae..3bcbce2 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
@@ -112,7 +112,7 @@ public class CSVRecordReader implements RecordReader {
return null;
}
- return DataTypeUtils.convertType(trimmed, dataType, dateFormat, timeFormat, timestampFormat, fieldName);
+ return DataTypeUtils.convertType(trimmed, dataType, () -> dateFormat, () -> timeFormat, () -> timestampFormat, fieldName);
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordSetWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordSetWriter.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordSetWriter.java
index 0b29f09..d4f066f 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordSetWriter.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordSetWriter.java
@@ -18,7 +18,6 @@
package org.apache.nifi.csv;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@@ -28,7 +27,6 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ConfigurationContext;
-import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.DateTimeTextRecordSetWriter;
@@ -69,8 +67,7 @@ public class CSVRecordSetWriter extends DateTimeTextRecordSetWriter implements R
}
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream in) throws SchemaNotFoundException, IOException {
- final RecordSchema schema = getSchema(flowFile, in);
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema) throws SchemaNotFoundException, IOException {
return new WriteCSVResult(csvFormat, schema, getSchemaAccessWriter(schema),
getDateFormat().orElse(null), getTimeFormat().orElse(null), getTimestampFormat().orElse(null), includeHeader);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonPathRowRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonPathRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonPathRowRecordReader.java
index 7d2a2c1..1ee35ba 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonPathRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonPathRowRecordReader.java
@@ -162,7 +162,7 @@ public class JsonPathRowRecordReader extends AbstractJsonRowRecordReader {
return new MapRecord(childSchema, coercedValues);
} else {
- return DataTypeUtils.convertType(value, dataType, dateFormat, timeFormat, timestampFormat, fieldName);
+ return DataTypeUtils.convertType(value, dataType, () -> dateFormat, () -> timeFormat, () -> timestampFormat, fieldName);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSetWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSetWriter.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSetWriter.java
index 163b2ec..91370a7 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSetWriter.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonRecordSetWriter.java
@@ -18,7 +18,6 @@
package org.apache.nifi.json;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@@ -27,7 +26,6 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ConfigurationContext;
-import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.DateTimeTextRecordSetWriter;
@@ -64,8 +62,7 @@ public class JsonRecordSetWriter extends DateTimeTextRecordSetWriter implements
}
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream flowFileContent) throws SchemaNotFoundException, IOException {
- final RecordSchema schema = getSchema(flowFile, flowFileContent);
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema) throws SchemaNotFoundException, IOException {
return new WriteJsonResult(logger, schema, getSchemaAccessWriter(schema), prettyPrint,
getDateFormat().orElse(null), getTimeFormat().orElse(null), getTimestampFormat().orElse(null));
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
index ee5bff9..b542ebe 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
@@ -127,13 +127,14 @@ public class JsonTreeRowRecordReader extends AbstractJsonRowRecordReader {
case SHORT:
return DataTypeUtils.toShort(getRawNodeValue(fieldNode), fieldName);
case STRING:
- return DataTypeUtils.toString(getRawNodeValue(fieldNode), DataTypeUtils.getDateFormat(desiredType.getFieldType(), dateFormat, timeFormat, timestampFormat));
+ return DataTypeUtils.toString(getRawNodeValue(fieldNode),
+ () -> DataTypeUtils.getDateFormat(desiredType.getFieldType(), () -> dateFormat, () -> timeFormat, () -> timestampFormat));
case DATE:
- return DataTypeUtils.toDate(getRawNodeValue(fieldNode), dateFormat, fieldName);
+ return DataTypeUtils.toDate(getRawNodeValue(fieldNode), () -> dateFormat, fieldName);
case TIME:
- return DataTypeUtils.toTime(getRawNodeValue(fieldNode), timeFormat, fieldName);
+ return DataTypeUtils.toTime(getRawNodeValue(fieldNode), () -> timeFormat, fieldName);
case TIMESTAMP:
- return DataTypeUtils.toTimestamp(getRawNodeValue(fieldNode), timestampFormat, fieldName);
+ return DataTypeUtils.toTimestamp(getRawNodeValue(fieldNode), () -> timestampFormat, fieldName);
case MAP: {
final DataType valueType = ((MapDataType) desiredType).getValueType();
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
index ac4fbee..bd7dc5e 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/json/WriteJsonResult.java
@@ -162,7 +162,7 @@ public class WriteJsonResult implements RecordSetWriter {
}
final DataType chosenDataType = dataType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(value, (ChoiceDataType) dataType) : dataType;
- final Object coercedValue = DataTypeUtils.convertType(value, chosenDataType, dateFormat, timeFormat, timestampFormat, fieldName);
+ final Object coercedValue = DataTypeUtils.convertType(value, chosenDataType, () -> dateFormat, () -> timeFormat, () -> timestampFormat, fieldName);
if (coercedValue == null) {
generator.writeNull();
return;
@@ -170,7 +170,7 @@ public class WriteJsonResult implements RecordSetWriter {
switch (chosenDataType.getFieldType()) {
case DATE: {
- final String stringValue = DataTypeUtils.toString(coercedValue, dateFormat);
+ final String stringValue = DataTypeUtils.toString(coercedValue, () -> dateFormat);
if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
} else {
@@ -179,7 +179,7 @@ public class WriteJsonResult implements RecordSetWriter {
break;
}
case TIME: {
- final String stringValue = DataTypeUtils.toString(coercedValue, timeFormat);
+ final String stringValue = DataTypeUtils.toString(coercedValue, () -> timeFormat);
if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
} else {
@@ -188,7 +188,7 @@ public class WriteJsonResult implements RecordSetWriter {
break;
}
case TIMESTAMP: {
- final String stringValue = DataTypeUtils.toString(coercedValue, timestampFormat);
+ final String stringValue = DataTypeUtils.toString(coercedValue, () -> timestampFormat);
if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
} else {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextRecordSetWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextRecordSetWriter.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextRecordSetWriter.java
index cb69444..022804e 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextRecordSetWriter.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextRecordSetWriter.java
@@ -17,7 +17,6 @@
package org.apache.nifi.text;
-import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
@@ -27,19 +26,19 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
-import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
-import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.SchemaRegistryRecordSetWriter;
+import org.apache.nifi.serialization.record.RecordSchema;
@Tags({"text", "freeform", "expression", "language", "el", "record", "recordset", "resultset", "writer", "serialize"})
@CapabilityDescription("Writes the contents of a RecordSet as free-form text. The configured "
+ "text is able to make use of the Expression Language to reference each of the fields that are available "
+ "in a Record. Each record in the RecordSet will be separated by a single newline character.")
-public class FreeFormTextRecordSetWriter extends AbstractControllerService implements RecordSetWriterFactory {
+public class FreeFormTextRecordSetWriter extends SchemaRegistryRecordSetWriter implements RecordSetWriterFactory {
static final PropertyDescriptor TEXT = new PropertyDescriptor.Builder()
.name("Text")
.description("The text to use when writing the results. This property will evaluate the Expression Language using any of the fields available in a Record.")
@@ -74,8 +73,7 @@ public class FreeFormTextRecordSetWriter extends AbstractControllerService imple
}
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream in) {
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema) {
return new FreeFormTextWriter(textValue, characterSet);
}
-
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 57b1835..45d0cc3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1459,6 +1459,11 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-path</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-avro-record-utils</artifactId>
<version>1.3.0-SNAPSHOT</version>
</dependency>
[4/5] nifi git commit: NIFI-3838: Initial implementation of
RecordPath and UpdateRecord processor
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
new file mode 100644
index 0000000..9966c3d
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiArrayIndexPath.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.ArrayIndexFieldValue;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.NumericRange;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
+
+public class MultiArrayIndexPath extends RecordPathSegment {
+ private final List<NumericRange> indices;
+
+ MultiArrayIndexPath(final List<NumericRange> indices, final RecordPathSegment parent, final boolean absolute) {
+ super(indices.toString(), parent, absolute);
+ this.indices = indices;
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .filter(Filters.fieldTypeFilter(RecordFieldType.ARRAY))
+ .flatMap(fieldValue -> {
+ final ArrayDataType arrayDataType = (ArrayDataType) fieldValue.getField().getDataType();
+ final DataType elementDataType = arrayDataType.getElementType();
+ final RecordField arrayField = new RecordField(fieldValue.getField().getFieldName(), elementDataType);
+
+ final Object[] values = (Object[]) fieldValue.getValue();
+
+ return indices.stream()
+ .filter(range -> values.length > Math.abs(range.getMin()))
+ .flatMap(range -> {
+ final List<Object> valuesWithinRange = new ArrayList<>();
+
+ final int min = range.getMin() < 0 ? values.length + range.getMin() : range.getMin();
+ final int max = range.getMax() < 0 ? values.length + range.getMax() : range.getMax();
+
+ for (int i = min; i <= max; i++) {
+ if (values.length > i) {
+ valuesWithinRange.add(values[i]);
+ }
+ }
+
+ return IntStream.range(0, valuesWithinRange.size())
+ .mapToObj(index -> {
+ final RecordField elementField = new RecordField(arrayField.getFieldName() + "[" + index + "]", elementDataType);
+ return new ArrayIndexFieldValue(valuesWithinRange.get(index), elementField, fieldValue, index);
+ });
+ });
+
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
new file mode 100644
index 0000000..e9b1874
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/MultiMapKeyPath.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.MapEntryFieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.type.MapDataType;
+
+public class MultiMapKeyPath extends RecordPathSegment {
+ private final List<String> mapKeys;
+
+ MultiMapKeyPath(final List<String> mapKeys, final RecordPathSegment parent, final boolean absolute) {
+ super(mapKeys.toString(), parent, absolute);
+ this.mapKeys = mapKeys;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .filter(Filters.fieldTypeFilter(RecordFieldType.MAP))
+ .flatMap(fieldValue -> {
+ final Map<String, ?> map = (Map<String, ?>) fieldValue.getValue();
+ return mapKeys.stream().map(key -> {
+ final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + key + "']", valueType);
+ return new MapEntryFieldValue(map.get(key), elementField, fieldValue, key);
+ });
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ParentPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ParentPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ParentPath.java
new file mode 100644
index 0000000..bd5bf97
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ParentPath.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+
+
+public class ParentPath extends RecordPathSegment {
+
+ ParentPath(final RecordPathSegment parent, final boolean absolute) {
+ super("..", parent, absolute);
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> stream;
+ final RecordPathSegment parentPath = getParentPath();
+ if (parentPath == null) {
+ stream = Stream.of(context.getContextNode());
+ } else {
+ stream = parentPath.evaluate(context);
+ }
+
+ return Filters.presentValues(stream.map(fieldVal -> fieldVal.getParent()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/PredicatePath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/PredicatePath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/PredicatePath.java
new file mode 100644
index 0000000..759a848
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/PredicatePath.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.filter.RecordPathFilter;
+
+public class PredicatePath extends RecordPathSegment {
+ private final RecordPathFilter filter;
+
+ public PredicatePath(final RecordPathSegment parent, final RecordPathFilter filter, final boolean absolute) {
+ super("[" + filter + "]", parent, absolute);
+ this.filter = filter;
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> valueStream = getParentPath().evaluate(context);
+
+ return valueStream.flatMap(fieldVal -> {
+ // For the duration of this Predicate, we want to consider the 'context node' to be
+ // whatever value is given to us in the field value. We then want to return the 'context node'
+ // back to what it was before this Predicate.
+ final FieldValue previousContextNode = context.getContextNode();
+ context.setContextNode(fieldVal);
+ try {
+ // Really what we want to do is filter out Stream<FieldValue> but that becomes very difficult
+ // to implement for the RecordPathFilter's. So, instead, we pass the FieldValue to field and
+ // the RecordPathEvaluationContext and receive back a Stream<FieldValue>. Since this is a Predicate,
+ // though, we don't want to transform our Stream - we just want to filter it. So we handle this by
+ // mapping the result back to fieldVal. And since this predicate shouldn't return the same field multiple
+ // times, we will limit the stream to 1 element. We also filter out any FieldValue whose value is null.
+ // This is done because if we have a predicate like [./iDoNotExist != 'hello'] then the relative path will
+ // return a value of null and that will be compared to 'hello'. Since they are not equal, the NotEqualsFilter
+ // will return 'true', so we will get back a FieldValue with a null value. This should not make the Predicate
+ // true.
+ return filter.filter(fieldVal, context)
+ .filter(fv -> fv.getValue() != null)
+ .limit(1)
+ .map(ignore -> fieldVal);
+ } finally {
+ context.setContextNode(previousContextNode);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
new file mode 100644
index 0000000..24b872a
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathCompiler.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import static org.apache.nifi.record.path.RecordPathParser.ARRAY_INDEX;
+import static org.apache.nifi.record.path.RecordPathParser.CHILD_REFERENCE;
+import static org.apache.nifi.record.path.RecordPathParser.CURRENT_FIELD;
+import static org.apache.nifi.record.path.RecordPathParser.DESCENDANT_REFERENCE;
+import static org.apache.nifi.record.path.RecordPathParser.EQUAL;
+import static org.apache.nifi.record.path.RecordPathParser.FIELD_NAME;
+import static org.apache.nifi.record.path.RecordPathParser.GREATER_THAN;
+import static org.apache.nifi.record.path.RecordPathParser.GREATER_THAN_EQUAL;
+import static org.apache.nifi.record.path.RecordPathParser.LESS_THAN;
+import static org.apache.nifi.record.path.RecordPathParser.LESS_THAN_EQUAL;
+import static org.apache.nifi.record.path.RecordPathParser.MAP_KEY;
+import static org.apache.nifi.record.path.RecordPathParser.NOT_EQUAL;
+import static org.apache.nifi.record.path.RecordPathParser.NUMBER;
+import static org.apache.nifi.record.path.RecordPathParser.NUMBER_LIST;
+import static org.apache.nifi.record.path.RecordPathParser.NUMBER_RANGE;
+import static org.apache.nifi.record.path.RecordPathParser.PARENT_REFERENCE;
+import static org.apache.nifi.record.path.RecordPathParser.PATH;
+import static org.apache.nifi.record.path.RecordPathParser.PREDICATE;
+import static org.apache.nifi.record.path.RecordPathParser.RELATIVE_PATH;
+import static org.apache.nifi.record.path.RecordPathParser.ROOT_REFERENCE;
+import static org.apache.nifi.record.path.RecordPathParser.STRING_LIST;
+import static org.apache.nifi.record.path.RecordPathParser.STRING_LITERAL;
+import static org.apache.nifi.record.path.RecordPathParser.WILDCARD;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import org.antlr.runtime.tree.Tree;
+import org.apache.nifi.record.path.NumericRange;
+import org.apache.nifi.record.path.exception.RecordPathException;
+import org.apache.nifi.record.path.filter.EqualsFilter;
+import org.apache.nifi.record.path.filter.GreaterThanFilter;
+import org.apache.nifi.record.path.filter.GreaterThanOrEqualFilter;
+import org.apache.nifi.record.path.filter.LessThanFilter;
+import org.apache.nifi.record.path.filter.LessThanOrEqualFilter;
+import org.apache.nifi.record.path.filter.NotEqualsFilter;
+import org.apache.nifi.record.path.filter.RecordPathFilter;
+
+public class RecordPathCompiler {
+
+ public static RecordPathSegment compile(final Tree pathTree, final RecordPathSegment root, final boolean absolute) {
+ RecordPathSegment parent = root;
+ for (int i = 0; i < pathTree.getChildCount(); i++) {
+ final Tree child = pathTree.getChild(i);
+ parent = RecordPathCompiler.buildPath(child, parent, absolute);
+ }
+
+ return parent;
+ }
+
+ public static RecordPathSegment buildPath(final Tree tree, final RecordPathSegment parent, final boolean absolute) {
+ switch (tree.getType()) {
+ case ROOT_REFERENCE: {
+ return new RootPath();
+ }
+ case CHILD_REFERENCE: {
+ final Tree childTree = tree.getChild(0);
+ final int childTreeType = childTree.getType();
+ if (childTreeType == FIELD_NAME) {
+ final String childName = childTree.getChild(0).getText();
+ return new ChildFieldPath(childName, parent, absolute);
+ } else if (childTreeType == WILDCARD) {
+ return new WildcardChildPath(parent, absolute);
+ } else {
+ throw new RecordPathException("Expected field name following '/' Token but found " + childTree);
+ }
+ }
+ case ARRAY_INDEX: {
+ final Tree indexListTree = tree.getChild(0);
+ if (indexListTree.getType() == NUMBER_LIST) {
+ if (indexListTree.getChildCount() == 1 && indexListTree.getChild(0).getType() == NUMBER) {
+ final Tree indexTree = indexListTree.getChild(0);
+ final int index = Integer.parseInt(indexTree.getText());
+ return new ArrayIndexPath(index, parent, absolute);
+ }
+
+ final List<NumericRange> indexList = new ArrayList<>();
+
+ for (int i = 0; i < indexListTree.getChildCount(); i++) {
+ final Tree indexTree = indexListTree.getChild(i);
+ if (indexTree.getType() == NUMBER) {
+ final int index = Integer.valueOf(indexTree.getText());
+ indexList.add(new NumericRange(index, index));
+ } else if (indexTree.getType() == NUMBER_RANGE) {
+ final int min = Integer.valueOf(indexTree.getChild(0).getText());
+ final int max = Integer.valueOf(indexTree.getChild(1).getText());
+ indexList.add(new NumericRange(min, max));
+ } else {
+ throw new RecordPathException("Expected Number or Range following '[' Token but found " + indexTree);
+ }
+ }
+
+ return new MultiArrayIndexPath(indexList, parent, absolute);
+ } else {
+ throw new RecordPathException("Expected Number or Range following '[' Token but found " + indexListTree);
+ }
+ }
+ case MAP_KEY: {
+ final Tree keyTree = tree.getChild(0);
+ if (keyTree.getType() == STRING_LIST) {
+ if (keyTree.getChildCount() == 1) {
+ return new SingularMapKeyPath(keyTree.getChild(0).getText(), parent, absolute);
+ }
+
+ final List<String> keys = new ArrayList<>(keyTree.getChildCount());
+ for (int i = 0; i < keyTree.getChildCount(); i++) {
+ keys.add(keyTree.getChild(i).getText());
+ }
+
+ return new MultiMapKeyPath(keys, parent, absolute);
+ } else {
+ throw new RecordPathException("Expected Map Key following '[' Token but found " + keyTree);
+ }
+ }
+ case WILDCARD: {
+ return new WildcardIndexPath(parent, absolute);
+ }
+ case DESCENDANT_REFERENCE: {
+ final Tree childTree = tree.getChild(0);
+ final int childTreeType = childTree.getType();
+ if (childTreeType == FIELD_NAME) {
+ final String descendantName = childTree.getChild(0).getText();
+ return new DescendantFieldPath(descendantName, parent, absolute);
+ } else {
+ throw new RecordPathException("Expected field name following '//' Token but found " + childTree);
+ }
+ }
+ case PARENT_REFERENCE: {
+ return new ParentPath(parent, absolute);
+ }
+ case CURRENT_FIELD: {
+ return new CurrentFieldPath(parent, absolute);
+ }
+ case STRING_LITERAL: {
+ return new LiteralValuePath(parent, tree.getText(), absolute);
+ }
+ case NUMBER: {
+ return new LiteralValuePath(parent, Integer.parseInt(tree.getText()), absolute);
+ }
+ case PREDICATE: {
+ final Tree operatorTree = tree.getChild(0);
+ final RecordPathFilter filter = createFilter(operatorTree, parent, absolute);
+ return new PredicatePath(parent, filter, absolute);
+ }
+ case RELATIVE_PATH: {
+ return compile(tree, parent, absolute);
+ }
+ case PATH: {
+ return compile(tree, new RootPath(), absolute);
+ }
+ }
+
+ throw new RecordPathException("Encountered unexpected token " + tree);
+ }
+
+ private static RecordPathFilter createFilter(final Tree operatorTree, final RecordPathSegment parent, final boolean absolute) {
+ switch (operatorTree.getType()) {
+ case EQUAL:
+ return createBinaryOperationFilter(operatorTree, parent, EqualsFilter::new, absolute);
+ case NOT_EQUAL:
+ return createBinaryOperationFilter(operatorTree, parent, NotEqualsFilter::new, absolute);
+ case LESS_THAN:
+ return createBinaryOperationFilter(operatorTree, parent, LessThanFilter::new, absolute);
+ case LESS_THAN_EQUAL:
+ return createBinaryOperationFilter(operatorTree, parent, LessThanOrEqualFilter::new, absolute);
+ case GREATER_THAN:
+ return createBinaryOperationFilter(operatorTree, parent, GreaterThanFilter::new, absolute);
+ case GREATER_THAN_EQUAL:
+ return createBinaryOperationFilter(operatorTree, parent, GreaterThanOrEqualFilter::new, absolute);
+ default:
+ throw new RecordPathException("Expected an Expression of form <value> <operator> <value> to follow '[' Token but found " + operatorTree);
+ }
+ }
+
+ private static RecordPathFilter createBinaryOperationFilter(final Tree operatorTree, final RecordPathSegment parent,
+ final BiFunction<RecordPathSegment, RecordPathSegment, RecordPathFilter> function, final boolean absolute) {
+ final Tree lhsTree = operatorTree.getChild(0);
+ final Tree rhsTree = operatorTree.getChild(1);
+ final RecordPathSegment lhsPath = buildPath(lhsTree, parent, absolute);
+ final RecordPathSegment rhsPath = buildPath(rhsTree, parent, absolute);
+ return function.apply(lhsPath, rhsPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathSegment.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathSegment.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathSegment.java
new file mode 100644
index 0000000..92ff010
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RecordPathSegment.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.record.path.StandardRecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.Record;
+
+public abstract class RecordPathSegment implements RecordPath {
+ private final String path;
+ private final RecordPathSegment parentPath;
+ private final boolean absolute;
+
+ RecordPathSegment(final String path, final RecordPathSegment parentPath, final boolean absolute) {
+ this.path = path;
+ this.parentPath = parentPath;
+ this.absolute = absolute;
+ }
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ RecordPathSegment getParentPath() {
+ return parentPath;
+ }
+
+ @Override
+ public String toString() {
+ return getPath();
+ }
+
+ @Override
+ public boolean isAbsolute() {
+ return absolute;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof RecordPath)) {
+ return false;
+ }
+
+ final RecordPath other = (RecordPath) obj;
+ return getPath().equals(other.getPath());
+ }
+
+ @Override
+ public final RecordPathResult evaluate(final Record record) {
+ final RecordPathEvaluationContext context = new StandardRecordPathEvaluationContext(record);
+ final Stream<FieldValue> selectedFields = evaluate(context);
+
+ return new RecordPathResult() {
+ @Override
+ public String getPath() {
+ return RecordPathSegment.this.getPath();
+ }
+
+ @Override
+ public Stream<FieldValue> getSelectedFields() {
+ return selectedFields;
+ }
+ };
+ }
+
+ @Override
+ public final RecordPathResult evaluate(final FieldValue contextNode) {
+ final RecordPathEvaluationContext context;
+ if (Filters.isRecord(contextNode.getField().getDataType(), contextNode.getValue())) {
+ final Record record = (Record) contextNode.getValue();
+ if (record == null) {
+ return new RecordPathResult() {
+ @Override
+ public String getPath() {
+ return RecordPathSegment.this.getPath();
+ }
+
+ @Override
+ public Stream<FieldValue> getSelectedFields() {
+ return Stream.empty();
+ }
+ };
+ }
+
+ context = new StandardRecordPathEvaluationContext(record);
+ } else {
+ final FieldValue parent = contextNode.getParent().orElse(null);
+ if (parent == null) {
+ context = new StandardRecordPathEvaluationContext(null);
+ } else {
+ context = new StandardRecordPathEvaluationContext(parent.getParentRecord().orElse(null));
+ }
+ }
+
+ context.setContextNode(contextNode);
+ final Stream<FieldValue> selectedFields = evaluate(context);
+
+ return new RecordPathResult() {
+ @Override
+ public String getPath() {
+ return RecordPathSegment.this.getPath();
+ }
+
+ @Override
+ public Stream<FieldValue> getSelectedFields() {
+ return selectedFields;
+ }
+ };
+ }
+
+ public abstract Stream<FieldValue> evaluate(RecordPathEvaluationContext context);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RootPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RootPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RootPath.java
new file mode 100644
index 0000000..6ec205e
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/RootPath.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+
+public class RootPath extends RecordPathSegment {
+ private static final String PATH = "";
+
+ public RootPath() {
+ super(PATH, null, true);
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final RecordField field = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(context.getRecord().getSchema()));
+ final FieldValue fieldValue = new StandardFieldValue(context.getRecord(), field, null);
+ return Stream.of(fieldValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
new file mode 100644
index 0000000..ee57f36
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/SingularMapKeyPath.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.MapEntryFieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.type.MapDataType;
+
+public class SingularMapKeyPath extends RecordPathSegment {
+ private final String mapKey;
+
+ SingularMapKeyPath(final String mapKey, final RecordPathSegment parent, final boolean absolute) {
+ super("[" + mapKey + "]", parent, absolute);
+ this.mapKey = mapKey;
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .filter(Filters.fieldTypeFilter(RecordFieldType.MAP))
+ .map(fieldValue -> {
+ final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + mapKey + "']", valueType);
+ return new MapEntryFieldValue(getMapValue(fieldValue), elementField, fieldValue, mapKey);
+ });
+ }
+
+ private Object getMapValue(final FieldValue fieldValue) {
+ final Map<?, ?> map = (Map<?, ?>) fieldValue.getValue();
+ return map.get(mapKey);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardChildPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardChildPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardChildPath.java
new file mode 100644
index 0000000..16513f9
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardChildPath.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.Record;
+
+public class WildcardChildPath extends RecordPathSegment {
+
+ WildcardChildPath(final RecordPathSegment parent, final boolean absolute) {
+ super("/*", parent, absolute);
+ }
+
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ return getParentPath().evaluate(context)
+ // map to Optional<FieldValue> containing child element
+ .flatMap(fieldVal -> getChildren(fieldVal));
+ }
+
+ private Stream<FieldValue> getChildren(final FieldValue fieldValue) {
+ if (fieldValue == null || fieldValue.getValue() == null || !Filters.isRecord(fieldValue)) {
+ return Stream.empty();
+ }
+
+ final Record record = (Record) fieldValue.getValue();
+ return Filters.presentValues(record.getSchema().getFields().stream()
+ .map(field -> {
+ final Object value = record.getValue(field);
+ if (value == null) {
+ return Optional.empty();
+ }
+
+ return Optional.of(new StandardFieldValue(value, field, fieldValue));
+ }));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
new file mode 100644
index 0000000..c2ce474
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/WildcardIndexPath.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.Map;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.ArrayIndexFieldValue;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.MapEntryFieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
+import org.apache.nifi.serialization.record.type.MapDataType;
+
+public class WildcardIndexPath extends RecordPathSegment {
+
+ WildcardIndexPath(final RecordPathSegment parent, final boolean absolute) {
+ super("[*]", parent, absolute);
+ }
+
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .filter(Filters.fieldTypeFilter(RecordFieldType.MAP, RecordFieldType.ARRAY))
+ .flatMap(fieldValue -> {
+ final RecordFieldType fieldType = fieldValue.getField().getDataType().getFieldType();
+
+ final Object value = fieldValue.getValue();
+ if (value == null) {
+ return Stream.empty();
+ }
+
+ if (fieldType == RecordFieldType.MAP) {
+ final Map<String, ?> map = (Map<String, ?>) value;
+ return map.entrySet().stream()
+ .map(entry -> {
+ final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "['" + entry.getKey() + "']", valueType);
+ return new MapEntryFieldValue(entry.getValue(), elementField, fieldValue, entry.getKey());
+ });
+ } else {
+ final Object[] array = (Object[]) value;
+ return IntStream.range(0, array.length)
+ .mapToObj(index -> {
+ final DataType elementDataType = ((ArrayDataType) fieldValue.getField().getDataType()).getElementType();
+ final RecordField elementField = new RecordField(fieldValue.getField().getFieldName() + "[" + index + "]", elementDataType);
+ return new ArrayIndexFieldValue(array[index], elementField, fieldValue, index);
+ });
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/Filters.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/Filters.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/Filters.java
new file mode 100644
index 0000000..4800d59
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/Filters.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.util;
+
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+
+public class Filters {
+
+ public static Predicate<FieldValue> fieldTypeFilter(final RecordFieldType fieldType, final RecordFieldType... alternativeTypes) {
+ return fieldVal -> {
+ final RecordFieldType recordFieldType = fieldVal.getField().getDataType().getFieldType();
+ if (recordFieldType == fieldType) {
+ return true;
+ }
+
+ for (final RecordFieldType alternate : alternativeTypes) {
+ if (recordFieldType == alternate) {
+ return true;
+ }
+ }
+
+ return false;
+ };
+ }
+
+ public static <T> Stream<T> presentValues(final Stream<Optional<T>> stream) {
+ return stream.filter(opt -> opt.isPresent())
+ .map(opt -> opt.get());
+ }
+
+ public static boolean isRecord(final FieldValue fieldValue) {
+ final DataType dataType = fieldValue.getField().getDataType();
+ final Object value = fieldValue.getValue();
+ return isRecord(dataType, value);
+ }
+
+ public static boolean isRecord(final DataType dataType, final Object value) {
+ if (dataType.getFieldType() == RecordFieldType.RECORD) {
+ return true;
+ }
+
+ if (value == null) {
+ return false;
+ }
+
+ if (value instanceof Record) {
+ return true;
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/RecordPathCache.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/RecordPathCache.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/RecordPathCache.java
new file mode 100644
index 0000000..243ad11
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/util/RecordPathCache.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.nifi.record.path.RecordPath;
+
+public class RecordPathCache {
+ private final Map<String, RecordPath> compiledRecordPaths;
+
+ public RecordPathCache(final int cacheSize) {
+ compiledRecordPaths = new LinkedHashMap<String, RecordPath>() {
+ @Override
+ protected boolean removeEldestEntry(final Map.Entry<String, RecordPath> eldest) {
+ return size() >= cacheSize;
+ }
+ };
+ }
+
+ public RecordPath getCompiled(final String path) {
+ RecordPath compiled;
+ synchronized (this) {
+ compiled = compiledRecordPaths.get(path);
+ }
+
+ if (compiled != null) {
+ return compiled;
+ }
+
+ compiled = RecordPath.compile(path);
+
+ synchronized (this) {
+ final RecordPath existing = compiledRecordPaths.putIfAbsent(path, compiled);
+ if (existing != null) {
+ compiled = existing;
+ }
+ }
+
+ return compiled;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathPropertyNameValidator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathPropertyNameValidator.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathPropertyNameValidator.java
new file mode 100644
index 0000000..0a53335
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathPropertyNameValidator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.validation;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.exception.RecordPathException;
+
+public class RecordPathPropertyNameValidator implements Validator {
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ try {
+ RecordPath.compile(subject);
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(true)
+ .explanation("Valid RecordPath")
+ .build();
+ } catch (final RecordPathException e) {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(false)
+ .explanation("Property Name is not a valid RecordPath value: " + e.getMessage())
+ .build();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathValidator.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathValidator.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathValidator.java
new file mode 100644
index 0000000..ef5e599
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/validation/RecordPathValidator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.validation;
+
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.exception.RecordPathException;
+
+public class RecordPathValidator implements Validator {
+
+ @Override
+ public ValidationResult validate(final String subject, final String input, final ValidationContext context) {
+ if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(true)
+ .explanation("Property uses Expression Language so no further validation is possible")
+ .build();
+ }
+
+ try {
+ RecordPath.compile(input);
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(true)
+ .explanation("Valid RecordPath")
+ .build();
+ } catch (final RecordPathException e) {
+ return new ValidationResult.Builder()
+ .input(input)
+ .subject(subject)
+ .valid(false)
+ .explanation("Property Value is not a valid RecordPath value: " + e.getMessage())
+ .build();
+ }
+ }
+
+}
[5/5] nifi git commit: NIFI-3838: Initial implementation of
RecordPath and UpdateRecord processor
Posted by ma...@apache.org.
NIFI-3838: Initial implementation of RecordPath and UpdateRecord processor
NIFI-3838: Updated version from 1.2.0-SNAPSHOT to 1.3.0-SNAPSHOT; removed unneeded value from AttributeExpression.ResultType enum
NIFI-3838: Addressed PR Review feedback
NIFI-3838: Allow for schemas to be merged together for a record; refactored RecordSetWriterFactory so that there is a method to obtain the schema and then the writer is created with that schema. Added additional unit tests
NIFI-3838: Addressed problems with documentation based on PR Review
NIFI-3838: Fixed checkstyle violation
NIFI-3838: Addressed issue of comparing different types of Number objects
Signed-off-by: Matt Burgess <ma...@apache.org>
This closes #1772
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b1901d5f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b1901d5f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b1901d5f
Branch: refs/heads/master
Commit: b1901d5fe0bf87be3dcce144f13b74eb995be168
Parents: 1c58e78
Author: Mark Payne <ma...@hotmail.com>
Authored: Wed Apr 26 09:02:55 2017 -0400
Committer: Matt Burgess <ma...@apache.org>
Committed: Fri May 12 12:36:52 2017 -0400
----------------------------------------------------------------------
nifi-commons/nifi-record-path/pom.xml | 73 ++
.../apache/nifi/record/path/RecordPathLexer.g | 145 ++++
.../apache/nifi/record/path/RecordPathParser.g | 195 +++++
.../nifi/record/path/ArrayIndexFieldValue.java | 55 ++
.../org/apache/nifi/record/path/FieldValue.java | 54 ++
.../nifi/record/path/MapEntryFieldValue.java | 39 +
.../apache/nifi/record/path/NumericRange.java | 62 ++
.../org/apache/nifi/record/path/RecordPath.java | 108 +++
.../path/RecordPathEvaluationContext.java | 28 +
.../nifi/record/path/RecordPathResult.java | 26 +
.../nifi/record/path/StandardFieldValue.java | 111 +++
.../StandardRecordPathEvaluationContext.java | 44 ++
.../record/path/StandardRecordPathResult.java | 40 +
.../path/exception/RecordPathException.java | 36 +
.../path/filter/BinaryOperatorFilter.java | 62 ++
.../nifi/record/path/filter/EqualsFilter.java | 64 ++
.../record/path/filter/GreaterThanFilter.java | 37 +
.../path/filter/GreaterThanOrEqualFilter.java | 37 +
.../nifi/record/path/filter/LessThanFilter.java | 37 +
.../path/filter/LessThanOrEqualFilter.java | 37 +
.../record/path/filter/NotEqualsFilter.java | 64 ++
.../filter/NumericBinaryOperatorFilter.java | 74 ++
.../record/path/filter/RecordPathFilter.java | 29 +
.../nifi/record/path/paths/ArrayIndexPath.java | 57 ++
.../nifi/record/path/paths/ChildFieldPath.java | 69 ++
.../record/path/paths/CurrentFieldPath.java | 46 ++
.../record/path/paths/DescendantFieldPath.java | 82 ++
.../record/path/paths/LiteralValuePath.java | 39 +
.../record/path/paths/MultiArrayIndexPath.java | 79 ++
.../nifi/record/path/paths/MultiMapKeyPath.java | 57 ++
.../nifi/record/path/paths/ParentPath.java | 46 ++
.../nifi/record/path/paths/PredicatePath.java | 64 ++
.../record/path/paths/RecordPathCompiler.java | 203 +++++
.../record/path/paths/RecordPathSegment.java | 146 ++++
.../apache/nifi/record/path/paths/RootPath.java | 41 +
.../record/path/paths/SingularMapKeyPath.java | 58 ++
.../record/path/paths/WildcardChildPath.java | 60 ++
.../record/path/paths/WildcardIndexPath.java | 77 ++
.../apache/nifi/record/path/util/Filters.java | 74 ++
.../nifi/record/path/util/RecordPathCache.java | 58 ++
.../RecordPathPropertyNameValidator.java | 48 ++
.../path/validation/RecordPathValidator.java | 57 ++
.../apache/nifi/record/path/TestRecordPath.java | 741 +++++++++++++++++++
.../nifi/serialization/record/MapRecord.java | 109 ++-
.../nifi/serialization/record/Record.java | 70 ++
.../nifi/serialization/record/RecordField.java | 5 +-
.../record/util/DataTypeUtils.java | 204 ++++-
nifi-commons/pom.xml | 8 +-
.../src/main/asciidoc/record-path-guide.adoc | 293 ++++++++
.../hadoop/AbstractFetchHDFSRecord.java | 3 +-
.../nifi-mock-record-utils/pom.xml | 4 +
.../serialization/record/MockRecordWriter.java | 9 +-
.../record/MockSchemaRegistry.java | 80 ++
.../main/webapp/WEB-INF/jsp/documentation.jsp | 1 +
.../processors/kafka/pubsub/ConsumerLease.java | 3 +-
.../kafka/pubsub/PublishKafkaRecord_0_10.java | 6 +-
.../kafka/pubsub/util/MockRecordWriter.java | 9 +-
.../processors/parquet/FetchParquetTest.java | 26 +-
.../record/script/ScriptedRecordSetWriter.java | 25 +-
.../script/ScriptedRecordSetWriterTest.groovy | 3 +-
.../groovy/test_record_writer_inline.groovy | 12 +-
.../nifi-standard-processors/pom.xml | 21 +
.../standard/AbstractRecordProcessor.java | 182 +++++
.../nifi/processors/standard/ConvertRecord.java | 127 +---
.../nifi/processors/standard/QueryRecord.java | 2 +-
.../nifi/processors/standard/SplitRecord.java | 2 +-
.../nifi/processors/standard/UpdateRecord.java | 192 +++++
.../org.apache.nifi.processor.Processor | 1 +
.../additionalDetails.html | 2 +-
.../additionalDetails.html | 332 +++++++++
.../processors/standard/TestQueryRecord.java | 15 +-
.../processors/standard/TestUpdateRecord.java | 217 ++++++
.../TestUpdateRecord/input/person.json | 7 +
.../output/person-with-firstname-lastname.json | 5 +
.../output/person-with-firstname.json | 4 +
.../schema/person-with-name-record.avsc | 16 +
.../schema/person-with-name-string-fields.avsc | 10 +
.../schema/person-with-name-string.avsc | 9 +
.../nifi-lookup-services/.gitignore | 1 +
.../serialization/RecordSetWriterFactory.java | 27 +-
.../nifi/avro/AvroReaderWithExplicitSchema.java | 2 -
.../apache/nifi/avro/AvroRecordSetWriter.java | 6 +-
.../org/apache/nifi/csv/CSVRecordReader.java | 2 +-
.../org/apache/nifi/csv/CSVRecordSetWriter.java | 5 +-
.../nifi/json/JsonPathRowRecordReader.java | 2 +-
.../apache/nifi/json/JsonRecordSetWriter.java | 5 +-
.../nifi/json/JsonTreeRowRecordReader.java | 9 +-
.../org/apache/nifi/json/WriteJsonResult.java | 8 +-
.../nifi/text/FreeFormTextRecordSetWriter.java | 10 +-
pom.xml | 5 +
90 files changed, 5420 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/pom.xml b/nifi-commons/nifi-record-path/pom.xml
new file mode 100644
index 0000000..96a7ec8
--- /dev/null
+++ b/nifi-commons/nifi-record-path/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-commons</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>nifi-record-path</artifactId>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr3-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <excludes>**/RecordPathParser.java,**/RecordPathLexer.java</excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/json/address-book.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathLexer.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathLexer.g b/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathLexer.g
new file mode 100644
index 0000000..6240e93
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathLexer.g
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+lexer grammar RecordPathLexer;
+
+@header {
+ package org.apache.nifi.record.path;
+ import org.apache.nifi.record.path.exception.RecordPathException;
+}
+
+@rulecatch {
+ catch(final Exception e) {
+ throw new RecordPathException(e);
+ }
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new RecordPathException(sb.toString());
+ }
+
+ public void recover(RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new RecordPathException(sb.toString());
+ }
+}
+
+
+// PUNCTUATION & SPECIAL CHARACTERS
+CHILD_SEPARATOR : '/';
+DESCENDANT_SEPARATOR : '//';
+LBRACKET : '[';
+RBRACKET : ']';
+NUMBER : '-'? ('0'..'9')+;
+QUOTE : '\'';
+COMMA : ',';
+RANGE : '..';
+CURRENT_FIELD : '.';
+
+WILDCARD : '*';
+
+
+
+// Operators
+LESS_THAN : '<';
+LESS_THAN_EQUAL : '<=';
+GREATER_THAN : '>';
+GREATER_THAN_EQUAL : '>=';
+EQUAL : '=';
+NOT_EQUAL : '!=';
+
+
+WHITESPACE : SPACE+ { skip(); };
+fragment SPACE : ' ' | '\t' | '\n' | '\r' | '\u000C';
+
+
+RAW_FIELD_NAME : (
+ ~('/' | '[' | ']' | '*' | '"' | '\'' | ',' | '\t' | '\r' | '\n' | '0'..'9' | ' ' | '.' | '-' | '=' | '?' | '<' | '>')
+ ~('/' | '[' | ']' | '*' | '"' | '\'' | ',' | '\t' | '\r' | '\n' | '=' | '?' | '<' | '>' | ' ')*
+);
+
+// STRINGS
+STRING_LITERAL
+@init{StringBuilder lBuf = new StringBuilder();}
+ :
+ (
+ '"'
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '"' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '"'
+ )
+ {
+ setText(lBuf.toString());
+ }
+ |
+ (
+ '\''
+ (
+ escaped=ESC {lBuf.append(getText());} |
+ normal = ~( '\'' | '\\' | '\n' | '\r' | '\t' ) { lBuf.appendCodePoint(normal);}
+ )*
+ '\''
+ )
+ {
+ setText(lBuf.toString());
+ }
+ ;
+
+
+fragment
+ESC
+ : '\\'
+ (
+ '"' { setText("\""); }
+ | '\'' { setText("\'"); }
+ | 'r' { setText("\r"); }
+ | 'n' { setText("\n"); }
+ | 't' { setText("\t"); }
+ | '\\' { setText("\\\\"); }
+ | nextChar = ~('"' | '\'' | 'r' | 'n' | 't' | '\\')
+ {
+ StringBuilder lBuf = new StringBuilder(); lBuf.append("\\\\").appendCodePoint(nextChar); setText(lBuf.toString());
+ }
+ )
+ ;
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathParser.g
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathParser.g b/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathParser.g
new file mode 100644
index 0000000..e9bad38
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/antlr3/org/apache/nifi/record/path/RecordPathParser.g
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+parser grammar RecordPathParser;
+
+options {
+ output=AST;
+ tokenVocab=RecordPathLexer;
+}
+
+tokens {
+ PATH_EXPRESSION;
+ PATH;
+ FIELD_NAME;
+ ROOT_REFERENCE;
+ CHILD_REFERENCE;
+ DESCENDANT_REFERENCE;
+ PARENT_REFERENCE;
+ STRING_LIST;
+ ARRAY_INDEX;
+ NUMBER_LIST;
+ NUMBER_RANGE;
+ MAP_KEY;
+ ARRAY_INDEX;
+ PREDICATE;
+ OPERATOR;
+ RELATIVE_PATH;
+}
+
+@header {
+ package org.apache.nifi.record.path;
+ import org.apache.nifi.record.path.exception.RecordPathException;
+}
+
+@members {
+ public void displayRecognitionError(String[] tokenNames, RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new RecordPathException(sb.toString());
+ }
+
+ public void recover(final RecognitionException e) {
+ final StringBuilder sb = new StringBuilder();
+ if ( e.token == null ) {
+ sb.append("Unrecognized token ");
+ } else {
+ sb.append("Unexpected token '").append(e.token.getText()).append("' ");
+ }
+ sb.append("at line ").append(e.line);
+ if ( e.approximateLineInfo ) {
+ sb.append(" (approximately)");
+ }
+ sb.append(", column ").append(e.charPositionInLine);
+ sb.append(". Query: ").append(e.input.toString());
+
+ throw new RecordPathException(sb.toString());
+ }
+}
+
+
+
+
+// Literals
+multipleStringLiterals : STRING_LITERAL (COMMA! STRING_LITERAL)*;
+
+stringList : multipleStringLiterals ->
+ ^(STRING_LIST multipleStringLiterals);
+
+rawOrLiteral : RAW_FIELD_NAME | STRING_LITERAL;
+
+
+
+
+//
+// Filtering
+//
+mapKey : stringList ->
+ ^(MAP_KEY stringList);
+
+range : NUMBER RANGE NUMBER ->
+ ^(NUMBER_RANGE NUMBER NUMBER);
+
+numberOrRange : NUMBER | range;
+
+multipleIndices : numberOrRange (COMMA numberOrRange)* ->
+ ^(NUMBER_LIST numberOrRange numberOrRange*);
+
+arrayIndex : multipleIndices ->
+ ^(ARRAY_INDEX multipleIndices);
+
+indexOrKey : mapKey | arrayIndex | WILDCARD;
+
+index : LBRACKET! indexOrKey RBRACKET!;
+
+
+
+//
+// Predicates
+//
+operator : LESS_THAN | LESS_THAN_EQUAL | GREATER_THAN | GREATER_THAN_EQUAL | EQUAL | NOT_EQUAL;
+
+literal : NUMBER | STRING_LITERAL;
+
+expression : path | literal;
+
+operation : relativePath operator^ expression;
+
+predicate : LBRACKET operation RBRACKET ->
+ ^(PREDICATE operation);
+
+
+
+
+//
+// References
+//
+
+fieldName : rawOrLiteral ->
+ ^(FIELD_NAME rawOrLiteral);
+
+wildcardFieldName : fieldName | WILDCARD;
+
+childReference : CHILD_SEPARATOR wildcardFieldName ->
+ ^(CHILD_REFERENCE wildcardFieldName);
+
+descendantReference : DESCENDANT_SEPARATOR wildcardFieldName ->
+ ^(DESCENDANT_REFERENCE wildcardFieldName);
+
+rootReference : CHILD_SEPARATOR ->
+ ^(CHILD_REFERENCE);
+
+selfReference : CHILD_SEPARATOR! CURRENT_FIELD;
+
+parentReference : CHILD_SEPARATOR RANGE ->
+ ^(PARENT_REFERENCE);
+
+nonSelfFieldRef : childReference | descendantReference | selfReference | parentReference;
+
+fieldRef : nonSelfFieldRef | CURRENT_FIELD;
+
+subPath : fieldRef | index | predicate;
+
+
+
+//
+// Paths
+//
+
+pathSegment : fieldRef subPath* ->
+ ^(PATH fieldRef subPath*);
+
+absolutePathSegment : nonSelfFieldRef subPath* ->
+ ^(PATH nonSelfFieldRef subPath*);
+
+absolutePath : rootReference | absolutePathSegment;
+
+relativePathSegment : nonSelfFieldRef subPath* ->
+ ^(RELATIVE_PATH nonSelfFieldRef subPath*);
+
+initialParentReference : RANGE ->
+ ^(PARENT_REFERENCE);
+
+currentOrParent : CURRENT_FIELD | initialParentReference;
+
+relativePath : currentOrParent relativePathSegment? ->
+ ^(RELATIVE_PATH currentOrParent relativePathSegment?);
+
+path : absolutePath | relativePath;
+
+pathExpression : path EOF ->
+ ^(PATH_EXPRESSION path);
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/ArrayIndexFieldValue.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/ArrayIndexFieldValue.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/ArrayIndexFieldValue.java
new file mode 100644
index 0000000..6a94e4f
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/ArrayIndexFieldValue.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import java.util.Objects;
+
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+
+public class ArrayIndexFieldValue extends StandardFieldValue {
+ private final int index;
+
+ public ArrayIndexFieldValue(final Object value, final RecordField field, final FieldValue parent, final int index) {
+ super(value, field, validateParent(parent));
+ this.index = index;
+ }
+
+ private static FieldValue validateParent(final FieldValue parent) {
+ Objects.requireNonNull(parent, "Cannot create an ArrayIndexFieldValue without a parent");
+ if (RecordFieldType.ARRAY != parent.getField().getDataType().getFieldType()) {
+ throw new IllegalArgumentException("Cannot create an ArrayIndexFieldValue with a parent of type " + parent.getField().getDataType().getFieldType());
+ }
+
+ final Object parentRecord = parent.getValue();
+ if (parentRecord == null) {
+ throw new IllegalArgumentException("Cannot create an ArrayIndexFieldValue without a parent Record");
+ }
+
+ return parent;
+ }
+
+ public int getArrayIndex() {
+ return index;
+ }
+
+ @Override
+ public void updateValue(final Object newValue) {
+ getParentRecord().get().setArrayValue(getField().getFieldName(), getArrayIndex(), newValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/FieldValue.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/FieldValue.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/FieldValue.java
new file mode 100644
index 0000000..88828a5
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/FieldValue.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import java.util.Optional;
+
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+
+public interface FieldValue {
+ /**
+ * @return the value of the field
+ */
+ Object getValue();
+
+ /**
+ * @return the field that the associated value belongs to
+ */
+ RecordField getField();
+
+ /**
+ * @return the FieldValue that applies to the parent of this field, or an empty Optional if the
+ * field represents the 'root' record.
+ */
+ Optional<FieldValue> getParent();
+
+ /**
+ * @return the Record that the field belongs to, or an empty Optional if the
+ * field represents the 'root' record.
+ */
+ Optional<Record> getParentRecord();
+
+ /**
+ * Updates the record to which the field belongs, so that it now has the given value.
+ *
+ * @param newValue the new value to set on the record field
+ */
+ void updateValue(Object newValue);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/MapEntryFieldValue.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/MapEntryFieldValue.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/MapEntryFieldValue.java
new file mode 100644
index 0000000..2553d9a
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/MapEntryFieldValue.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import org.apache.nifi.serialization.record.RecordField;
+
+public class MapEntryFieldValue extends StandardFieldValue {
+ private final String mapKey;
+
+ public MapEntryFieldValue(final Object value, final RecordField field, final FieldValue parent, final String mapKey) {
+ super(value, field, validateParentRecord(parent));
+ this.mapKey = mapKey;
+ }
+
+ public String getMapKey() {
+ return mapKey;
+ }
+
+ @Override
+ public void updateValue(final Object newValue) {
+ getParentRecord().get().setMapValue(getField().getFieldName(), getMapKey(), newValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/NumericRange.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/NumericRange.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/NumericRange.java
new file mode 100644
index 0000000..95ff28c
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/NumericRange.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+public class NumericRange {
+ private final int min;
+ private final int max;
+
+ public NumericRange(final int min, final int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public int getMin() {
+ return min;
+ }
+
+ public int getMax() {
+ return max;
+ }
+
+ @Override
+ public String toString() {
+ return min + ".." + max;
+ }
+
+ @Override
+ public int hashCode() {
+ return 41 + 31 * Integer.hashCode(min) + 31 * Integer.hashCode(max);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof NumericRange)) {
+ return false;
+ }
+
+ final NumericRange other = (NumericRange) obj;
+ return getMin() == other.getMin() && getMax() == other.getMax();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPath.java
new file mode 100644
index 0000000..482390d
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPath.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import static org.apache.nifi.record.path.RecordPathParser.PATH;
+import static org.apache.nifi.record.path.RecordPathParser.CHILD_REFERENCE;
+
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.tree.Tree;
+import org.apache.nifi.record.path.exception.RecordPathException;
+import org.apache.nifi.record.path.paths.RecordPathCompiler;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+import org.apache.nifi.record.path.paths.RootPath;
+import org.apache.nifi.serialization.record.Record;
+
+public interface RecordPath {
+
+ /**
+ * Returns the textual representation of the RecordPath
+ *
+ * @return the textual representation of the RecordPath
+ */
+ String getPath();
+
+ /**
+ * Evaluates the RecordPath against the given Record, returning a RecordPathResult that contains
+ * a FieldValue for each field that matches
+ *
+ * @param record the Record to evaluate
+ * @return a RecordPathResult that contains a FieldValue for each field that matches
+ */
+ RecordPathResult evaluate(Record record);
+
+
+ /**
+ * Evaluates a RecordPath against the given context node. This allows a RecordPath to be evaluated
+ * against a Record via {@link #evaluate(Record)} and then have a Relative RecordPath evaluated against
+ * the results. This method will throw an Exception if this RecordPath is an Absolute RecordPath.
+ *
+ * @param contextNode the context node that represents where in the Record the 'current node' or 'context node' is
+ * @return a RecordPathResult that contains a FieldValue for each field that matches
+ */
+ RecordPathResult evaluate(FieldValue contextNode);
+
+ /**
+ * Indicates whether the RecordPath is an Absolute Path (starts with a '/' character) or a Relative Path (starts with a '.' character).
+ *
+ * @return <code>true</code> if the RecordPath is an Absolute Path, <code>false</code> if the RecordPath is a Relative Path
+ */
+ boolean isAbsolute();
+
+ /**
+ * Compiles a RecordPath from the given text
+ *
+ * @param path the textual representation of the RecordPath
+ * @return the compiled RecordPath
+ * @throws RecordPathException if the given text is not a valid RecordPath
+ */
+ public static RecordPath compile(final String path) throws RecordPathException {
+ try {
+ final CharStream input = new ANTLRStringStream(path);
+ final RecordPathLexer lexer = new RecordPathLexer(input);
+ final CommonTokenStream lexerTokenStream = new CommonTokenStream(lexer);
+
+ final RecordPathParser parser = new RecordPathParser(lexerTokenStream);
+ final Tree tree = (Tree) parser.pathExpression().getTree();
+
+ // We look at the first child, because 'tree' is a PATH_EXPRESSION and we really
+ // want the underlying PATH token.
+ final Tree firstChild = tree.getChild(0);
+
+ final int childType = firstChild.getType();
+ final boolean absolute;
+ final RecordPathSegment rootPath;
+ if (childType == PATH || childType == CHILD_REFERENCE) {
+ rootPath = new RootPath();
+ absolute = true;
+ } else {
+ rootPath = null;
+ absolute = false;
+ }
+
+ return RecordPathCompiler.compile(firstChild, rootPath, absolute);
+ } catch (final RecordPathException e) {
+ throw e;
+ } catch (final Exception e) {
+ throw new RecordPathException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathEvaluationContext.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathEvaluationContext.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathEvaluationContext.java
new file mode 100644
index 0000000..3c5e72e
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathEvaluationContext.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import org.apache.nifi.serialization.record.Record;
+
+public interface RecordPathEvaluationContext {
+ Record getRecord();
+
+ FieldValue getContextNode();
+
+ void setContextNode(FieldValue fieldValue);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathResult.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathResult.java
new file mode 100644
index 0000000..42d99d2
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/RecordPathResult.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import java.util.stream.Stream;
+
+public interface RecordPathResult {
+ String getPath();
+
+ Stream<FieldValue> getSelectedFields();
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java
new file mode 100644
index 0000000..4447fed
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+
+public class StandardFieldValue implements FieldValue {
+ private final Object value;
+ private final RecordField field;
+ private final Optional<FieldValue> parent;
+
+ public StandardFieldValue(final Object value, final RecordField field, final FieldValue parent) {
+ this.value = value;
+ this.field = field;
+ this.parent = Optional.ofNullable(parent);
+ }
+
+ @Override
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public RecordField getField() {
+ return field;
+ }
+
+ @Override
+ public Optional<FieldValue> getParent() {
+ return parent;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value, field, parent);
+ }
+
+ @Override
+ public String toString() {
+ if (value instanceof Object[]) {
+ return Arrays.toString((Object[]) value);
+ }
+
+ return value.toString();
+ }
+
+ protected static FieldValue validateParentRecord(final FieldValue parent) {
+ Objects.requireNonNull(parent, "Cannot create an ArrayIndexFieldValue without a parent");
+ if (!Filters.isRecord(parent)) {
+ if (!parent.getParentRecord().isPresent()) {
+ throw new IllegalArgumentException("Field must have a Parent Record");
+ }
+ }
+
+ final Object parentRecord = parent.getValue();
+ if (parentRecord == null) {
+ throw new IllegalArgumentException("Parent Record cannot be null");
+ }
+
+ return parent;
+ }
+
+ private Optional<Record> getParentRecord(final Optional<FieldValue> fieldValueOption) {
+ if (!fieldValueOption.isPresent()) {
+ return Optional.empty();
+ }
+
+ final FieldValue fieldValue = fieldValueOption.get();
+ if (Filters.isRecord(fieldValue)) {
+ return Optional.ofNullable((Record) fieldValue.getValue());
+ }
+
+ return getParentRecord(fieldValue.getParent());
+ }
+
+ @Override
+ public Optional<Record> getParentRecord() {
+ return getParentRecord(parent);
+ }
+
+ @Override
+ public void updateValue(final Object newValue) {
+ final Optional<Record> parentRecord = getParentRecord();
+ if (!parentRecord.isPresent()) {
+ throw new UnsupportedOperationException("Cannot update the field value because the value is not associated with any record");
+ }
+
+ parentRecord.get().setValue(getField().getFieldName(), newValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathEvaluationContext.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathEvaluationContext.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathEvaluationContext.java
new file mode 100644
index 0000000..4e40ab9
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathEvaluationContext.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import org.apache.nifi.serialization.record.Record;
+
+public class StandardRecordPathEvaluationContext implements RecordPathEvaluationContext {
+ private final Record record;
+ private FieldValue contextNode;
+
+ public StandardRecordPathEvaluationContext(final Record record) {
+ this.record = record;
+ }
+
+ @Override
+ public Record getRecord() {
+ return record;
+ }
+
+ @Override
+ public FieldValue getContextNode() {
+ return contextNode;
+ }
+
+ @Override
+ public void setContextNode(final FieldValue contextNode) {
+ this.contextNode = contextNode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathResult.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathResult.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathResult.java
new file mode 100644
index 0000000..b69656f
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardRecordPathResult.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import java.util.stream.Stream;
+
+public class StandardRecordPathResult implements RecordPathResult {
+ private final String path;
+ private final Stream<FieldValue> selectedFields;
+
+ public StandardRecordPathResult(final String path, final Stream<FieldValue> selectedFields) {
+ this.path = path;
+ this.selectedFields = selectedFields;
+ }
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public Stream<FieldValue> getSelectedFields() {
+ return selectedFields;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/exception/RecordPathException.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/exception/RecordPathException.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/exception/RecordPathException.java
new file mode 100644
index 0000000..faf6be6
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/exception/RecordPathException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.exception;
+
+import org.apache.nifi.processor.exception.ProcessException;
+
+public class RecordPathException extends ProcessException {
+
+ public RecordPathException(final String message) {
+ super(message);
+ }
+
+ public RecordPathException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+
+ public RecordPathException(final Throwable cause) {
+ super(cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/BinaryOperatorFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/BinaryOperatorFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/BinaryOperatorFilter.java
new file mode 100644
index 0000000..6d85436
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/BinaryOperatorFilter.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public abstract class BinaryOperatorFilter implements RecordPathFilter {
+ private final RecordPathSegment lhs;
+ private final RecordPathSegment rhs;
+
+ public BinaryOperatorFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ }
+
+ @Override
+ public Stream<FieldValue> filter(final FieldValue currentNode, final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> rhsStream = rhs.evaluate(context);
+ final Optional<FieldValue> firstMatch = rhsStream
+ .filter(fieldVal -> fieldVal.getValue() != null)
+ .findFirst();
+
+ if (!firstMatch.isPresent()) {
+ return Stream.empty();
+ }
+
+ final FieldValue fieldValue = firstMatch.get();
+ final Object value = fieldValue.getValue();
+
+ final Stream<FieldValue> lhsStream = lhs.evaluate(context);
+ return lhsStream.filter(fieldVal -> test(fieldVal, value));
+ }
+
+ @Override
+ public String toString() {
+ return lhs + " " + getOperator() + " " + rhs;
+ }
+
+ protected abstract String getOperator();
+
+ protected abstract boolean test(FieldValue fieldValue, Object rhsValue);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/EqualsFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/EqualsFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/EqualsFilter.java
new file mode 100644
index 0000000..e03b6df
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/EqualsFilter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class EqualsFilter extends BinaryOperatorFilter {
+
+ public EqualsFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean test(final FieldValue fieldValue, final Object rhsValue) {
+ final Object lhsValue = fieldValue.getValue();
+ if (lhsValue == null) {
+ return rhsValue == null;
+ }
+
+ if (lhsValue instanceof Number) {
+ if (rhsValue instanceof Number) {
+ return compareNumbers((Number) lhsValue, (Number) rhsValue);
+ } else {
+ return false;
+ }
+ } else if (rhsValue instanceof Number) {
+ return false;
+ }
+
+ return lhsValue.equals(rhsValue);
+ }
+
+ @Override
+ protected String getOperator() {
+ return "=";
+ }
+
+ private boolean compareNumbers(final Number lhs, final Number rhs) {
+ final boolean lhsLongCompatible = (lhs instanceof Long || lhs instanceof Integer || lhs instanceof Short || lhs instanceof Byte);
+ final boolean rhsLongCompatible = (rhs instanceof Long || rhs instanceof Integer || lhs instanceof Short || lhs instanceof Byte);
+
+ if (lhsLongCompatible && rhsLongCompatible) {
+ return lhs.longValue() == rhs.longValue();
+ }
+
+ return lhs.doubleValue() == rhs.doubleValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanFilter.java
new file mode 100644
index 0000000..78215a4
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class GreaterThanFilter extends NumericBinaryOperatorFilter {
+
+ public GreaterThanFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Number lhsNumber, final Number rhsNumber) {
+ return Double.compare(lhsNumber.doubleValue(), rhsNumber.doubleValue()) > 0;
+ }
+
+ @Override
+ protected String getOperator() {
+ return ">";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanOrEqualFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanOrEqualFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanOrEqualFilter.java
new file mode 100644
index 0000000..4276582
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/GreaterThanOrEqualFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class GreaterThanOrEqualFilter extends NumericBinaryOperatorFilter {
+
+ public GreaterThanOrEqualFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Number lhsNumber, final Number rhsNumber) {
+ return Double.compare(lhsNumber.doubleValue(), rhsNumber.doubleValue()) >= 0;
+ }
+
+ @Override
+ protected String getOperator() {
+ return ">=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanFilter.java
new file mode 100644
index 0000000..8485437
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class LessThanFilter extends NumericBinaryOperatorFilter {
+
+ public LessThanFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Number lhsNumber, final Number rhsNumber) {
+ return Double.compare(lhsNumber.doubleValue(), rhsNumber.doubleValue()) < 0;
+ }
+
+ @Override
+ protected String getOperator() {
+ return "<";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanOrEqualFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanOrEqualFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanOrEqualFilter.java
new file mode 100644
index 0000000..cec9620
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/LessThanOrEqualFilter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class LessThanOrEqualFilter extends NumericBinaryOperatorFilter {
+
+ public LessThanOrEqualFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean compare(final Number lhsNumber, final Number rhsNumber) {
+ return Double.compare(lhsNumber.doubleValue(), rhsNumber.doubleValue()) <= 0;
+ }
+
+ @Override
+ protected String getOperator() {
+ return "<=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NotEqualsFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NotEqualsFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NotEqualsFilter.java
new file mode 100644
index 0000000..159da4a
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NotEqualsFilter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+
+public class NotEqualsFilter extends BinaryOperatorFilter {
+
+ public NotEqualsFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean test(final FieldValue fieldValue, final Object rhsValue) {
+ final Object lhsValue = fieldValue.getValue();
+ if (lhsValue == null) {
+ return rhsValue != null;
+ }
+
+ if (lhsValue instanceof Number) {
+ if (rhsValue instanceof Number) {
+ return compareNumbers((Number) lhsValue, (Number) rhsValue);
+ } else {
+ return false;
+ }
+ } else if (rhsValue instanceof Number) {
+ return false;
+ }
+
+ return !fieldValue.getValue().equals(rhsValue);
+ }
+
+ private boolean compareNumbers(final Number lhs, final Number rhs) {
+ final boolean lhsLongCompatible = (lhs instanceof Long || lhs instanceof Integer || lhs instanceof Short || lhs instanceof Byte);
+ final boolean rhsLongCompatible = (rhs instanceof Long || rhs instanceof Integer || lhs instanceof Short || lhs instanceof Byte);
+
+ if (lhsLongCompatible && rhsLongCompatible) {
+ return lhs.longValue() != rhs.longValue();
+ }
+
+ return lhs.doubleValue() != rhs.doubleValue();
+ }
+
+ @Override
+ protected String getOperator() {
+ return "!=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NumericBinaryOperatorFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NumericBinaryOperatorFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NumericBinaryOperatorFilter.java
new file mode 100644
index 0000000..f23fc5b
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/NumericBinaryOperatorFilter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.paths.RecordPathSegment;
+import org.apache.nifi.serialization.record.util.DataTypeUtils;
+
+public abstract class NumericBinaryOperatorFilter extends BinaryOperatorFilter {
+
+ public NumericBinaryOperatorFilter(final RecordPathSegment lhs, final RecordPathSegment rhs) {
+ super(lhs, rhs);
+ }
+
+ @Override
+ protected boolean test(final FieldValue fieldValue, final Object rhsValue) {
+ if (fieldValue.getValue() == null) {
+ return false;
+ }
+
+ final Object value = fieldValue.getValue();
+
+ boolean lhsNumeric;
+ final boolean lhsLongCompatible = DataTypeUtils.isLongTypeCompatible(value);
+ final boolean lhsDoubleCompatible;
+ if (lhsLongCompatible) {
+ lhsNumeric = true;
+ } else {
+ lhsDoubleCompatible = DataTypeUtils.isDoubleTypeCompatible(value);
+ lhsNumeric = lhsDoubleCompatible;
+ }
+
+ if (!lhsNumeric) {
+ return false;
+ }
+
+
+ boolean rhsNumeric;
+ final boolean rhsLongCompatible = DataTypeUtils.isLongTypeCompatible(rhsValue);
+ final boolean rhsDoubleCompatible;
+ if (rhsLongCompatible) {
+ rhsNumeric = true;
+ } else {
+ rhsDoubleCompatible = DataTypeUtils.isDoubleTypeCompatible(rhsValue);
+ rhsNumeric = rhsDoubleCompatible;
+ }
+
+ if (!rhsNumeric) {
+ return false;
+ }
+
+ final String fieldName = fieldValue.getField() == null ? "<Anonymous Inner Field>" : fieldValue.getField().getFieldName();
+ final Number lhsNumber = lhsLongCompatible ? DataTypeUtils.toLong(value, fieldName) : DataTypeUtils.toDouble(value, fieldName);
+ final Number rhsNumber = rhsLongCompatible ? DataTypeUtils.toLong(rhsValue, fieldName) : DataTypeUtils.toDouble(rhsValue, fieldName);
+ return compare(lhsNumber, rhsNumber);
+ }
+
+ protected abstract boolean compare(final Number lhsNumber, final Number rhsNumber);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/RecordPathFilter.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/RecordPathFilter.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/RecordPathFilter.java
new file mode 100644
index 0000000..501fca0
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/filter/RecordPathFilter.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.filter;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+
+public interface RecordPathFilter {
+
+ Stream<FieldValue> filter(FieldValue currentNode, RecordPathEvaluationContext context);
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ArrayIndexPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ArrayIndexPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ArrayIndexPath.java
new file mode 100644
index 0000000..287ae2d
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ArrayIndexPath.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.ArrayIndexFieldValue;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
+
+public class ArrayIndexPath extends RecordPathSegment {
+ private final int index;
+
+ ArrayIndexPath(final int index, final RecordPathSegment parent, final boolean absolute) {
+ super("[" + index + "]", parent, absolute);
+ this.index = index;
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .filter(Filters.fieldTypeFilter(RecordFieldType.ARRAY))
+ .filter(fieldValue -> fieldValue.getValue() != null && ((Object[]) fieldValue.getValue()).length >= Math.abs(index) - 1)
+ .map(fieldValue -> {
+ final ArrayDataType arrayDataType = (ArrayDataType) fieldValue.getField().getDataType();
+ final DataType elementDataType = arrayDataType.getElementType();
+ final RecordField arrayField = new RecordField(fieldValue.getField().getFieldName(), elementDataType);
+ final Object[] values = (Object[]) fieldValue.getValue();
+ final int arrayIndex = index < 0 ? values.length + index : index;
+ final RecordField elementField = new RecordField(arrayField.getFieldName() + "[" + arrayIndex + "]", elementDataType);
+ final FieldValue result = new ArrayIndexFieldValue(values[arrayIndex], elementField, fieldValue, arrayIndex);
+ return result;
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java
new file mode 100644
index 0000000..93f3d65
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+
+public class ChildFieldPath extends RecordPathSegment {
+ private final String childName;
+
+ ChildFieldPath(final String childName, final RecordPathSegment parent, final boolean absolute) {
+ super("/" + childName, parent, absolute);
+ this.childName = childName;
+ }
+
+ private FieldValue missingChild(final FieldValue parent) {
+ final RecordField field = new RecordField(childName, RecordFieldType.CHOICE.getChoiceDataType(RecordFieldType.STRING.getDataType(), RecordFieldType.RECORD.getDataType()));
+ return new StandardFieldValue(null, field, parent);
+ }
+
+ private FieldValue getChild(final FieldValue fieldValue) {
+ if (!Filters.isRecord(fieldValue)) {
+ return missingChild(fieldValue);
+ }
+
+ final Record record = (Record) fieldValue.getValue();
+ final Object value = record.getValue(childName);
+ if (value == null) {
+ return missingChild(fieldValue);
+ }
+
+ final Optional<RecordField> field = record.getSchema().getField(childName);
+ if (!field.isPresent()) {
+ return missingChild(fieldValue);
+ }
+
+ return new StandardFieldValue(value, field.get(), fieldValue);
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ return getParentPath().evaluate(context)
+ // map to Optional<FieldValue> containing child element
+ .map(fieldVal -> getChild(fieldVal));
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/CurrentFieldPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/CurrentFieldPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/CurrentFieldPath.java
new file mode 100644
index 0000000..836da65
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/CurrentFieldPath.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+
+public class CurrentFieldPath extends RecordPathSegment {
+
+ public CurrentFieldPath(final RecordPathSegment parentPath, final boolean absolute) {
+ super(parentPath == null ? "." : parentPath.getPath() + "/.", parentPath, absolute);
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final FieldValue contextNode = context.getContextNode();
+ if (contextNode != null) {
+ return Stream.of(contextNode);
+ }
+
+ final RecordPathSegment parentPath = getParentPath();
+ if (parentPath == null) {
+ return Stream.of(context.getContextNode());
+ } else {
+ return parentPath.evaluate(context);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/DescendantFieldPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/DescendantFieldPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/DescendantFieldPath.java
new file mode 100644
index 0000000..8946bc1
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/DescendantFieldPath.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+import org.apache.nifi.record.path.util.Filters;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+
+public class DescendantFieldPath extends RecordPathSegment {
+ private final String descendantName;
+
+ DescendantFieldPath(final String descendantName, final RecordPathSegment parent, final boolean absolute) {
+ super("//" + descendantName, parent, absolute);
+ this.descendantName = descendantName;
+ }
+
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
+
+ return parentResult
+ .flatMap(recordFieldVal -> findDescendants(recordFieldVal).stream());
+ }
+
+ private List<FieldValue> findDescendants(final FieldValue fieldValue) {
+ if (fieldValue == null || fieldValue.getValue() == null) {
+ return Collections.emptyList();
+ }
+ if (!Filters.isRecord(fieldValue)) {
+ return Collections.emptyList();
+ }
+
+ final Record record = (Record) fieldValue.getValue();
+ final List<FieldValue> matchingValues = new ArrayList<>();
+
+ for (final RecordField childField : record.getSchema().getFields()) {
+ if (childField.getFieldName().equals(descendantName) || childField.getAliases().contains(descendantName)) {
+ final Object value = record.getValue(descendantName);
+ if (value != null) {
+ final FieldValue descendantFieldValue = new StandardFieldValue(value, childField, fieldValue);
+ matchingValues.add(descendantFieldValue);
+ }
+ }
+
+ final Object recordValue = record.getValue(childField.getFieldName());
+ if (recordValue == null) {
+ continue;
+ }
+
+ if (Filters.isRecord(childField.getDataType(), recordValue)) {
+ final FieldValue childFieldValue = new StandardFieldValue(recordValue, childField, fieldValue);
+ matchingValues.addAll(findDescendants(childFieldValue));
+ }
+ }
+
+ return matchingValues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/LiteralValuePath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/LiteralValuePath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/LiteralValuePath.java
new file mode 100644
index 0000000..3437934
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/LiteralValuePath.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path.paths;
+
+import java.util.stream.Stream;
+
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPathEvaluationContext;
+import org.apache.nifi.record.path.StandardFieldValue;
+
+public class LiteralValuePath extends RecordPathSegment {
+ private final FieldValue fieldValue;
+
+ LiteralValuePath(final RecordPathSegment parentPath, final Object value, final boolean absolute) {
+ super(String.valueOf(value), parentPath, absolute);
+ this.fieldValue = new StandardFieldValue(value, null, null);
+ }
+
+ @Override
+ public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
+ return Stream.of(fieldValue);
+ }
+
+}
[2/5] nifi git commit: NIFI-3838: Initial implementation of
RecordPath and UpdateRecord processor
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
index fd8660a..f3beab4 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp
@@ -61,6 +61,7 @@
<li class="component-item"><a class="document-link getting-started" href="html/getting-started.html" target="component-usage">Getting Started</a></li>
<li class="component-item"><a class="document-link user-guide" href="html/user-guide.html" target="component-usage">User Guide</a></li>
<li class="component-item"><a class="document-link expression-language-guide" href="html/expression-language-guide.html" target="component-usage">Expression Language Guide</a></li>
+ <li class="component-item"><a class="document-link record-path-guide" href="html/record-path-guide.html" target="component-usage">RecordPath Guide</a></li>
<li class="component-item"><a class="document-link admin-guide" href="html/administration-guide.html" target="component-usage">Admin Guide</a></li>
</ul>
<span class="no-matching no-components hidden">No matching guides</span>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
index ac24e1f..1ccce07 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/ConsumerLease.java
@@ -415,7 +415,8 @@ public abstract class ConsumerLease implements Closeable, ConsumerRebalanceListe
try {
final RecordSetWriter writer;
try {
- writer = writerFactory.createWriter(logger, flowFile, new ByteArrayInputStream(records.get(0).value()));
+ final RecordSchema schema = writerFactory.getSchema(flowFile, new ByteArrayInputStream(records.get(0).value()));
+ writer = writerFactory.createWriter(logger, schema);
} catch (final Exception e) {
logger.error(
"Failed to obtain a Record Writer for serializing Kafka messages. This generally happens because the "
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublishKafkaRecord_0_10.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublishKafkaRecord_0_10.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublishKafkaRecord_0_10.java
index f96a575..442ccc5 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublishKafkaRecord_0_10.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/main/java/org/apache/nifi/processors/kafka/pubsub/PublishKafkaRecord_0_10.java
@@ -59,6 +59,7 @@ import org.apache.nifi.serialization.RecordReader;
import org.apache.nifi.serialization.RecordReaderFactory;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.serialization.RecordWriter;
+import org.apache.nifi.serialization.record.RecordSchema;
@Tags({"Apache", "Kafka", "Record", "csv", "json", "avro", "logs", "Put", "Send", "Message", "PubSub", "0.10.x"})
@CapabilityDescription("Sends the contents of a FlowFile as individual records to Apache Kafka using the Kafka 0.10.x Producer API. "
@@ -325,7 +326,10 @@ public class PublishKafkaRecord_0_10 extends AbstractProcessor {
final RecordWriter writer;
try (final InputStream in = new BufferedInputStream(session.read(flowFile))) {
- writer = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class).createWriter(getLogger(), flowFile, in);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+ final RecordSchema schema = writerFactory.getSchema(flowFile, in);
+
+ writer = writerFactory.createWriter(getLogger(), schema);
} catch (final Exception e) {
getLogger().error("Failed to create a Record Writer for {}; routing to failure", new Object[] {flowFile, e});
session.transfer(flowFile, REL_FAILURE);
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
index 22d7249..a1abda4 100644
--- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
+++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-0-10-processors/src/test/java/org/apache/nifi/processors/kafka/pubsub/util/MockRecordWriter.java
@@ -25,10 +25,12 @@ import java.util.Collections;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
public class MockRecordWriter extends AbstractControllerService implements RecordSetWriterFactory {
@@ -51,7 +53,12 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
}
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream in) {
+ public RecordSchema getSchema(FlowFile flowFile, InputStream content) throws SchemaNotFoundException, IOException {
+ return null;
+ }
+
+ @Override
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema) {
return new RecordSetWriter() {
@Override
public WriteResult write(final RecordSet rs, final OutputStream out) throws IOException {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/processors/parquet/FetchParquetTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/processors/parquet/FetchParquetTest.java b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/processors/parquet/FetchParquetTest.java
index a4f1513..5b4819a 100644
--- a/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/processors/parquet/FetchParquetTest.java
+++ b/nifi-nar-bundles/nifi-parquet-bundle/nifi-parquet-processors/src/test/java/org/apache/nifi/processors/parquet/FetchParquetTest.java
@@ -16,6 +16,17 @@
*/
package org.apache.nifi.processors.parquet;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
@@ -33,6 +44,7 @@ import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
import org.apache.nifi.serialization.record.MockRecordWriter;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
@@ -44,18 +56,6 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.when;
-
public class FetchParquetTest {
static final String DIRECTORY = "target";
@@ -221,7 +221,7 @@ public class FetchParquetTest {
final RecordSetWriterFactory recordSetWriterFactory = Mockito.mock(RecordSetWriterFactory.class);
when(recordSetWriterFactory.getIdentifier()).thenReturn("mock-writer-factory");
- when(recordSetWriterFactory.createWriter(any(ComponentLog.class), any(FlowFile.class), any(InputStream.class))).thenReturn(recordSetWriter);
+ when(recordSetWriterFactory.createWriter(any(ComponentLog.class), any(RecordSchema.class))).thenReturn(recordSetWriter);
testRunner.addControllerService("mock-writer-factory", recordSetWriterFactory);
testRunner.enableControllerService(recordSetWriterFactory);
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/ScriptedRecordSetWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/ScriptedRecordSetWriter.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/ScriptedRecordSetWriter.java
index dd0be2b..1dde047 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/ScriptedRecordSetWriter.java
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/ScriptedRecordSetWriter.java
@@ -28,6 +28,7 @@ import org.apache.nifi.processors.script.ScriptEngineConfigurator;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.RecordSchema;
import javax.script.Invocable;
import javax.script.ScriptException;
@@ -45,16 +46,21 @@ import java.util.HashSet;
@Restricted("Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.")
public class ScriptedRecordSetWriter extends AbstractScriptedRecordFactory<RecordSetWriterFactory> implements RecordSetWriterFactory {
+ @Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
super.onEnabled(context);
}
+ public RecordSetWriter createWriter(ComponentLog logger, FlowFile flowFile, InputStream in) throws SchemaNotFoundException, IOException {
+ return createWriter(logger, getSchema(flowFile, in));
+ }
+
@Override
- public RecordSetWriter createWriter(ComponentLog logger, FlowFile flowFile, InputStream flowFileContent) throws SchemaNotFoundException, IOException {
+ public RecordSetWriter createWriter(ComponentLog logger, RecordSchema schema) throws SchemaNotFoundException, IOException {
if (recordFactory.get() != null) {
try {
- return recordFactory.get().createWriter(logger, flowFile, flowFileContent);
+ return recordFactory.get().createWriter(logger, schema);
} catch (UndeclaredThrowableException ute) {
throw new IOException(ute.getCause());
}
@@ -69,6 +75,7 @@ public class ScriptedRecordSetWriter extends AbstractScriptedRecordFactory<Recor
* @param scriptBody An input stream associated with the script content
* @return Whether the script was successfully reloaded
*/
+ @Override
protected boolean reloadScript(final String scriptBody) {
// note we are starting here with a fresh listing of validation
// results since we are (re)loading a new/updated script. any
@@ -142,4 +149,18 @@ public class ScriptedRecordSetWriter extends AbstractScriptedRecordFactory<Recor
// return whether there was any issues loading the configured script
return results.isEmpty();
}
+
+ @Override
+ public RecordSchema getSchema(FlowFile flowFile, InputStream in) throws SchemaNotFoundException, IOException {
+ final RecordSetWriterFactory writerFactory = recordFactory.get();
+ if (writerFactory == null) {
+ return null;
+ }
+
+ try {
+ return writerFactory.getSchema(flowFile, in);
+ } catch (UndeclaredThrowableException ute) {
+ throw new IOException(ute.getCause());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/groovy/org/apache/nifi/record/script/ScriptedRecordSetWriterTest.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/groovy/org/apache/nifi/record/script/ScriptedRecordSetWriterTest.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/groovy/org/apache/nifi/record/script/ScriptedRecordSetWriterTest.groovy
index d4e7d5a..2e1c03d 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/groovy/org/apache/nifi/record/script/ScriptedRecordSetWriterTest.groovy
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/groovy/org/apache/nifi/record/script/ScriptedRecordSetWriterTest.groovy
@@ -103,7 +103,8 @@ class ScriptedRecordSetWriterTest {
MockFlowFile mockFlowFile = new MockFlowFile(1L)
InputStream inStream = new ByteArrayInputStream('Flow file content not used'.bytes)
- RecordSetWriter recordSetWriter = recordSetWriterFactory.createWriter(logger, mockFlowFile, inStream)
+ def schema = recordSetWriterFactory.getSchema(mockFlowFile, inStream)
+ RecordSetWriter recordSetWriter = recordSetWriterFactory.createWriter(logger, schema)
assertNotNull(recordSetWriter)
def recordSchema = new SimpleRecordSchema(
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_record_writer_inline.groovy
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_record_writer_inline.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_record_writer_inline.groovy
index e17b701..4fae4fe 100644
--- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_record_writer_inline.groovy
+++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_record_writer_inline.groovy
@@ -16,6 +16,10 @@
*/
import groovy.xml.MarkupBuilder
+
+import java.io.IOException
+import java.io.InputStream
+
import org.apache.nifi.controller.AbstractControllerService
import org.apache.nifi.flowfile.FlowFile
import org.apache.nifi.logging.ComponentLog
@@ -24,6 +28,7 @@ import org.apache.nifi.serialization.RecordSetWriter
import org.apache.nifi.serialization.RecordSetWriterFactory
import org.apache.nifi.serialization.WriteResult
import org.apache.nifi.serialization.record.Record
+import org.apache.nifi.serialization.record.RecordSchema
import org.apache.nifi.serialization.record.RecordSet
import org.apache.nifi.stream.io.NonCloseableOutputStream
@@ -73,7 +78,12 @@ class GroovyRecordSetWriter implements RecordSetWriter {
class GroovyRecordSetWriterFactory extends AbstractControllerService implements RecordSetWriterFactory {
@Override
- RecordSetWriter createWriter(ComponentLog logger, FlowFile flowFile, InputStream flowFileContent) throws SchemaNotFoundException, IOException {
+ public RecordSchema getSchema(FlowFile flowFile, InputStream inStream) throws SchemaNotFoundException, IOException {
+ return null;
+ }
+
+ @Override
+ RecordSetWriter createWriter(ComponentLog logger, RecordSchema schema) throws SchemaNotFoundException, IOException {
return new GroovyRecordSetWriter()
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 7de1dab..1372735 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -34,6 +34,10 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-path</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service-api</artifactId>
</dependency>
<dependency>
@@ -233,6 +237,17 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-serialization-services</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-schema-registry-service-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<scope>test</scope>
@@ -435,6 +450,12 @@
<exclude>src/test/resources/TestExtractGrok/apache.log</exclude>
<exclude>src/test/resources/TestExtractGrok/simple_text.log</exclude>
<exclude>src/test/resources/TestExtractGrok/patterns</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/input/person.json</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname.json</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc</exclude>
+ <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc</exclude>
<!-- This file is copied from https://github.com/jeremyh/jBCrypt because the binary is compiled for Java 8 and we must support Java 7 -->
<exclude>src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractRecordProcessor.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractRecordProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractRecordProcessor.java
new file mode 100644
index 0000000..b6cc83b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractRecordProcessor.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.io.StreamCallback;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.RecordSet;
+
+public abstract class AbstractRecordProcessor extends AbstractProcessor {
+
+ static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for reading incoming data")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(true)
+ .build();
+ static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("Specifies the Controller Service to use for writing out the records")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .required(true)
+ .build();
+
+ static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("FlowFiles that are successfully transformed will be routed to this relationship")
+ .build();
+ static final Relationship REL_FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("If a FlowFile cannot be transformed from the configured input format to the configured output format, "
+ + "the unchanged FlowFile will be routed to this relationship")
+ .build();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>();
+ properties.add(RECORD_READER);
+ properties.add(RECORD_WRITER);
+ return properties;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ final Set<Relationship> relationships = new HashSet<>();
+ relationships.add(REL_SUCCESS);
+ relationships.add(REL_FAILURE);
+ return relationships;
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+ if (flowFile == null) {
+ return;
+ }
+
+ final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
+ final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+ final RecordSetWriter writer;
+ final RecordSchema writeSchema;
+ try (final InputStream rawIn = session.read(flowFile);
+ final InputStream in = new BufferedInputStream(rawIn)) {
+ writeSchema = writerFactory.getSchema(flowFile, in);
+ writer = writerFactory.createWriter(getLogger(), writeSchema);
+ } catch (final Exception e) {
+ getLogger().error("Failed to convert records for {}; will route to failure", new Object[] {flowFile, e});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ final AtomicReference<WriteResult> writeResultRef = new AtomicReference<>();
+
+ final FlowFile original = flowFile;
+ try {
+ flowFile = session.write(flowFile, new StreamCallback() {
+ @Override
+ public void process(final InputStream in, final OutputStream out) throws IOException {
+ try (final RecordReader reader = readerFactory.createRecordReader(original, in, getLogger())) {
+
+ final RecordSet recordSet = new RecordSet() {
+ @Override
+ public RecordSchema getSchema() throws IOException {
+ try {
+ return reader.getSchema();
+ } catch (final MalformedRecordException e) {
+ throw new IOException(e);
+ } catch (final Exception e) {
+ throw new ProcessException(e);
+ }
+ }
+
+ @Override
+ public Record next() throws IOException {
+ try {
+ final Record record = reader.nextRecord();
+ if (record == null) {
+ return null;
+ }
+
+ return AbstractRecordProcessor.this.process(record, writeSchema, original, context);
+ } catch (final MalformedRecordException e) {
+ throw new IOException(e);
+ } catch (final Exception e) {
+ throw new ProcessException(e);
+ }
+ }
+ };
+
+ final WriteResult writeResult = writer.write(recordSet, out);
+ writeResultRef.set(writeResult);
+
+ } catch (final SchemaNotFoundException | MalformedRecordException e) {
+ throw new ProcessException("Could not parse incoming data", e);
+ }
+ }
+ });
+ } catch (final Exception e) {
+ getLogger().error("Failed to convert {}", new Object[] {flowFile, e});
+ session.transfer(flowFile, REL_FAILURE);
+ return;
+ }
+
+ final WriteResult writeResult = writeResultRef.get();
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType());
+ attributes.putAll(writeResult.getAttributes());
+
+ flowFile = session.putAllAttributes(flowFile, attributes);
+ session.transfer(flowFile, REL_SUCCESS);
+ session.adjustCounter("Records Processed", writeResult.getRecordCount(), false);
+ getLogger().info("Successfully converted {} records for {}", new Object[] {writeResult.getRecordCount(), flowFile});
+ }
+
+ protected abstract Record process(Record record, RecordSchema writeSchema, FlowFile flowFile, ProcessContext context);
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertRecord.java
index 2b2caa4..f87339d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertRecord.java
@@ -17,18 +17,6 @@
package org.apache.nifi.processors.standard;
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -38,28 +26,16 @@ import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.io.StreamCallback;
-import org.apache.nifi.schema.access.SchemaNotFoundException;
-import org.apache.nifi.serialization.MalformedRecordException;
-import org.apache.nifi.serialization.RecordReader;
-import org.apache.nifi.serialization.RecordReaderFactory;
-import org.apache.nifi.serialization.RecordSetWriter;
-import org.apache.nifi.serialization.RecordSetWriterFactory;
-import org.apache.nifi.serialization.WriteResult;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
@EventDriven
@SupportsBatching
@InputRequirement(Requirement.INPUT_REQUIRED)
@SideEffectFree
-@Tags({"convert", "generic", "schema", "json", "csv", "avro", "log", "logs", "freeform", "text"})
+@Tags({"convert", "record", "generic", "schema", "json", "csv", "avro", "log", "logs", "freeform", "text"})
@WritesAttributes({
@WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Record Writer"),
@WritesAttribute(attribute = "record.count", description = "The number of records in the FlowFile")
@@ -70,102 +46,11 @@ import org.apache.nifi.serialization.WriteResult;
+ "the output schema can have a field named \"balance\" with a type of string, double, or float. If any field is present in the input that is not present in the output, "
+ "the field will be left out of the output. If any field is specified in the output schema but is not present in the input data/schema, then the field will not be "
+ "present in the output or will have a null value, depending on the writer.")
-public class ConvertRecord extends AbstractProcessor {
-
- static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
- .name("record-reader")
- .displayName("Record Reader")
- .description("Specifies the Controller Service to use for reading incoming data")
- .identifiesControllerService(RecordReaderFactory.class)
- .required(true)
- .build();
- static final PropertyDescriptor RECORD_WRITER = new PropertyDescriptor.Builder()
- .name("record-writer")
- .displayName("Record Writer")
- .description("Specifies the Controller Service to use for writing out the records")
- .identifiesControllerService(RecordSetWriterFactory.class)
- .required(true)
- .build();
-
- static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("FlowFiles that are successfully transformed will be routed to this relationship")
- .build();
- static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("If a FlowFile cannot be transformed from the configured input format to the configured output format, "
- + "the unchanged FlowFile will be routed to this relationship")
- .build();
-
- @Override
- protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- final List<PropertyDescriptor> properties = new ArrayList<>();
- properties.add(RECORD_READER);
- properties.add(RECORD_WRITER);
- return properties;
- }
-
- @Override
- public Set<Relationship> getRelationships() {
- final Set<Relationship> relationships = new HashSet<>();
- relationships.add(REL_SUCCESS);
- relationships.add(REL_FAILURE);
- return relationships;
- }
+public class ConvertRecord extends AbstractRecordProcessor {
@Override
- public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
- FlowFile flowFile = session.get();
- if (flowFile == null) {
- return;
- }
-
- final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
- final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
- final RecordSetWriter writer;
- try (final InputStream rawIn = session.read(flowFile);
- final InputStream in = new BufferedInputStream(rawIn)) {
- writer = writerFactory.createWriter(getLogger(), flowFile, in);
- } catch (final Exception e) {
- getLogger().error("Failed to convert records for {}; will route to failure", new Object[] {flowFile, e});
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
-
- final AtomicReference<WriteResult> writeResultRef = new AtomicReference<>();
-
- final FlowFile original = flowFile;
- try {
- flowFile = session.write(flowFile, new StreamCallback() {
- @Override
- public void process(final InputStream in, final OutputStream out) throws IOException {
- try (final RecordReader reader = readerFactory.createRecordReader(original, in, getLogger())) {
-
- final WriteResult writeResult = writer.write(reader.createRecordSet(), out);
- writeResultRef.set(writeResult);
-
- } catch (final SchemaNotFoundException | MalformedRecordException e) {
- throw new ProcessException("Could not parse incoming data", e);
- }
- }
- });
- } catch (final ProcessException e) {
- getLogger().error("Failed to convert {}", new Object[] {flowFile, e});
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
-
- final WriteResult writeResult = writeResultRef.get();
-
- final Map<String, String> attributes = new HashMap<>();
- attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
- attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType());
- attributes.putAll(writeResult.getAttributes());
-
- flowFile = session.putAllAttributes(flowFile, attributes);
- session.transfer(flowFile, REL_SUCCESS);
- session.adjustCounter("Records Converted", writeResult.getRecordCount(), false);
- getLogger().info("Successfully converted {} records for {}", new Object[] {writeResult.getRecordCount(), flowFile});
+ protected Record process(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context) {
+ return record;
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
index 5982908..3cb5cce 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryRecord.java
@@ -254,7 +254,7 @@ public class QueryRecord extends AbstractProcessor {
final RecordSetWriter resultSetWriter;
try (final InputStream rawIn = session.read(original);
final InputStream in = new BufferedInputStream(rawIn)) {
- resultSetWriter = resultSetWriterFactory.createWriter(getLogger(), original, in);
+ resultSetWriter = resultSetWriterFactory.createWriter(getLogger(), resultSetWriterFactory.getSchema(original, in));
}
for (final PropertyDescriptor descriptor : context.getProperties().keySet()) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitRecord.java
index 05aa98c..62ca521 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitRecord.java
@@ -136,7 +136,7 @@ public class SplitRecord extends AbstractProcessor {
final RecordSetWriter writer;
try (final InputStream rawIn = session.read(original);
final InputStream in = new BufferedInputStream(rawIn)) {
- writer = writerFactory.createWriter(getLogger(), original, in);
+ writer = writerFactory.createWriter(getLogger(), writerFactory.getSchema(original, in));
} catch (final Exception e) {
getLogger().error("Failed to create Record Writer for {}; routing to failure", new Object[] {original, e});
session.transfer(original, REL_FAILURE);
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
new file mode 100644
index 0000000..9151cde
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UpdateRecord.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.SeeAlso;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.record.path.FieldValue;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.record.path.util.RecordPathCache;
+import org.apache.nifi.record.path.validation.RecordPathPropertyNameValidator;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+
+
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
+@Tags({"update", "record", "generic", "schema", "json", "csv", "avro", "log", "logs", "freeform", "text"})
+@CapabilityDescription("Updates the contents of a FlowFile that contains Record-oriented data (i.e., data that can be read via a RecordReader and written by a RecordWriter). "
+ + "This Processor requires that at least one user-defined Property be added. The name of the Property should indicate a RecordPath that determines the field that should "
+ + "be updated. The value of the Property is either a replacement value (optionally making use of the Expression Language) or is itself a RecordPath that extracts a value from "
+ + "the Record. Whether the Property value is determined to be a RecordPath or a literal value depends on the configuration of the <Replacement Value Strategy> Property.")
+@SeeAlso({ConvertRecord.class})
+public class UpdateRecord extends AbstractRecordProcessor {
+
+ private volatile RecordPathCache recordPathCache;
+ private volatile List<String> recordPaths;
+
+ static final AllowableValue LITERAL_VALUES = new AllowableValue("literal-value", "Literal Value",
+ "The value entered for a Property (after Expression Language has been evaluated) is the desired value to update the Record Fields with.");
+ static final AllowableValue RECORD_PATH_VALUES = new AllowableValue("record-path-value", "Record Path Value",
+ "The value entered for a Property (after Expression Language has been evaluated) is not the literal value to use but rather is a Record Path "
+ + "that should be evaluated against the Record, and the result of the RecordPath will be used to update the Record. Note that if this option is selected, "
+ + "and the Record Path results in multiple values for a given Record, the input FlowFile will be routed to the 'failure' Relationship.");
+
+ static final PropertyDescriptor REPLACEMENT_VALUE_STRATEGY = new PropertyDescriptor.Builder()
+ .name("replacement-value-strategy")
+ .displayName("Replacement Value Strategy")
+ .description("Specifies how to interpret the configured replacement values")
+ .allowableValues(LITERAL_VALUES, RECORD_PATH_VALUES)
+ .defaultValue(LITERAL_VALUES.getValue())
+ .expressionLanguageSupported(false)
+ .required(true)
+ .build();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ final List<PropertyDescriptor> properties = new ArrayList<>(super.getSupportedPropertyDescriptors());
+ properties.add(REPLACEMENT_VALUE_STRATEGY);
+ return properties;
+ }
+
+ @Override
+ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
+ return new PropertyDescriptor.Builder()
+ .name(propertyDescriptorName)
+ .description("Specifies the value to use to replace fields in the record that match the RecordPath: " + propertyDescriptorName)
+ .required(false)
+ .dynamic(true)
+ .expressionLanguageSupported(true)
+ .addValidator(new RecordPathPropertyNameValidator())
+ .build();
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final boolean containsDynamic = validationContext.getProperties().keySet().stream()
+ .anyMatch(property -> property.isDynamic());
+
+ if (containsDynamic) {
+ return Collections.emptyList();
+ }
+
+ return Collections.singleton(new ValidationResult.Builder()
+ .subject("User-defined Properties")
+ .valid(false)
+ .explanation("At least one RecordPath must be specified")
+ .build());
+ }
+
+ @OnScheduled
+ public void createRecordPaths(final ProcessContext context) {
+ recordPathCache = new RecordPathCache(context.getProperties().size() * 2);
+
+ final List<String> recordPaths = new ArrayList<>(context.getProperties().size() - 2);
+ for (final PropertyDescriptor property : context.getProperties().keySet()) {
+ if (property.isDynamic()) {
+ recordPaths.add(property.getName());
+ }
+ }
+
+ this.recordPaths = recordPaths;
+ }
+
+ @Override
+ protected Record process(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context) {
+ final boolean evaluateValueAsRecordPath = context.getProperty(REPLACEMENT_VALUE_STRATEGY).getValue().equals(RECORD_PATH_VALUES.getValue());
+
+ // Incorporate the RecordSchema that we will use for writing records into the Schema that we have
+ // for the record, because it's possible that the updates to the record will not be valid otherwise.
+ record.incorporateSchema(writeSchema);
+
+ for (final String recordPathText : recordPaths) {
+ final RecordPath recordPath = recordPathCache.getCompiled(recordPathText);
+ final RecordPathResult result = recordPath.evaluate(record);
+
+ final String replacementValue = context.getProperty(recordPathText).evaluateAttributeExpressions(flowFile).getValue();
+ if (evaluateValueAsRecordPath) {
+ final RecordPath replacementRecordPath = recordPathCache.getCompiled(replacementValue);
+
+ // If we have an Absolute RecordPath, we need to evaluate the RecordPath only once against the Record.
+ // If the RecordPath is a Relative Path, then we have to evaluate it against each FieldValue.
+ if (replacementRecordPath.isAbsolute()) {
+ processAbsolutePath(replacementRecordPath, result.getSelectedFields(), record, replacementValue);
+ } else {
+ processRelativePath(replacementRecordPath, result.getSelectedFields(), record, replacementValue);
+ }
+ } else {
+ result.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(replacementValue));
+ }
+ }
+
+ return record;
+ }
+
+ private void processAbsolutePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record, final String replacementValue) {
+ final RecordPathResult replacementResult = replacementRecordPath.evaluate(record);
+ final Object replacementObject = getReplacementObject(replacementResult, replacementValue);
+ destinationFields.forEach(fieldVal -> fieldVal.updateValue(replacementObject));
+ }
+
+ private void processRelativePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record, final String replacementValue) {
+ destinationFields.forEach(fieldVal -> {
+ final RecordPathResult replacementResult = replacementRecordPath.evaluate(fieldVal);
+ final Object replacementObject = getReplacementObject(replacementResult, replacementValue);
+ fieldVal.updateValue(replacementObject);
+ });
+ }
+
+ private Object getReplacementObject(final RecordPathResult recordPathResult, final String replacementValue) {
+ final List<FieldValue> selectedFields = recordPathResult.getSelectedFields().collect(Collectors.toList());
+
+ if (selectedFields.size() > 1) {
+ throw new ProcessException("Cannot update Record because the Replacement Record Path \"" + replacementValue + "\" yielded "
+ + selectedFields.size() + " results but this Processor only supports a single result.");
+ }
+
+ if (selectedFields.isEmpty()) {
+ return null;
+ } else {
+ return selectedFields.get(0).getValue();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
index b4085c8..221fe0a 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -102,3 +102,4 @@ org.apache.nifi.processors.standard.FetchDistributedMapCache
org.apache.nifi.processors.standard.ListFTP
org.apache.nifi.processors.standard.FetchFTP
org.apache.nifi.processors.standard.UpdateCounter
+org.apache.nifi.processors.standard.UpdateRecord
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.QueryRecord/additionalDetails.html
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.QueryRecord/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.QueryRecord/additionalDetails.html
index 93bbe2a..d289069 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.QueryRecord/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.QueryRecord/additionalDetails.html
@@ -18,7 +18,7 @@
<meta charset="utf-8" />
<title>QueryRecord</title>
- <link rel="stylesheet" href="../../css/component-usage.css" type="text/css" />
+ <link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.UpdateRecord/additionalDetails.html
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.UpdateRecord/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.UpdateRecord/additionalDetails.html
new file mode 100644
index 0000000..c20f48a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.UpdateRecord/additionalDetails.html
@@ -0,0 +1,332 @@
+<!DOCTYPE html>
+<html lang="en">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <head>
+ <meta charset="utf-8" />
+ <title>UpdateRecord</title>
+
+ <link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
+ </head>
+
+ <body>
+ <p>
+ UpdateRecord makes use of the NiFi <a href="/nifi-docs/html/record-path-guide.html">
+ RecordPath Domain-Specific Language (DSL)</a> to allow the user
+ to indicate which field(s) in the Record should be updated. Users do this by adding a User-defined
+ Property to the Processor's configuration. The name of the User-defined Property must be the
+ RecordPath text that should be evaluated against each Record. The value of the Property specifies
+ what value should go into that selected Record field.
+ </p>
+
+ <p>
+ When specifying the replacement value (the value of the User-defined Property), the user is able
+ to specify a literal value such as the number <code>10</code>; an Expression Language Expression
+ to reference FlowFile attributes, such as <code>${filename}</code>; or another RecordPath path
+ from which to retrieve the desired value from the Record itself. Whether the value entered should
+ be interpreted as a literal or a RecordPath path is determined by the value of the <Replacement
+ Value Strategy> Property.
+ </p>
+
+ <p>
+ If a RecordPath is given and does not match any field in an input Record, that Property will be skipped
+ and all other Properties will still be evaluated. If the RecordPath matches exactly one field, that field
+ will be updated with the corresponding value. If multiple fields match the RecordPath, then all fields that
+ match will be updated. If the replacement value is itself a RecordPath that does not match, then a
+ <code>null</code> value will be set for the field. For instances where this is not the desired behavior,
+ RecordPath predicates can be used to filter the fields that match so that no fields will be selected.
+ See <a href="/nifi-docs/html/record-path-guide.html#predicates">RecordPath Predicates</a> for more information.
+ </p>
+
+ <p>
+ Below, we lay out some examples in order to provide clarity about the Processor's behavior. For all of
+ the examples below, consider the example to operate on the following set of 2 (JSON) records:
+ </p>
+
+<code>
+<pre>
+ [{
+ "id": 17,
+ "name": "John",
+ "child": {
+ "id": "1"
+ },
+ "siblingIds": [4, 8],
+ "siblings": [
+ { "name": "Jeremy", "id": 4 },
+ { "name": "Julia", "id": 8 }
+ ]
+ },
+ {
+ "id": 98,
+ "name": "Jane",
+ "child": {
+ "id": 2
+ },
+ "gender": "F",
+ "siblingIds": [],
+ "siblings": []
+ }]
+</pre>
+</code>
+
+ <p>
+ For brevity, we will omit the corresponding schema and configuration of the RecordReader and
+ RecordWriter. Otherwise, consider the following set of Properties are configured for the Processor
+ and their associated outputs.
+ </p>
+
+ <h3>Example 1 - Replace with Literal</h3>
+
+ <p>
+ Here, we will replace the name of each Record with the name 'Jeremy' and set the gender to 'M':
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Replacement Value Strategy</td>
+ <td>Literal Value</td>
+ </tr>
+ <tr>
+ <td>/name</td>
+ <td>Jeremy</td>
+ </tr>
+ <tr>
+ <td>/gender</td>
+ <td>M</td>
+ </tr>
+ </table>
+
+ <p>
+ This will yield the following output:
+ </p>
+
+<code>
+<pre>
+ [{
+ "id": 17,
+ "name": "Jeremy",
+ "child": {
+ "id": "1"
+ },
+ "gender": "M",
+ "siblingIds": [4, 8],
+ "siblings": [
+ { "name": "Jeremy", "id": 4 },
+ { "name": "Julia", "id": 8 }
+ ]
+ },
+ {
+ "id": 98,
+ "name": "Jeremy",
+ "child": {
+ "id": 2
+ },
+ "gender": "M",
+ "siblingIds": [],
+ "siblings": []
+ }]
+</pre>
+</code>
+
+ <p>
+ Note that even though the first record did not have a "gender" field in the input, one
+ will be added after the "child" field, as that's where the field is located in the schema.
+ </p>
+
+
+ <h3>Example 2 - Replace with RecordPath</h3>
+
+ <p>
+ This example will replace the value in one field of the Record with the value from another field.
+ For this example, consider the following set of Properties:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Replacement Value Strategy</td>
+ <td>Record Path Value</td>
+ </tr>
+ <tr>
+ <td>/name</td>
+ <td>/siblings[0]/name</td>
+ </tr>
+ </table>
+
+ <p>
+ This will yield the following output:
+ </p>
+
+<code>
+<pre>
+ [{
+ "id": 17,
+ "name": "Jeremy",
+ "child": {
+ "id": "1"
+ },
+ "siblingIds": [4, 8],
+ "siblings": [
+ { "name": "Jeremy", "id": 4 },
+ { "name": "Julia", "id": 8 }
+ ]
+ },
+ {
+ "id": 98,
+ "name": null,
+ "child": {
+ "id": 2
+ },
+ "gender": "F",
+ "siblingIds": [],
+ "siblings": []
+ }]
+</pre>
+</code>
+
+
+
+ <h3>Example 3 - Replace with Relative RecordPath</h3>
+
+ <p>
+ In the above example, we replaced the value of field based on another RecordPath. That RecordPath was an "absolute RecordPath,"
+ meaning that it starts with a "slash" character (<code>/</code>) and therefore it specifies the path from the "root" or "outer most" element.
+ However, sometimes we want to reference a field in such a way that we defined the RecordPath relative to the field being updated. This example
+ does just that. For each of the siblings given in the "siblings" array, we will replace the sibling's name with their id's. To do so, we will
+ configure the processor with the following properties:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Replacement Value Strategy</td>
+ <td>Record Path Value</td>
+ </tr>
+ <tr>
+ <td>/siblings[*]/name</td>
+ <td>../id</td>
+ </tr>
+ </table>
+
+ <p>
+ Note that the RecordPath that was given for the value starts with <code>..</code>, which is a reference to the parent. We do this because the field
+ that we are going to update is the "name" field of the sibling. To get to the associated "id" field, we need to go to the "name" field's parent and then
+ to its "id" child field. The above example results in the following output:
+ </p>
+
+<code>
+<pre>
+ [{
+ "id": 17,
+ "name": "John",
+ "child": {
+ "id": "1"
+ },
+ "siblingIds": [4, 8],
+ "siblings": [
+ { "name": "4", "id": 4 },
+ { "name": "8", "id": 8 }
+ ]
+ },
+ {
+ "id": 98,
+ "name": "Jane",
+ "child": {
+ "id": 2
+ },
+ "gender": "F",
+ "siblingIds": [],
+ "siblings": []
+ }]
+</pre>
+</code>
+
+
+
+
+ <h3>Example 4 - Replace Multiple Values</h3>
+
+ <p>
+ This example will replace the value of all fields that have the name "id", regardless of
+ where in the Record hierarchy the field is found. The value that it uses references the Expression Language,
+ so for this example, let's assume that the incoming FlowFile has an attribute named "replacement.id" that
+ has a value of "91":
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Replacement Value Strategy</td>
+ <td>Literal Value</td>
+ </tr>
+ <tr>
+ <td>//id</td>
+ <td>${replacement.id}</td>
+ </tr>
+ </table>
+
+ <p>
+ This will yield the following output:
+ </p>
+
+<code>
+<pre>
+ [{
+ "id": 91,
+ "name": "John",
+ "child": {
+ "id": "91"
+ },
+ "siblingIds": [4, 8],
+ "siblings": [
+ { "name": "Jeremy", "id": 91 },
+ { "name": "Julia", "id": 91 }
+ ]
+ },
+ {
+ "id": 91,
+ "name": "Jane",
+ "child": {
+ "id": 91
+ },
+ "gender": "F",
+ "siblingIds": [],
+ "siblings": []
+ }]
+</pre>
+</code>
+
+ <p>
+ It is also worth noting that in this example, some of the "id" fields were of type STRING, while
+ others were of type INT. This is okay because the RecordReaders and RecordWriters should handle
+ these simple type coercions for us.
+ </p>
+
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestQueryRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestQueryRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestQueryRecord.java
index 70d3e87..023fba7 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestQueryRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestQueryRecord.java
@@ -20,13 +20,17 @@ import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.MockRecordParser;
import org.apache.nifi.serialization.record.MockRecordWriter;
import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
@@ -41,6 +45,7 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
public class TestQueryRecord {
@@ -256,7 +261,15 @@ public class TestQueryRecord {
}
@Override
- public RecordSetWriter createWriter(ComponentLog logger, FlowFile flowFile, InputStream in) {
+ public RecordSchema getSchema(FlowFile flowFile, InputStream content) throws SchemaNotFoundException, IOException {
+ final List<RecordField> recordFields = columnNames.stream()
+ .map(name -> new RecordField(name, RecordFieldType.STRING.getDataType()))
+ .collect(Collectors.toList());
+ return new SimpleRecordSchema(recordFields);
+ }
+
+ @Override
+ public RecordSetWriter createWriter(ComponentLog logger, RecordSchema schema) {
return new RecordSetWriter() {
@Override
public WriteResult write(final RecordSet rs, final OutputStream out) throws IOException {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java
new file mode 100644
index 0000000..6d88e57
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+
+import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.serialization.record.MockRecordParser;
+import org.apache.nifi.serialization.record.MockRecordWriter;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestUpdateRecord {
+
+ private TestRunner runner;
+ private MockRecordParser readerService;
+ private MockRecordWriter writerService;
+
+ @Before
+ public void setup() throws InitializationException {
+ readerService = new MockRecordParser();
+ writerService = new MockRecordWriter("header", false);
+
+ runner = TestRunners.newTestRunner(UpdateRecord.class);
+ runner.addControllerService("reader", readerService);
+ runner.enableControllerService(readerService);
+ runner.addControllerService("writer", writerService);
+ runner.enableControllerService(writerService);
+
+ runner.setProperty(UpdateRecord.RECORD_READER, "reader");
+ runner.setProperty(UpdateRecord.RECORD_WRITER, "writer");
+
+ readerService.addSchemaField("name", RecordFieldType.STRING);
+ readerService.addSchemaField("age", RecordFieldType.INT);
+ }
+
+
+ @Test
+ public void testLiteralReplacementValue() {
+ runner.setProperty("/name", "Jane Doe");
+ runner.enqueue("");
+
+ readerService.addRecord("John Doe", 35);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final MockFlowFile out = runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0);
+ out.assertContentEquals("header\nJane Doe,35\n");
+ }
+
+ @Test
+ public void testLiteralReplacementValueExpressionLanguage() {
+ runner.setProperty("/name", "${newName}");
+ runner.enqueue("", Collections.singletonMap("newName", "Jane Doe"));
+
+ readerService.addRecord("John Doe", 35);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final MockFlowFile out = runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0);
+ out.assertContentEquals("header\nJane Doe,35\n");
+ }
+
+ @Test
+ public void testRecordPathReplacementValue() {
+ runner.setProperty("/name", "/age");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES.getValue());
+ runner.enqueue("");
+
+ readerService.addRecord("John Doe", 35);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final MockFlowFile out = runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0);
+ out.assertContentEquals("header\n35,35\n");
+ }
+
+ @Test
+ public void testInvalidRecordPathUsingExpressionLanguage() {
+ runner.setProperty("/name", "${recordPath}");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES.getValue());
+ runner.enqueue("", Collections.singletonMap("recordPath", "hello"));
+
+ readerService.addRecord("John Doe", 35);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_FAILURE, 1);
+ }
+
+ @Test
+ public void testReplaceWithMissingRecordPath() throws InitializationException {
+ readerService = new MockRecordParser();
+ readerService.addSchemaField("name", RecordFieldType.STRING);
+ readerService.addSchemaField("siblings", RecordFieldType.ARRAY);
+ runner.addControllerService("reader", readerService);
+ runner.enableControllerService(readerService);
+
+ runner.setProperty("/name", "/siblings[0]/name");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES.getValue());
+
+ runner.enqueue("", Collections.singletonMap("recordPath", "hello"));
+
+ readerService.addRecord("John Doe", null);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0);
+ mff.assertContentEquals("header\n,\n");
+ }
+
+ @Test
+ public void testRelativePath() throws InitializationException {
+ readerService = new MockRecordParser();
+ readerService.addSchemaField("name", RecordFieldType.STRING);
+ readerService.addSchemaField("nickname", RecordFieldType.STRING);
+ runner.addControllerService("reader", readerService);
+ runner.enableControllerService(readerService);
+
+ runner.setProperty("/name", "../nickname");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES.getValue());
+
+ runner.enqueue("", Collections.singletonMap("recordPath", "hello"));
+
+ readerService.addRecord("John Doe", "Johnny");
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final MockFlowFile mff = runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0);
+ mff.assertContentEquals("header\nJohnny,Johnny\n");
+ }
+
+ @Test
+ public void testChangingSchema() throws InitializationException, IOException {
+ final JsonTreeReader jsonReader = new JsonTreeReader();
+ runner.addControllerService("reader", jsonReader);
+
+ final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc")));
+ final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc")));
+
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText);
+ runner.enableControllerService(jsonReader);
+
+ final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
+ runner.addControllerService("writer", jsonWriter);
+ runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
+ runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
+ runner.enableControllerService(jsonWriter);
+
+ runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person.json"));
+ runner.setProperty("/name", "/name/first");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES);
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/person-with-firstname.json")));
+ runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
+ }
+
+ @Test
+ public void testAddFieldNotInInputRecord() throws InitializationException, IOException {
+ final JsonTreeReader jsonReader = new JsonTreeReader();
+ runner.addControllerService("reader", jsonReader);
+
+ final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc")));
+ final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc")));
+
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText);
+ runner.enableControllerService(jsonReader);
+
+ final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
+ runner.addControllerService("writer", jsonWriter);
+ runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
+ runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
+ runner.enableControllerService(jsonWriter);
+
+ runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person.json"));
+ runner.setProperty("/firstName", "/name/first");
+ runner.setProperty("/lastName", "/name/last");
+ runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES);
+
+ runner.run();
+ runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
+ final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json")));
+ runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person.json
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person.json
new file mode 100644
index 0000000..7538381
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person.json
@@ -0,0 +1,7 @@
+{
+ "id": 485,
+ "name": {
+ "last": "Doe",
+ "first": "John"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json
new file mode 100644
index 0000000..8e6dea7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json
@@ -0,0 +1,5 @@
+[ {
+ "id" : 485,
+ "firstName" : "John",
+ "lastName" : "Doe"
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname.json
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname.json
new file mode 100644
index 0000000..b91adc5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-firstname.json
@@ -0,0 +1,4 @@
+[ {
+ "id" : 485,
+ "name" : "John"
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc
new file mode 100644
index 0000000..baa6c05
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc
@@ -0,0 +1,16 @@
+{
+ "name": "personWithNameRecord",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "id", "type": "int" },
+ { "name": "name", "type": {
+ "type": "record",
+ "name": "nameRecord",
+ "fields": [
+ { "name": "last", "type": "string" },
+ { "name": "first", "type": "string" }
+ ]
+ }}
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc
new file mode 100644
index 0000000..eb6c644
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc
@@ -0,0 +1,10 @@
+{
+ "name": "personWithNameRecord",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "id", "type": "int" },
+ { "name": "firstName", "type": "string" },
+ { "name": "lastName", "type": "string" }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc
new file mode 100644
index 0000000..2fdfd1e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc
@@ -0,0 +1,9 @@
+{
+ "name": "personWithNameRecord",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "id", "type": "int" },
+ { "name": "name", "type": "string" }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/.gitignore
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/.gitignore b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/.gitignore
@@ -0,0 +1 @@
+/bin/
[3/5] nifi git commit: NIFI-3838: Initial implementation of
RecordPath and UpdateRecord processor
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
new file mode 100644
index 0000000..4f9b53d
--- /dev/null
+++ b/nifi-commons/nifi-record-path/src/test/java/org/apache/nifi/record/path/TestRecordPath.java
@@ -0,0 +1,741 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.record.path;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.junit.Test;
+
+public class TestRecordPath {
+
+ @Test
+ public void testCompile() {
+ System.out.println(RecordPath.compile("/person/name/last"));
+ System.out.println(RecordPath.compile("/person[2]"));
+ System.out.println(RecordPath.compile("//person[2]"));
+ System.out.println(RecordPath.compile("/person/child[1]//sibling/name"));
+ }
+
+ @Test
+ public void testChildField() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ final Record record = new MapRecord(schema, values);
+
+ assertEquals(48, RecordPath.compile("/id").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertEquals(record, RecordPath.compile("/id").evaluate(record).getSelectedFields().findFirst().get().getParentRecord().get());
+
+ assertEquals("John Doe", RecordPath.compile("/name").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertEquals(record, RecordPath.compile("/name").evaluate(record).getSelectedFields().findFirst().get().getParentRecord().get());
+
+ assertEquals(accountRecord, RecordPath.compile("/mainAccount").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertEquals(record, RecordPath.compile("/mainAccount").evaluate(record).getSelectedFields().findFirst().get().getParentRecord().get());
+
+ assertEquals(1, RecordPath.compile("/mainAccount/id").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertEquals(accountRecord, RecordPath.compile("/mainAccount/id").evaluate(record).getSelectedFields().findFirst().get().getParentRecord().get());
+
+ assertEquals(123.45D, RecordPath.compile("/mainAccount/balance").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ assertEquals(accountRecord, RecordPath.compile("/mainAccount/id").evaluate(record).getSelectedFields().findFirst().get().getParentRecord().get());
+ }
+
+ @Test
+ public void testRootRecord() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue fieldValue = RecordPath.compile("/").evaluate(record).getSelectedFields().findFirst().get();
+ assertEquals(Optional.empty(), fieldValue.getParent());
+ assertEquals(record, fieldValue.getValue());
+ }
+
+ @Test
+ public void testWildcardChild() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/mainAccount/*").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(2, fieldValues.size());
+
+ for (final FieldValue fieldValue : fieldValues) {
+ assertEquals(accountRecord, fieldValue.getParentRecord().get());
+ }
+
+ assertEquals("id", fieldValues.get(0).getField().getFieldName());
+ assertEquals(1, fieldValues.get(0).getValue());
+
+ assertEquals("balance", fieldValues.get(1).getField().getFieldName());
+ assertEquals(123.45D, fieldValues.get(1).getValue());
+ }
+
+ @Test
+ public void testWildcardWithArray() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("accounts", new Object[] {accountRecord});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/*[0]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertTrue(fieldValue.getField().getFieldName().startsWith("accounts["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ assertEquals(accountRecord, fieldValue.getValue());
+ }
+
+ @Test
+ public void testDescendantField() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("//id").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(2, fieldValues.size());
+
+ final FieldValue first = fieldValues.get(0);
+ final FieldValue second = fieldValues.get(1);
+
+ assertEquals(RecordFieldType.INT, first.getField().getDataType().getFieldType());
+ assertEquals(RecordFieldType.INT, second.getField().getDataType().getFieldType());
+
+ assertEquals(48, first.getValue());
+ assertEquals(1, second.getValue());
+ }
+
+ @Test
+ public void testParent() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("//id/..").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(2, fieldValues.size());
+
+ final FieldValue first = fieldValues.get(0);
+ final FieldValue second = fieldValues.get(1);
+
+ assertEquals(RecordFieldType.RECORD, first.getField().getDataType().getFieldType());
+ assertEquals(RecordFieldType.RECORD, second.getField().getDataType().getFieldType());
+
+ assertEquals(record, first.getValue());
+ assertEquals(accountRecord, second.getValue());
+ }
+
+ @Test
+ public void testMapKey() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("city", "New York");
+ attributes.put("state", "NY");
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("attributes", attributes);
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue fieldValue = RecordPath.compile("/attributes['city']").evaluate(record).getSelectedFields().findFirst().get();
+ assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertEquals("New York", fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ @Test
+ public void testMapWildcard() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("city", "New York");
+ attributes.put("state", "NY");
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("attributes", attributes);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/attributes[*]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(2, fieldValues.size());
+
+ assertEquals("New York", fieldValues.get(0).getValue());
+ assertEquals("NY", fieldValues.get(1).getValue());
+
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+ }
+
+ @Test
+ public void testMapMultiKey() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, String> attributes = new HashMap<>();
+ attributes.put("city", "New York");
+ attributes.put("state", "NY");
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("attributes", attributes);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/attributes['city', 'state']").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(2, fieldValues.size());
+
+ assertEquals("New York", fieldValues.get(0).getValue());
+ assertEquals("NY", fieldValues.get(1).getValue());
+
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("attributes['"));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+ }
+
+ @Test
+ public void testEscapedFieldName() {
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+ fields.add(new RecordField("name,date", RecordFieldType.STRING.getDataType()));
+
+ final RecordSchema schema = new SimpleRecordSchema(fields);
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name,date", "John Doe");
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue fieldValue = RecordPath.compile("/'name,date'").evaluate(record).getSelectedFields().findFirst().get();
+ assertEquals("name,date", fieldValue.getField().getFieldName());
+ assertEquals("John Doe", fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ @Test
+ public void testSingleArrayIndex() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue fieldValue = RecordPath.compile("/numbers[3]").evaluate(record).getSelectedFields().findFirst().get();
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(3, fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ @Test
+ public void testSingleArrayRange() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/numbers[0..1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ assertEquals(2, fieldValues.size());
+ for (int i = 0; i < 1; i++) {
+ assertEquals(i, fieldValues.get(0).getValue());
+ }
+ }
+
+
+ @Test
+ public void testMultiArrayIndex() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/numbers[3,6, -1, -2]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ int i = 0;
+ final int[] expectedValues = new int[] {3, 6, 9, 8};
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(expectedValues[i++], fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ }
+
+ @Test
+ public void testMultiArrayIndexWithRanges() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("numbers", new Object[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+ final Record record = new MapRecord(schema, values);
+
+ List<FieldValue> fieldValues = RecordPath.compile("/numbers[0, 2, 4..7, 9]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ int[] expectedValues = new int[] {0, 2, 4, 5, 6, 7, 9};
+ assertEquals(expectedValues.length, fieldValues.size());
+ for (int i = 0; i < expectedValues.length; i++) {
+ assertEquals(expectedValues[i], fieldValues.get(i).getValue());
+ }
+
+ fieldValues = RecordPath.compile("/numbers[0..-1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+ expectedValues = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ assertEquals(expectedValues.length, fieldValues.size());
+ for (int i = 0; i < expectedValues.length; i++) {
+ assertEquals(expectedValues[i], fieldValues.get(i).getValue());
+ }
+
+
+ fieldValues = RecordPath.compile("/numbers[-1..-1]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+ expectedValues = new int[] {9};
+ assertEquals(expectedValues.length, fieldValues.size());
+ for (int i = 0; i < expectedValues.length; i++) {
+ assertEquals(expectedValues[i], fieldValues.get(i).getValue());
+ }
+
+ fieldValues = RecordPath.compile("/numbers[*]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ for (final FieldValue fieldValue : fieldValues) {
+ assertTrue(fieldValue.getField().getFieldName().startsWith("numbers["));
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+ expectedValues = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ assertEquals(expectedValues.length, fieldValues.size());
+ for (int i = 0; i < expectedValues.length; i++) {
+ assertEquals(expectedValues[i], fieldValues.get(i).getValue());
+ }
+
+ fieldValues = RecordPath.compile("/xx[1,2,3]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(0, fieldValues.size());
+ }
+
+ @Test
+ public void testEqualsPredicate() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ values.put("numbers", new Object[] {1, 2, 3, 4, 4, 4, 5});
+ final Record record = new MapRecord(schema, values);
+
+
+ List<FieldValue> fieldValues = RecordPath.compile("/numbers[0..-1][. = 4]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(3, fieldValues.size());
+
+ for (final FieldValue fieldValue : fieldValues) {
+ final String fieldName = fieldValue.getField().getFieldName();
+ assertTrue(Pattern.compile("numbers\\[\\d\\]").matcher(fieldName).matches());
+ assertEquals(RecordFieldType.INT, fieldValue.getField().getDataType().getFieldType());
+ assertEquals(4, fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ fieldValues = RecordPath.compile("//id[. = 48]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+ final FieldValue fieldValue = fieldValues.get(0);
+
+ assertEquals("id", fieldValue.getField().getFieldName());
+ assertEquals(RecordFieldType.INT.getDataType(), fieldValue.getField().getDataType());
+ assertEquals(48, fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ }
+
+ @Test
+ public void testRelativePath() {
+ final Map<String, Object> accountValues = new HashMap<>();
+ accountValues.put("id", 1);
+ accountValues.put("balance", 123.45D);
+ final Record accountRecord = new MapRecord(getAccountSchema(), accountValues);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("mainAccount", accountRecord);
+ final Record record = new MapRecord(schema, values);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("/mainAccount/././balance/.").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals(accountRecord, fieldValue.getParentRecord().get());
+ assertEquals(123.45D, fieldValue.getValue());
+ assertEquals("balance", fieldValue.getField().getFieldName());
+ }
+
+ @Test
+ public void testCompareToLiteral() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("numbers", new Object[] {0, 1, 2});
+ final Record record = new MapRecord(schema, values);
+
+ List<FieldValue> fieldValues = RecordPath.compile("/id[. > 42]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ fieldValues = RecordPath.compile("/id[. < 42]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(0, fieldValues.size());
+ }
+
+ @Test
+ public void testCompareToAbsolute() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("numbers", new Object[] {0, 1, 2});
+ final Record record = new MapRecord(schema, values);
+
+ List<FieldValue> fieldValues = RecordPath.compile("/numbers[0..-1][. < /id]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(3, fieldValues.size());
+
+ fieldValues = RecordPath.compile("/id[. > /numbers[-1]]").evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+ }
+
+ @Test
+ public void testCompareWithEmbeddedPaths() {
+ final Map<String, Object> accountValues1 = new HashMap<>();
+ accountValues1.put("id", 1);
+ accountValues1.put("balance", 10_000.00D);
+ final Record accountRecord1 = new MapRecord(getAccountSchema(), accountValues1);
+
+ final Map<String, Object> accountValues2 = new HashMap<>();
+ accountValues2.put("id", 2);
+ accountValues2.put("balance", 48.02D);
+ final Record accountRecord2 = new MapRecord(getAccountSchema(), accountValues2);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("accounts", new Object[] {accountRecord1, accountRecord2});
+ final Record record = new MapRecord(schema, values);
+
+ final RecordPath recordPath = RecordPath.compile("/accounts[0..-1][./balance > 100]");
+ List<FieldValue> fieldValues = recordPath.evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("accounts[0]", fieldValue.getField().getFieldName());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ assertEquals(accountRecord1, fieldValue.getValue());
+ }
+
+ @Test
+ public void testPredicateInMiddleOfPath() {
+ final Map<String, Object> accountValues1 = new HashMap<>();
+ accountValues1.put("id", 1);
+ accountValues1.put("balance", 10_000.00D);
+ final Record accountRecord1 = new MapRecord(getAccountSchema(), accountValues1);
+
+ final Map<String, Object> accountValues2 = new HashMap<>();
+ accountValues2.put("id", 2);
+ accountValues2.put("balance", 48.02D);
+ final Record accountRecord2 = new MapRecord(getAccountSchema(), accountValues2);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("accounts", new Object[] {accountRecord1, accountRecord2});
+ final Record record = new MapRecord(schema, values);
+
+ final RecordPath recordPath = RecordPath.compile("/accounts[0..-1][./balance > 100]/id");
+ List<FieldValue> fieldValues = recordPath.evaluate(record).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("id", fieldValue.getField().getFieldName());
+ assertEquals(accountRecord1, fieldValue.getParentRecord().get());
+ assertEquals(1, fieldValue.getValue());
+ }
+
+ @Test
+ public void testUpdateValueOnMatchingFields() {
+ final Map<String, Object> accountValues1 = new HashMap<>();
+ accountValues1.put("id", 1);
+ accountValues1.put("balance", 10_000.00D);
+ final Record accountRecord1 = new MapRecord(getAccountSchema(), accountValues1);
+
+ final Map<String, Object> accountValues2 = new HashMap<>();
+ accountValues2.put("id", 2);
+ accountValues2.put("balance", 48.02D);
+ final Record accountRecord2 = new MapRecord(getAccountSchema(), accountValues2);
+
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ values.put("accounts", new Object[] {accountRecord1, accountRecord2});
+ final Record record = new MapRecord(schema, values);
+
+ final RecordPath recordPath = RecordPath.compile("/accounts[0..-1][./balance > 100]/id");
+ recordPath.evaluate(record).getSelectedFields().findFirst().get().updateValue(100);
+
+ assertEquals(48, record.getValue("id"));
+ assertEquals(100, accountRecord1.getValue("id"));
+ assertEquals(2, accountRecord2.getValue("id"));
+ }
+
+ @Test
+ public void testPredicateDoesNotIncludeFieldsThatDontHaveRelativePath() {
+ final List<RecordField> addressFields = new ArrayList<>();
+ addressFields.add(new RecordField("city", RecordFieldType.STRING.getDataType()));
+ addressFields.add(new RecordField("state", RecordFieldType.STRING.getDataType()));
+ addressFields.add(new RecordField("zip", RecordFieldType.STRING.getDataType()));
+ final RecordSchema addressSchema = new SimpleRecordSchema(addressFields);
+
+ final List<RecordField> detailsFields = new ArrayList<>();
+ detailsFields.add(new RecordField("position", RecordFieldType.STRING.getDataType()));
+ detailsFields.add(new RecordField("managerName", RecordFieldType.STRING.getDataType()));
+ final RecordSchema detailsSchema = new SimpleRecordSchema(detailsFields);
+
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+ fields.add(new RecordField("address", RecordFieldType.RECORD.getRecordDataType(addressSchema)));
+ fields.add(new RecordField("details", RecordFieldType.RECORD.getRecordDataType(detailsSchema)));
+ final RecordSchema recordSchema = new SimpleRecordSchema(fields);
+
+ final Record record = new MapRecord(recordSchema, new HashMap<>());
+ record.setValue("name", "John Doe");
+
+ final Record addressRecord = new MapRecord(addressSchema, new HashMap<>());
+ addressRecord.setValue("city", "San Francisco");
+ addressRecord.setValue("state", "CA");
+ addressRecord.setValue("zip", "12345");
+ record.setValue("address", addressRecord);
+
+ final Record detailsRecord = new MapRecord(detailsSchema, new HashMap<>());
+ detailsRecord.setValue("position", "Developer");
+ detailsRecord.setValue("managerName", "Jane Doe");
+ record.setValue("details", detailsRecord);
+
+ final RecordPath recordPath = RecordPath.compile("/*[./state != 'NY']");
+ final RecordPathResult result = recordPath.evaluate(record);
+ final List<FieldValue> fieldValues = result.getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("address", fieldValue.getField().getFieldName());
+
+ assertEquals("12345", RecordPath.compile("/*[./state != 'NY']/zip").evaluate(record).getSelectedFields().findFirst().get().getValue());
+ }
+
+ @Test
+ public void testPredicateWithAbsolutePath() {
+ final List<RecordField> addressFields = new ArrayList<>();
+ addressFields.add(new RecordField("city", RecordFieldType.STRING.getDataType()));
+ addressFields.add(new RecordField("state", RecordFieldType.STRING.getDataType()));
+ addressFields.add(new RecordField("zip", RecordFieldType.STRING.getDataType()));
+ final RecordSchema addressSchema = new SimpleRecordSchema(addressFields);
+
+ final List<RecordField> detailsFields = new ArrayList<>();
+ detailsFields.add(new RecordField("position", RecordFieldType.STRING.getDataType()));
+ detailsFields.add(new RecordField("preferredState", RecordFieldType.STRING.getDataType()));
+ final RecordSchema detailsSchema = new SimpleRecordSchema(detailsFields);
+
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+ fields.add(new RecordField("address1", RecordFieldType.RECORD.getRecordDataType(addressSchema)));
+ fields.add(new RecordField("address2", RecordFieldType.RECORD.getRecordDataType(addressSchema)));
+ fields.add(new RecordField("details", RecordFieldType.RECORD.getRecordDataType(detailsSchema)));
+ final RecordSchema recordSchema = new SimpleRecordSchema(fields);
+
+ final Record record = new MapRecord(recordSchema, new HashMap<>());
+ record.setValue("name", "John Doe");
+
+ final Record address1Record = new MapRecord(addressSchema, new HashMap<>());
+ address1Record.setValue("city", "San Francisco");
+ address1Record.setValue("state", "CA");
+ address1Record.setValue("zip", "12345");
+ record.setValue("address1", address1Record);
+
+ final Record address2Record = new MapRecord(addressSchema, new HashMap<>());
+ address2Record.setValue("city", "New York");
+ address2Record.setValue("state", "NY");
+ address2Record.setValue("zip", "01234");
+ record.setValue("address2", address2Record);
+
+ final Record detailsRecord = new MapRecord(detailsSchema, new HashMap<>());
+ detailsRecord.setValue("position", "Developer");
+ detailsRecord.setValue("preferredState", "NY");
+ record.setValue("details", detailsRecord);
+
+ final RecordPath recordPath = RecordPath.compile("/*[./state = /details/preferredState]");
+ final RecordPathResult result = recordPath.evaluate(record);
+ final List<FieldValue> fieldValues = result.getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("address2", fieldValue.getField().getFieldName());
+ }
+
+ @Test
+ public void testRelativePathOnly() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue recordFieldValue = new StandardFieldValue(record, new RecordField("record", RecordFieldType.RECORD.getDataType()), null);
+
+ final List<FieldValue> fieldValues = RecordPath.compile("./name").evaluate(recordFieldValue).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("John Doe", fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ assertEquals("name", fieldValue.getField().getFieldName());
+ }
+
+ @Test
+ public void testRelativePathAgainstNonRecordField() {
+ final RecordSchema schema = new SimpleRecordSchema(getDefaultFields());
+ final Map<String, Object> values = new HashMap<>();
+ values.put("id", 48);
+ values.put("name", "John Doe");
+ final Record record = new MapRecord(schema, values);
+
+ final FieldValue recordFieldValue = new StandardFieldValue(record, new RecordField("root", RecordFieldType.RECORD.getRecordDataType(record.getSchema())), null);
+ final FieldValue nameFieldValue = new StandardFieldValue("John Doe", new RecordField("name", RecordFieldType.STRING.getDataType()), recordFieldValue);
+
+ final List<FieldValue> fieldValues = RecordPath.compile(".").evaluate(nameFieldValue).getSelectedFields().collect(Collectors.toList());
+ assertEquals(1, fieldValues.size());
+
+ final FieldValue fieldValue = fieldValues.get(0);
+ assertEquals("John Doe", fieldValue.getValue());
+ assertEquals(record, fieldValue.getParentRecord().get());
+ assertEquals("name", fieldValue.getField().getFieldName());
+
+ fieldValue.updateValue("Jane Doe");
+ assertEquals("Jane Doe", record.getValue("name"));
+ }
+
+ private List<RecordField> getDefaultFields() {
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+ fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+ fields.add(new RecordField("attributes", RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType())));
+ fields.add(new RecordField("mainAccount", RecordFieldType.RECORD.getRecordDataType(getAccountSchema())));
+ fields.add(new RecordField("numbers", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType())));
+
+ final DataType accountDataType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
+ final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountDataType);
+ final RecordField accountsField = new RecordField("accounts", accountsType);
+ fields.add(accountsField);
+ return fields;
+ }
+
+ private RecordSchema getAccountSchema() {
+ final List<RecordField> accountFields = new ArrayList<>();
+ accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType()));
+ accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
+
+ final RecordSchema accountSchema = new SimpleRecordSchema(accountFields);
+ return accountSchema;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
index 8d98c33..ca33e32 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/MapRecord.java
@@ -17,17 +17,21 @@
package org.apache.nifi.serialization.record;
-import org.apache.nifi.serialization.record.util.DataTypeUtils;
-
import java.util.Date;
+import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import org.apache.nifi.serialization.record.type.ArrayDataType;
+import org.apache.nifi.serialization.record.type.MapDataType;
+import org.apache.nifi.serialization.record.util.DataTypeUtils;
+import org.apache.nifi.serialization.record.util.IllegalTypeConversionException;
+
public class MapRecord implements Record {
- private final RecordSchema schema;
+ private RecordSchema schema;
private final Map<String, Object> values;
- private final Optional<SerializedForm> serializedForm;
+ private Optional<SerializedForm> serializedForm;
public MapRecord(final RecordSchema schema, final Map<String, Object> values) {
this.schema = Objects.requireNonNull(schema);
@@ -192,7 +196,7 @@ public class MapRecord implements Record {
@Override
public Date getAsDate(final String fieldName, final String format) {
- return DataTypeUtils.toDate(getValue(fieldName), format == null ? null : DataTypeUtils.getDateFormat(format), fieldName);
+ return DataTypeUtils.toDate(getValue(fieldName), () -> DataTypeUtils.getDateFormat(format), fieldName);
}
@Override
@@ -223,11 +227,104 @@ public class MapRecord implements Record {
@Override
public String toString() {
- return "MapRecord[values=" + values + "]";
+ return "MapRecord[" + values + "]";
}
@Override
public Optional<SerializedForm> getSerializedForm() {
return serializedForm;
}
+
+ @Override
+ public void setValue(final String fieldName, final Object value) {
+ final Optional<RecordField> field = getSchema().getField(fieldName);
+ if (!field.isPresent()) {
+ return;
+ }
+
+ final RecordField recordField = field.get();
+ final Object coerced = DataTypeUtils.convertType(value, recordField.getDataType(), fieldName);
+ final Object previousValue = values.put(recordField.getFieldName(), coerced);
+ if (!Objects.equals(coerced, previousValue)) {
+ serializedForm = Optional.empty();
+ }
+ }
+
+ @Override
+ public void setArrayValue(final String fieldName, final int arrayIndex, final Object value) {
+ final Optional<RecordField> field = getSchema().getField(fieldName);
+ if (!field.isPresent()) {
+ return;
+ }
+
+ final RecordField recordField = field.get();
+ final DataType dataType = recordField.getDataType();
+ if (dataType.getFieldType() != RecordFieldType.ARRAY) {
+ throw new IllegalTypeConversionException("Cannot set the value of an array index on Record because the field '" + fieldName
+ + "' is of type '" + dataType + "' and cannot be coerced into an ARRAY type");
+ }
+
+ final Object arrayObject = values.get(recordField.getFieldName());
+ if (arrayObject == null) {
+ return;
+ }
+ if (!(arrayObject instanceof Object[])) {
+ return;
+ }
+
+ final Object[] array = (Object[]) arrayObject;
+ if (arrayIndex >= array.length) {
+ return;
+ }
+
+ final ArrayDataType arrayDataType = (ArrayDataType) dataType;
+ final DataType elementType = arrayDataType.getElementType();
+ final Object coerced = DataTypeUtils.convertType(value, elementType, fieldName);
+
+ final boolean update = !Objects.equals(coerced, array[arrayIndex]);
+ if (update) {
+ array[arrayIndex] = coerced;
+ serializedForm = Optional.empty();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void setMapValue(final String fieldName, final String mapKey, final Object value) {
+ final Optional<RecordField> field = getSchema().getField(fieldName);
+ if (!field.isPresent()) {
+ return;
+ }
+
+ final RecordField recordField = field.get();
+ final DataType dataType = recordField.getDataType();
+ if (dataType.getFieldType() != RecordFieldType.MAP) {
+ throw new IllegalTypeConversionException("Cannot set the value of map entry on Record because the field '" + fieldName
+ + "' is of type '" + dataType + "' and cannot be coerced into an MAP type");
+ }
+
+ Object mapObject = values.get(recordField.getFieldName());
+ if (mapObject == null) {
+ mapObject = new HashMap<String, Object>();
+ }
+ if (!(mapObject instanceof Map)) {
+ return;
+ }
+
+ final Map<String, Object> map = (Map<String, Object>) mapObject;
+
+ final MapDataType mapDataType = (MapDataType) dataType;
+ final DataType valueDataType = mapDataType.getValueType();
+ final Object coerced = DataTypeUtils.convertType(value, valueDataType, fieldName);
+
+ final Object replaced = map.put(mapKey, coerced);
+ if (replaced == null || !replaced.equals(coerced)) {
+ serializedForm = Optional.empty();
+ }
+ }
+
+ @Override
+ public void incorporateSchema(RecordSchema other) {
+ this.schema = DataTypeUtils.merge(this.schema, other);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/Record.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/Record.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/Record.java
index 31aaab7..822352d 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/Record.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/Record.java
@@ -20,11 +20,30 @@ package org.apache.nifi.serialization.record;
import java.util.Date;
import java.util.Optional;
+import org.apache.nifi.serialization.record.util.IllegalTypeConversionException;
+
public interface Record {
RecordSchema getSchema();
/**
+ * Updates the Record's schema to to incorporate all of the fields in the given schema. If both schemas have a
+ * field with the same name but a different type, then the existing schema will be updated to have a
+ * {@link RecordFieldType#CHOICE} field with both types as choices. If two fields have the same name but different
+ * default values, then the default value that is already in place will remain the default value, unless the current
+ * default value is <code>null</code>. Note that all values for this Record will still be valid according
+ * to this Record's Schema after this operation completes, as no type will be changed except to become more
+ * lenient. However, if incorporating the other schema does modify this schema, then the schema text
+ * returned by {@link #getSchemaText()}, the schema format returned by {@link #getSchemaFormat()}, and
+ * the SchemaIdentifier returned by {@link #getIdentifier()} for this record's schema may all become Empty.
+ *
+ * @param other the other schema to incorporate into this Record's schema
+ *
+ * @throws UnsupportedOperationException if this record does not support incorporating other schemas
+ */
+ void incorporateSchema(RecordSchema other);
+
+ /**
* <p>
* Returns a view of the the values of the fields in this Record.
* </p>
@@ -64,4 +83,55 @@ public interface Record {
Object[] getAsArray(String fieldName);
Optional<SerializedForm> getSerializedForm();
+
+ /**
+ * Updates the value of the field with the given name to the given value. If the field specified
+ * is not present in this Record's schema, this method will do nothing. If this method does change
+ * any value in the Record, any {@link SerializedForm} that was provided will be removed (i.e., any
+ * subsequent call to {@link #getSerializedForm()} will return an empty Optional).
+ *
+ * @param fieldName the name of the field to update
+ * @param value the new value to set
+ *
+ * @throws IllegalTypeConversionException if the value is not of the correct type, as defined
+ * by the schema, and cannot be coerced into the correct type.
+ */
+ void setValue(String fieldName, Object value);
+
+ /**
+ * Updates the value of a the specified index of a field. If the field specified
+ * is not present in this Record's schema, this method will do nothing. If the field specified
+ * is not an Array, an IllegalArgumentException will be thrown. If the field specified is an array
+ * but the array has fewer elements than the specified index, this method will do nothing. If this method does change
+ * any value in the Record, any {@link SerializedForm} that was provided will be removed (i.e., any
+ * subsequent call to {@link #getSerializedForm()} will return an empty Optional).
+ *
+ * @param fieldName the name of the field to update
+ * @param arrayIndex the 0-based index into the array that should be updated. If this value is larger than the
+ * number of elements in the array, or if the array is null, this method will do nothing.
+ * @param value the new value to set
+ *
+ * @throws IllegalTypeConversionException if the value is not of the correct type, as defined
+ * by the schema, and cannot be coerced into the correct type; or if the field with the given
+ * name is not an Array
+ * @throws IllegalArgumentException if the arrayIndex is less than 0.
+ */
+ void setArrayValue(String fieldName, int arrayIndex, Object value);
+
+ /**
+ * Updates the value of a the specified key in a Map field. If the field specified
+ * is not present in this Record's schema, this method will do nothing. If the field specified
+ * is not a Map field, an IllegalArgumentException will be thrown. If this method does change
+ * any value in the Record, any {@link SerializedForm} that was provided will be removed (i.e., any
+ * subsequent call to {@link #getSerializedForm()} will return an empty Optional).
+ *
+ * @param fieldName the name of the field to update
+ * @param mapKey the key in the map of the entry to update
+ * @param value the new value to set
+ *
+ * @throws IllegalTypeConversionException if the value is not of the correct type, as defined
+ * by the schema, and cannot be coerced into the correct type; or if the field with the given
+ * name is not a Map
+ */
+ void setMapValue(String fieldName, String mapKey, Object value);
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/RecordField.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/RecordField.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/RecordField.java
index dc68b01..c7cd290 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/RecordField.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/RecordField.java
@@ -97,5 +97,8 @@ public class RecordField {
return dataType.equals(other.getDataType()) && fieldName.equals(other.getFieldName()) && aliases.equals(other.getAliases()) && Objects.equals(defaultValue, other.defaultValue);
}
-
+ @Override
+ public String toString() {
+ return "RecordField[name=" + fieldName + ", dataType=" + dataType + (aliases.isEmpty() ? "" : ", aliases=" + aliases) + "]";
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
----------------------------------------------------------------------
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
index 9f1e463..1396ce1 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
@@ -17,14 +17,6 @@
package org.apache.nifi.serialization.record.util;
-import org.apache.nifi.serialization.record.type.ChoiceDataType;
-import org.apache.nifi.serialization.record.DataType;
-import org.apache.nifi.serialization.record.MapRecord;
-import org.apache.nifi.serialization.record.Record;
-import org.apache.nifi.serialization.record.type.RecordDataType;
-import org.apache.nifi.serialization.record.RecordFieldType;
-import org.apache.nifi.serialization.record.RecordSchema;
-
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Time;
@@ -32,37 +24,59 @@ import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.TimeZone;
import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import org.apache.nifi.serialization.SimpleRecordSchema;
+import org.apache.nifi.serialization.record.DataType;
+import org.apache.nifi.serialization.record.MapRecord;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordField;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.type.ChoiceDataType;
+import org.apache.nifi.serialization.record.type.RecordDataType;
public class DataTypeUtils {
private static final TimeZone gmt = TimeZone.getTimeZone("gmt");
public static Object convertType(final Object value, final DataType dataType, final String fieldName) {
- return convertType(value, dataType, getDateFormat(RecordFieldType.DATE.getDefaultFormat()), getDateFormat(RecordFieldType.TIME.getDefaultFormat()),
- getDateFormat(RecordFieldType.TIMESTAMP.getDefaultFormat()), fieldName);
+ return convertType(value, dataType, () -> getDateFormat(RecordFieldType.DATE.getDefaultFormat()), () -> getDateFormat(RecordFieldType.TIME.getDefaultFormat()),
+ () -> getDateFormat(RecordFieldType.TIMESTAMP.getDefaultFormat()), fieldName);
}
- public static DateFormat getDateFormat(final RecordFieldType fieldType, final DateFormat dateFormat, final DateFormat timeFormat, final DateFormat timestampFormat) {
+ public static DateFormat getDateFormat(final RecordFieldType fieldType, final Supplier<DateFormat> dateFormat,
+ final Supplier<DateFormat> timeFormat, final Supplier<DateFormat> timestampFormat) {
switch (fieldType) {
case DATE:
- return dateFormat;
+ return dateFormat.get();
case TIME:
- return timeFormat;
+ return timeFormat.get();
case TIMESTAMP:
- return timestampFormat;
+ return timestampFormat.get();
}
return null;
}
- public static Object convertType(final Object value, final DataType dataType, final DateFormat dateFormat, final DateFormat timeFormat,
- final DateFormat timestampFormat, final String fieldName) {
+ public static Object convertType(final Object value, final DataType dataType, final Supplier<DateFormat> dateFormat, final Supplier<DateFormat> timeFormat,
+ final Supplier<DateFormat> timestampFormat, final String fieldName) {
+
+ if (value == null) {
+ return null;
+ }
+
switch (dataType.getFieldType()) {
case BIGINT:
return toBigInt(value, fieldName);
@@ -85,7 +99,7 @@ public class DataTypeUtils {
case SHORT:
return toShort(value, fieldName);
case STRING:
- return toString(value, getDateFormat(dataType.getFieldType(), dateFormat, timeFormat, timestampFormat));
+ return toString(value, () -> getDateFormat(dataType.getFieldType(), dateFormat, timeFormat, timestampFormat));
case TIME:
return toTime(value, timeFormat, fieldName);
case TIMESTAMP:
@@ -99,10 +113,6 @@ public class DataTypeUtils {
final RecordSchema childSchema = recordType.getChildSchema();
return toRecord(value, childSchema, fieldName);
case CHOICE: {
- if (value == null) {
- return null;
- }
-
final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
final DataType chosenDataType = chooseDataType(value, choiceDataType);
if (chosenDataType == null) {
@@ -289,7 +299,7 @@ public class DataTypeUtils {
}
- public static String toString(final Object value, final DateFormat format) {
+ public static String toString(final Object value, final Supplier<DateFormat> format) {
if (value == null) {
return null;
}
@@ -302,22 +312,22 @@ public class DataTypeUtils {
return String.valueOf(((java.util.Date) value).getTime());
}
- if (value instanceof java.sql.Date) {
- return format.format((java.util.Date) value);
- }
- if (value instanceof java.sql.Time) {
- return format.format((java.util.Date) value);
- }
- if (value instanceof java.sql.Timestamp) {
- return format.format((java.util.Date) value);
- }
if (value instanceof java.util.Date) {
- return format.format((java.util.Date) value);
+ return formatDate((java.util.Date) value, format);
}
return value.toString();
}
+ private static String formatDate(final java.util.Date date, final Supplier<DateFormat> formatSupplier) {
+ final DateFormat dateFormat = formatSupplier.get();
+ if (dateFormat == null) {
+ return String.valueOf((date).getTime());
+ }
+
+ return dateFormat.format(date);
+ }
+
public static String toString(final Object value, final String format) {
if (value == null) {
return null;
@@ -355,7 +365,7 @@ public class DataTypeUtils {
return value != null;
}
- public static java.sql.Date toDate(final Object value, final DateFormat format, final String fieldName) {
+ public static java.sql.Date toDate(final Object value, final Supplier<DateFormat> format, final String fieldName) {
if (value == null) {
return null;
}
@@ -380,7 +390,11 @@ public class DataTypeUtils {
return new Date(Long.parseLong(string));
}
- final java.util.Date utilDate = format.parse(string);
+ final DateFormat dateFormat = format.get();
+ if (dateFormat == null) {
+ return new Date(Long.parseLong(string));
+ }
+ final java.util.Date utilDate = dateFormat.parse(string);
return new Date(utilDate.getTime());
} catch (final ParseException | NumberFormatException e) {
throw new IllegalTypeConversionException("Could not convert value [" + value
@@ -430,7 +444,7 @@ public class DataTypeUtils {
return true;
}
- public static Time toTime(final Object value, final DateFormat format, final String fieldName) {
+ public static Time toTime(final Object value, final Supplier<DateFormat> format, final String fieldName) {
if (value == null) {
return null;
}
@@ -455,7 +469,11 @@ public class DataTypeUtils {
return new Time(Long.parseLong(string));
}
- final java.util.Date utilDate = format.parse(string);
+ final DateFormat dateFormat = format.get();
+ if (dateFormat == null) {
+ return new Time(Long.parseLong(string));
+ }
+ final java.util.Date utilDate = dateFormat.parse(string);
return new Time(utilDate.getTime());
} catch (final ParseException e) {
throw new IllegalTypeConversionException("Could not convert value [" + value
@@ -467,6 +485,9 @@ public class DataTypeUtils {
}
public static DateFormat getDateFormat(final String format) {
+ if (format == null) {
+ return null;
+ }
final DateFormat df = new SimpleDateFormat(format);
df.setTimeZone(gmt);
return df;
@@ -476,7 +497,7 @@ public class DataTypeUtils {
return isDateTypeCompatible(value, format);
}
- public static Timestamp toTimestamp(final Object value, final DateFormat format, final String fieldName) {
+ public static Timestamp toTimestamp(final Object value, final Supplier<DateFormat> format, final String fieldName) {
if (value == null) {
return null;
}
@@ -501,7 +522,11 @@ public class DataTypeUtils {
return new Timestamp(Long.parseLong(string));
}
- final java.util.Date utilDate = format.parse(string);
+ final DateFormat dateFormat = format.get();
+ if (dateFormat == null) {
+ return new Timestamp(Long.parseLong(string));
+ }
+ final java.util.Date utilDate = dateFormat.parse(string);
return new Timestamp(utilDate.getTime());
} catch (final ParseException e) {
throw new IllegalTypeConversionException("Could not convert value [" + value
@@ -765,4 +790,107 @@ public class DataTypeUtils {
return value != null && (value instanceof Character || (value instanceof CharSequence && ((CharSequence) value).length() > 0));
}
+ public static RecordSchema merge(final RecordSchema thisSchema, final RecordSchema otherSchema) {
+ if (thisSchema == null) {
+ return otherSchema;
+ }
+ if (otherSchema == null) {
+ return thisSchema;
+ }
+
+ final List<RecordField> otherFields = otherSchema.getFields();
+ if (otherFields.isEmpty()) {
+ return thisSchema;
+ }
+
+ final List<RecordField> thisFields = thisSchema.getFields();
+ if (thisFields.isEmpty()) {
+ return otherSchema;
+ }
+
+ final Map<String, Integer> fieldIndices = new HashMap<>();
+ final List<RecordField> fields = new ArrayList<>();
+ for (int i = 0; i < thisFields.size(); i++) {
+ final RecordField field = thisFields.get(i);
+
+ final Integer index = Integer.valueOf(i);
+
+ fieldIndices.put(field.getFieldName(), index);
+ for (final String alias : field.getAliases()) {
+ fieldIndices.put(alias, index);
+ }
+
+ fields.add(field);
+ }
+
+ for (final RecordField otherField : otherFields) {
+ Integer fieldIndex = fieldIndices.get(otherField.getFieldName());
+
+ // Find the field in 'thisSchema' that corresponds to 'otherField',
+ // if one exists.
+ if (fieldIndex == null) {
+ for (final String alias : otherField.getAliases()) {
+ fieldIndex = fieldIndices.get(alias);
+ if (fieldIndex != null) {
+ break;
+ }
+ }
+ }
+
+ // If there is no field with the same name then just add 'otherField'.
+ if (fieldIndex == null) {
+ fields.add(otherField);
+ continue;
+ }
+
+ // Merge the two fields, if necessary
+ final RecordField thisField = fields.get(fieldIndex);
+ if (isMergeRequired(thisField, otherField)) {
+ final RecordField mergedField = merge(thisField, otherField);
+ fields.set(fieldIndex, mergedField);
+ }
+ }
+
+ return new SimpleRecordSchema(fields);
+ }
+
+
+ private static boolean isMergeRequired(final RecordField thisField, final RecordField otherField) {
+ if (!thisField.getDataType().equals(otherField.getDataType())) {
+ return true;
+ }
+
+ if (!thisField.getAliases().equals(otherField.getAliases())) {
+ return true;
+ }
+
+ if (!Objects.equals(thisField.getDefaultValue(), otherField.getDefaultValue())) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public static RecordField merge(final RecordField thisField, final RecordField otherField) {
+ final String fieldName = thisField.getFieldName();
+ final Set<String> aliases = new HashSet<>();
+ aliases.addAll(thisField.getAliases());
+ aliases.addAll(otherField.getAliases());
+
+ final Object defaultValue;
+ if (thisField.getDefaultValue() == null && otherField.getDefaultValue() != null) {
+ defaultValue = otherField.getDefaultValue();
+ } else {
+ defaultValue = thisField.getDefaultValue();
+ }
+
+ final DataType dataType;
+ if (thisField.getDataType().equals(otherField.getDataType())) {
+ dataType = thisField.getDataType();
+ } else {
+ dataType = RecordFieldType.CHOICE.getChoiceDataType(thisField.getDataType(), otherField.getDataType());
+ }
+
+ return new RecordField(fieldName, dataType, defaultValue, aliases);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-commons/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml
index f030b92..30cd8ed 100644
--- a/nifi-commons/pom.xml
+++ b/nifi-commons/pom.xml
@@ -12,8 +12,7 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+--><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
@@ -37,6 +36,7 @@
<module>nifi-site-to-site-client</module>
<module>nifi-hl7-query-language</module>
<module>nifi-schema-utils</module>
- <module>nifi-record</module>
+ <module>nifi-record</module>
+ <module>nifi-record-path</module>
</modules>
-</project>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-docs/src/main/asciidoc/record-path-guide.adoc
----------------------------------------------------------------------
diff --git a/nifi-docs/src/main/asciidoc/record-path-guide.adoc b/nifi-docs/src/main/asciidoc/record-path-guide.adoc
new file mode 100644
index 0000000..d38a5d3
--- /dev/null
+++ b/nifi-docs/src/main/asciidoc/record-path-guide.adoc
@@ -0,0 +1,293 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+Apache NiFi RecordPath Guide
+============================
+Apache NiFi Team <de...@nifi.apache.org>
+:homepage: http://nifi.apache.org
+
+[[overview]]
+Overview
+--------
+Apache NiFi offers a very robust set of Processors that are capable of ingesting, processing,
+routing, transforming, and delivering data of any format. This is possible because the NiFi
+framework itself is data-agnostic. It doesn't care whether your data is a 100-byte JSON message
+or a 100-gigabyte video. This is an incredibly powerful feature. However, there are many patterns
+that have quickly developed to handle data of differing types.
+
+One class of data that is often processed by NiFi is record-oriented data. When we say record-oriented
+data, we are often (but not always) talking about structured data such as JSON, CSV, and Avro. There
+are many other types of data that can also be represented as "records" or "messages," though. As a result,
+a set of Controller Services have been developed for parsing these different data formats and representing
+the data in a consistent way by using the RecordReader API. This allows data that has been written in any
+data format to be treated the same, so long as there is a RecordReader that is capable of producing a Record
+object that represents the data.
+
+When we talk about a Record, this is an abstraction that allows us to treat data in the same
+way, regardless of the format that it is in. A Record is made up of one or more Fields. Each Field has a name
+and a Type associated with it. The Fields of a Record are described using a Record Schema. The Schema indicates
+which fields make up a specific type of Record. The Type of a Field will be one of the following:
+
+- String
+- Boolean
+- Byte
+- Character
+- Short
+- Integer
+- Long
+- BigInt
+- Float
+- Double
+- Date - Represents a Date without a Time component
+- Time - Represents a Time of Day without a Date component
+- Timestamp - Represents a Date and Time
+- Embedded Record - Hierarchical data, such as JSON, can be represented by allowing a field to be of Type Record itself.
+- Choice - A field may be any one of several types.
+- Array - All elements of an array have the same type.
+- Map - All Map Keys are of type String. The Values are of the same type.
+
+
+Once a stream of data has been converted into Records, the RecordWriter API
+allows us to then serialize those Records back into streams of bytes so that they can be passed onto other
+systems.
+
+Of course, there's not much point in reading and writing this data if we aren't going to do something with
+the data in between. There are several processors that have already been developed for NiFi that provide some
+very powerful capabilities for routing, querying, and transforming Record-oriented data. Often times, in order
+to perform the desired function, a processor will need input from the user in order to determine which fields
+in a Record or which values in a Record should be operated on.
+
+Enter the NiFi RecordPath language. RecordPath is intended to be a simple, easy-to-use Domain-Specific Language
+(DSL) to specify which fields in a Record we care about or want to access when configuring a processor.
+
+
+
+[[structure]]
+Structure of a RecordPath
+-------------------------
+
+A Record in NiFi is made up of (potentially) many fields, and each of these fields could actually be itself a Record. This means that
+a Record can be thought of as having a hierarchical, or nested, structure. We talk about an "inner Record" as being the child of the
+"outer Record." The child of an inner Record, then, is a descendant of the outer-most Record. Similarly, we can refer to an outer Record
+as being an ancestor of an inner Record.
+
+
+[[child]]
+== Child Operator
+The RecordPath language is structured in such a way that we are able to easily reference fields of the outer-most Record, or fields of a
+child Record, or descendant Record. To accomplish this, we separate the names of the children with a slash character (`/`), which we
+refer to as the `child` operator. For example,
+let's assume that we have a Record that is made up of two fields: `name` and `details`. Also, assume that `details` is a field that is
+itself a Record and has two Fields: `identifier` and `address`. Further, let's consider that `address` is itself a Record that contains
+5 fields: `number`, `street`, `city`, `state`, and `zip`. An example, written here in JSON for illustrative purposes may look like this:
+
+----
+{
+ "name": "John Doe",
+ "details": {
+ "identifier": 100,
+ "address": {
+ "number": "123",
+ "street": "5th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "10020"
+ }
+ }
+}
+----
+
+We can reference the `zip` field by using the RecordPath: `/details/address/zip`. This tells us that we want to use the `details` field of
+the "root" Record. We then want to reference the `address` field of the child Record and the `zip` field of that Record.
+
+
+[[descendant]]
+== Descendant Operator
+In addition to providing an explicit path to reach the `zip` field, it may sometimes be useful to reference the `zip` field without knowing
+the full path. In such a case, we can use the `descendant` operator (`//`) instead of the `child` operator (`/`). To reach the same `zip`
+field as above, we can accomplish this by simply using the path `//zip`.
+
+There is a very important distinction, though, between the `child` operator and the `descendant` operator: the `descendant` operator may match
+many fields, whereas the `child` operator will match at most one field. To help understand this, consider the following Record:
+
+----
+{
+ "name": "John Doe",
+ "workAddress": {
+ "number": "123",
+ "street": "5th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "10020"
+ },
+ "homeAddress": {
+ "number": "456",
+ "street": "116th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "11697"
+ }
+}
+----
+
+Now, if we use the RecordPath `/workAddress/zip`, we will be referencing the `zip` field that has a value of "10020." The RecordPath `/homeAddress/zip` will
+reference the `zip` field that has a value of "11697." However, the RecordPath `//zip` will reference both of these fields.
+
+
+
+[[filters]]
+== Filters
+
+With the above examples and explanation, we are able to easily reference a specific field within a Record. However, in real scenarios, the data is rarely as
+simple as in the examples above. Often times, we need to filter out or refine which fields we are referencing. Examples of when we might want to do this are
+when we reference an Array field and want to only reference some of the elements in the array; when we reference a Map field and want to reference one or a few
+specific entries in the Map; or when we want to reference a Record only if it adheres to some criteria. We can accomplish this by providing our criteria to the
+RecordPath within square brackets (using the `[` and `]` characters). We will go over each of these cases below.
+
+
+[[arrays]]
+=== Arrays
+
+When we reference an Array field, the value of the field may be an array that contains several elements, but we may want only a few of those elements. For example,
+we may want to reference only the first element; only the last element; or perhaps the first, second, third, and last elements. We can reference a specific element simply by
+using the index of the element within square brackets (the index is 0-based). So let us consider a modified version of the Record above:
+
+----
+{
+ "name": "John Doe",
+ "addresses": [
+ "work": {
+ "number": "123",
+ "street": "5th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "10020"
+ },
+ "home": {
+ "number": "456",
+ "street": "116th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "11697"
+ }
+ ]
+}
+----
+
+We can now reference the first element in the `addresses` array by using the RecordPath `/addresses[0]`. We can access the second element using the RecordPath `/addresses[1]`.
+There may be times, though, that we don't know how many elements will exist in the array. So we can use negative indices to count backward from the end of the array. For example,
+we can access the last element as `/addresses[-1]` or the next-to-last element as `/addresses[-2]`. If we want to reference several elements, we can use a comma-separated list of
+elements, such as `/addresses[0, 1, 2, 3]`. Or, to access elements 0 through 8, we can use the `range` operator (`..`), as in `/addresses[0..8]`. We can also mix these, and reference
+all elements by using the syntax `/addresses[0..-1]` or even `/addresses[0, 1, 4, 6..-1]`. Of course, not all of the indices referenced here will match on the Record above, because
+the `addresses` array has only 2 elements. The indices that do not match will simply be skipped.
+
+
+[[maps]]
+=== Maps
+
+Similar to an Array field, a Map field may actually consist of several different values. RecordPath gives us the ability to select a set of values based on their keys.
+We do this by using a quoted String within square brackets. As an example, let's re-visit our original Record from above:
+
+----
+{
+ "name": "John Doe",
+ "details": {
+ "identifier": 100,
+ "address": {
+ "number": "123",
+ "street": "5th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "10020"
+ }
+ }
+}
+----
+
+Now, though, let's consider that the Schema that is associated with the Record indicates that the `address` field is not a Record but rather a `Map` field.
+In this case, if we attempt to reference the `zip` using the RecordPath `/details/address/zip` the RecordPath will not match because the `address` field is not a Record
+and therefore does not have any Child Record named `zip`. Instead, it is a Map field with keys and values of type String.
+Unfortunately, when looking at JSON this may seem a bit confusing because JSON does not truly have a Type system. When we convert the JSON into a Record object in order
+to operate on the data, though, this distinction can be important.
+
+In the case laid out above, we can still access the `zip` field using RecordPath. We must now use the a slightly different syntax, though: `/details/address['zip']`. This
+is telling the RecordPath that we want to access the `details` field at the highest level. We then want to access its `address` field. Since the `address` field is a `Map`
+field we can use square brackets to indicate that we want to specify a Map Key, and we can then specify the key in quotes.
+
+Further, we can select more than one Map Key, using a comma-separated list: `/details/address['city', 'state', 'zip']`. We can also select all of the fields, if we want,
+using the Wildcard operator (`*`): `/details/address[*]`. Map fields do not contain any sort of ordering, so it is not possible to reference the keys by numeric indices.
+
+
+[[predicates]]
+=== Predicates
+
+Thus far, we have discussed two different types of filters. Each of them allows us to select one or more elements out from a field that allows for many values.
+Often times, though, we need to apply a filter that allows us to restrict which Record fields are selected. For example, what if we want to select the `zip` field but
+only for an `address` field where the state is not New York? The above examples do not give us any way to do this.
+
+RecordPath provides the user the ability to specify a Predicate. A Predicate is simply a filter that can be applied to a field in order to determine whether or not the
+field should be included in the results. Like other filters, a Predicate is specified within square brackets. The syntax of the Predicate is
+`<Relative RecordPath> <Operator> <Expression>`. The `Relative RecordPath` works just like any other RecordPath but must start with a `.` (to reference the current field)
+or a `..` (to reference the current field's parent) instead of a slash and references
+fields relative to the field that the Predicate applies to. The `Operator` must be one of:
+
+- Equals (`=`)
+- Not Equal (`!=`)
+- Greater Than (`>`)
+- Greater Than or Equal To (`>=`)
+- Less Than (`<`)
+- Less Than or Equal To (`<=`)
+
+The `Expression` can be a literal value such as `50` or `Hello` or can be another RecordPath.
+
+To illustrate this, let's take the following Record as an example:
+
+----
+{
+ "name": "John Doe",
+ "workAddress": {
+ "number": "123",
+ "street": "5th Avenue",
+ "city": "New York",
+ "state": "NY",
+ "zip": "10020"
+ },
+ "homeAddress": {
+ "number": "456",
+ "street": "Grand St",
+ "city": "Jersey City",
+ "state": "NJ",
+ "zip": "07304"
+ },
+ "details": {
+ "position": "Dataflow Engineer",
+ "preferredState": "NY"
+ }
+}
+----
+
+Now we can use a Predicate to choose only the fields where the state is not New York. For example, we can use `/*[./state != 'NY']`. This will select any Record field
+that has a `state` field if the state does not have a value of "NY". Note that the `details` Record will not be returned because it does not have a field named `state`.
+So in this example, the RecordPath will select only the `homeAddress` field. Once we have selected that field, we can continue on with our RecordPath. As we stated
+above, we can select the `zip` field: `/*[./state != 'NY']/zip`. This RecordPath will result in selecting the `zip` field only from the `homeAddress` field.
+
+We can also compare the value in one field with the value in another field. For example, we can select the address that is in the person's preferred state by using
+the RecordPath `/*[./state = /details/preferredState]`. In this example, this RecordPath will retrieve the `workAddress` field because its `state` field matches the
+value of the `preferredState` field.
+
+Additionally, we can write a RecordPath that references the "city" field of any record whose state is "NJ" by using the parent operator (`..`): `/*/city[../state = 'NJ']`.
+
+
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractFetchHDFSRecord.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractFetchHDFSRecord.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractFetchHDFSRecord.java
index 9f4f5ac..8883965 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractFetchHDFSRecord.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-hadoop-record-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractFetchHDFSRecord.java
@@ -187,7 +187,8 @@ public abstract class AbstractFetchHDFSRecord extends AbstractHadoopProcessor {
final AtomicReference<WriteResult> writeResult = new AtomicReference<>();
final RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
- final RecordSetWriter recordSetWriter = recordSetWriterFactory.createWriter(getLogger(), originalFlowFile, new NullInputStream(0));
+ final RecordSchema schema = recordSetWriterFactory.getSchema(originalFlowFile, new NullInputStream(0));
+ final RecordSetWriter recordSetWriter = recordSetWriterFactory.createWriter(getLogger(), schema);
final StopWatch stopWatch = new StopWatch(true);
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/pom.xml
index 3340e81..8e568d9 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/pom.xml
@@ -39,6 +39,10 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-schema-registry-service-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-record</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
index 99c72e4..b4253ee 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
@@ -25,8 +25,10 @@ import java.util.Collections;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.RecordSetWriter;
import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.WriteResult;
public class MockRecordWriter extends AbstractControllerService implements RecordSetWriterFactory {
@@ -49,7 +51,12 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
}
@Override
- public RecordSetWriter createWriter(final ComponentLog logger, final FlowFile flowFile, final InputStream in) {
+ public RecordSchema getSchema(final FlowFile flowFile, final InputStream content) throws SchemaNotFoundException, IOException {
+ return new SimpleRecordSchema(Collections.emptyList());
+ }
+
+ @Override
+ public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema schema) {
return new RecordSetWriter() {
@Override
public WriteResult write(final RecordSet rs, final OutputStream out) throws IOException {
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1901d5f/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockSchemaRegistry.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockSchemaRegistry.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockSchemaRegistry.java
new file mode 100644
index 0000000..36bbe58
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockSchemaRegistry.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.serialization.record;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.schema.access.SchemaField;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.schemaregistry.services.SchemaRegistry;
+import org.apache.nifi.util.Tuple;
+
+public class MockSchemaRegistry extends AbstractControllerService implements SchemaRegistry {
+ private final ConcurrentMap<String, RecordSchema> schemaNameMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Tuple<Long, Integer>, RecordSchema> schemaIdVersionMap = new ConcurrentHashMap<>();
+
+ public void addSchema(final String name, final RecordSchema schema) {
+ schemaNameMap.put(name, schema);
+ }
+
+ @Override
+ public String retrieveSchemaText(final String schemaName) throws IOException, SchemaNotFoundException {
+ final RecordSchema schema = schemaNameMap.get(schemaName);
+ if (schema == null) {
+ return null;
+ }
+
+ final Optional<String> text = schema.getSchemaText();
+ return text.orElse(null);
+ }
+
+ @Override
+ public String retrieveSchemaText(final long schemaId, final int version) throws IOException, SchemaNotFoundException {
+ final Tuple<Long, Integer> tuple = new Tuple<>(schemaId, version);
+ final RecordSchema schema = schemaIdVersionMap.get(tuple);
+ if (schema == null) {
+ return null;
+ }
+
+ final Optional<String> text = schema.getSchemaText();
+ return text.orElse(null);
+ }
+
+ @Override
+ public RecordSchema retrieveSchema(final String schemaName) throws IOException, SchemaNotFoundException {
+ return schemaNameMap.get(schemaName);
+ }
+
+ @Override
+ public RecordSchema retrieveSchema(final long schemaId, final int version) throws IOException, SchemaNotFoundException {
+ final Tuple<Long, Integer> tuple = new Tuple<>(schemaId, version);
+ final RecordSchema schema = schemaIdVersionMap.get(tuple);
+ return schema;
+ }
+
+ @Override
+ public Set<SchemaField> getSuppliedSchemaFields() {
+ return EnumSet.allOf(SchemaField.class);
+ }
+}