You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2020/01/07 19:02:22 UTC
[nifi] branch master updated: NIFI-6963: Fix ValidateRecord
handling of missing required array fields
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new f4ef456 NIFI-6963: Fix ValidateRecord handling of missing required array fields
f4ef456 is described below
commit f4ef45678eff4ca0612f03ff4de3e5d2e29e4e2a
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Fri Dec 20 12:49:21 2019 -0500
NIFI-6963: Fix ValidateRecord handling of missing required array fields
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #3948.
---
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 2 +-
.../nifi-standard-processors/pom.xml | 3 +
.../processors/standard/TestValidateRecord.java | 74 ++++++++++++++++++++++
.../missing-array-with-default.avsc | 22 +++++++
.../TestValidateRecord/missing-array.avsc | 22 +++++++
.../TestValidateRecord/missing-array.json | 5 ++
6 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index a0eea8b..caa2743 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -626,7 +626,7 @@ public class AvroTypeUtil {
recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable));
} else {
Object defaultValue = field.defaultVal();
- if (fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) {
+ if (defaultValue != null && fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) {
defaultValue = defaultValue instanceof List ? ((List<?>) defaultValue).toArray() : new Object[0];
}
recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable));
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 3da027a..72cdb65 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -570,6 +570,9 @@
<exclude>src/test/resources/TestForkRecord/schema/schema.avsc</exclude>
<exclude>src/test/resources/TestConvertRecord/schema/person.avsc</exclude>
<exclude>src/test/resources/TestConvertRecord/input/person.json</exclude>
+ <exclude>src/test/resources/TestValidateRecord/missing-array.json</exclude>
+ <exclude>src/test/resources/TestValidateRecord/missing-array.avsc</exclude>
+ <exclude>src/test/resources/TestValidateRecord/missing-array-with-default.avsc</exclude>
<exclude>src/test/resources/TestValidateRecord/nested-map-input.json</exclude>
<exclude>src/test/resources/TestValidateRecord/nested-map-schema.avsc</exclude>
<exclude>src/test/resources/TestValidateRecord/timestamp.avsc</exclude>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
index bbcc9a9..f53dd5c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
@@ -412,6 +412,80 @@ public class TestValidateRecord {
runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0);
}
+ @Test
+ public void testValidateMissingRequiredArray() throws InitializationException, IOException {
+ final String validateSchema = new String(Files.readAllBytes(Paths.get("src/test/resources/TestValidateRecord/missing-array.avsc")), StandardCharsets.UTF_8);
+
+ final JsonTreeReader jsonReader = new JsonTreeReader();
+ runner.addControllerService("reader", jsonReader);
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, "schema-text-property");
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, validateSchema);
+ runner.enableControllerService(jsonReader);
+
+ final JsonRecordSetWriter validWriter = new JsonRecordSetWriter();
+ runner.addControllerService("writer", validWriter);
+ runner.setProperty(validWriter, "Schema Write Strategy", "full-schema-attribute");
+ runner.enableControllerService(validWriter);
+
+ final MockRecordWriter invalidWriter = new MockRecordWriter("invalid", true);
+ runner.addControllerService("invalid-writer", invalidWriter);
+ runner.enableControllerService(invalidWriter);
+
+ runner.setProperty(ValidateRecord.RECORD_READER, "reader");
+ runner.setProperty(ValidateRecord.RECORD_WRITER, "writer");
+ runner.setProperty(ValidateRecord.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(ValidateRecord.SCHEMA_TEXT, validateSchema);
+ runner.setProperty(ValidateRecord.INVALID_RECORD_WRITER, "invalid-writer");
+ runner.setProperty(ValidateRecord.ALLOW_EXTRA_FIELDS, "true");
+
+ // The record is invalid due to not containing the required array from the schema
+ runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "false");
+ runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/missing-array.json"));
+ runner.run();
+
+ runner.assertTransferCount(ValidateRecord.REL_VALID, 0);
+ runner.assertTransferCount(ValidateRecord.REL_INVALID, 1);
+ runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0);
+ runner.clearTransferState();
+ }
+
+ @Test
+ public void testValidateMissingRequiredArrayWithDefault() throws InitializationException, IOException {
+ final String validateSchema = new String(Files.readAllBytes(Paths.get("src/test/resources/TestValidateRecord/missing-array-with-default.avsc")), StandardCharsets.UTF_8);
+
+ final JsonTreeReader jsonReader = new JsonTreeReader();
+ runner.addControllerService("reader", jsonReader);
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, "schema-text-property");
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, validateSchema);
+ runner.enableControllerService(jsonReader);
+
+ final JsonRecordSetWriter validWriter = new JsonRecordSetWriter();
+ runner.addControllerService("writer", validWriter);
+ runner.setProperty(validWriter, "Schema Write Strategy", "full-schema-attribute");
+ runner.enableControllerService(validWriter);
+
+ final MockRecordWriter invalidWriter = new MockRecordWriter("invalid", true);
+ runner.addControllerService("invalid-writer", invalidWriter);
+ runner.enableControllerService(invalidWriter);
+
+ runner.setProperty(ValidateRecord.RECORD_READER, "reader");
+ runner.setProperty(ValidateRecord.RECORD_WRITER, "writer");
+ runner.setProperty(ValidateRecord.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(ValidateRecord.SCHEMA_TEXT, validateSchema);
+ runner.setProperty(ValidateRecord.INVALID_RECORD_WRITER, "invalid-writer");
+ runner.setProperty(ValidateRecord.ALLOW_EXTRA_FIELDS, "true");
+
+ // The record is invalid due to not containing the required array from the schema
+ runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "false");
+ runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/missing-array.json"));
+ runner.run();
+
+ runner.assertTransferCount(ValidateRecord.REL_VALID, 1);
+ runner.assertTransferCount(ValidateRecord.REL_INVALID, 0);
+ runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0);
+ runner.clearTransferState();
+ }
+
@Test
public void testValidateJsonTimestamp() throws IOException, InitializationException {
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc
new file mode 100644
index 0000000..b01911c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc
@@ -0,0 +1,22 @@
+{
+ "name": "Test",
+ "type": "record",
+ "fields": [
+ {
+ "name": "Records",
+ "type": {
+ "type": "array",
+ "items": {
+ "name": "Records_record",
+ "type": "record",
+ "fields": [
+ {
+ "name": "id",
+ "type": "string"
+ }
+ ]
+ }
+ }, "default": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc
new file mode 100644
index 0000000..9175a4c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc
@@ -0,0 +1,22 @@
+{
+ "name": "Test",
+ "type": "record",
+ "fields": [
+ {
+ "name": "Records",
+ "type": {
+ "type": "array",
+ "items": {
+ "name": "Records_record",
+ "type": "record",
+ "fields": [
+ {
+ "name": "id",
+ "type": "string"
+ }
+ ]
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json
new file mode 100644
index 0000000..43b8c4c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json
@@ -0,0 +1,5 @@
+{
+ "NotRecords": [{
+ "id": "1"
+ }]
+}