You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/05/16 17:17:27 UTC
nifi git commit: NIFI-3910: Fixed issue where CSVRecordReader throws
an Exception if a field is missing instead of using a null value
Repository: nifi
Updated Branches:
refs/heads/master 824712bff -> 3f4b276b7
NIFI-3910: Fixed issue where CSVRecordReader throws an Exception if a field is missing instead of using a null value
This closes #1807.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3f4b276b
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3f4b276b
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3f4b276b
Branch: refs/heads/master
Commit: 3f4b276b713464ad0ccf13b5164e08d118e2cc25
Parents: 824712b
Author: Mark Payne <ma...@hotmail.com>
Authored: Tue May 16 11:09:14 2017 -0400
Committer: Mark Payne <ma...@hotmail.com>
Committed: Tue May 16 13:16:46 2017 -0400
----------------------------------------------------------------------
.../org/apache/nifi/csv/CSVRecordReader.java | 12 ++++---
.../apache/nifi/csv/TestCSVRecordReader.java | 33 ++++++++++++++++++++
2 files changed, 40 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/3f4b276b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
index 3bcbce2..095f488 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java
@@ -68,17 +68,19 @@ public class CSVRecordReader implements RecordReader {
final Map<String, Object> rowValues = new HashMap<>(schema.getFieldCount());
for (final RecordField recordField : schema.getFields()) {
- String rawValue = csvRecord.get(recordField.getFieldName());
- if (rawValue == null) {
+ String rawValue = null;
+ final String fieldName = recordField.getFieldName();
+ if (csvRecord.isSet(fieldName)) {
+ rawValue = csvRecord.get(fieldName);
+ } else {
for (final String alias : recordField.getAliases()) {
- rawValue = csvRecord.get(alias);
- if (rawValue != null) {
+ if (csvRecord.isSet(alias)) {
+ rawValue = csvRecord.get(alias);
break;
}
}
}
- final String fieldName = recordField.getFieldName();
if (rawValue == null) {
rowValues.put(fieldName, null);
continue;
http://git-wip-us.apache.org/repos/asf/nifi/blob/3f4b276b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java
index cb790f1..a02e0b1 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java
@@ -18,6 +18,7 @@
package org.apache.nifi.csv;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import java.io.ByteArrayInputStream;
@@ -143,4 +144,36 @@ public class TestCSVRecordReader {
assertNull(reader.nextRecord());
}
}
+
+ @Test
+ public void testMissingField() throws IOException, MalformedRecordException {
+ final List<RecordField> fields = getDefaultFields();
+ fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f);
+
+ final RecordSchema schema = new SimpleRecordSchema(fields);
+
+ final String headerLine = "id, name, balance, address, city, state, zipCode, country";
+ final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111";
+ final String csvData = headerLine + "\n" + inputRecord;
+ final byte[] inputData = csvData.getBytes();
+
+ try (final InputStream baos = new ByteArrayInputStream(inputData)) {
+ final CSVRecordReader reader = new CSVRecordReader(baos, Mockito.mock(ComponentLog.class), schema, format,
+ RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat());
+
+ final Record record = reader.nextRecord();
+ assertNotNull(record);
+
+ assertEquals("1", record.getValue("id"));
+ assertEquals("John", record.getValue("name"));
+ assertEquals(40.8D, record.getValue("balance"));
+ assertEquals("123 My Street", record.getValue("address"));
+ assertEquals("My City", record.getValue("city"));
+ assertEquals("MS", record.getValue("state"));
+ assertEquals("11111", record.getValue("zipCode"));
+ assertNull(record.getValue("country"));
+
+ assertNull(reader.nextRecord());
+ }
+ }
}