You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2020/08/13 02:05:23 UTC
[nifi] branch main updated: NIFI-7734: Added Record Separator
property to CSVReader
This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new b7b2533 NIFI-7734: Added Record Separator property to CSVReader
b7b2533 is described below
commit b7b2533ffe556ef5bb1982e819f465ce1c74881b
Author: Matthew Burgess <ma...@apache.org>
AuthorDate: Wed Aug 12 17:01:15 2020 -0400
NIFI-7734: Added Record Separator property to CSVReader
---
.../src/main/java/org/apache/nifi/csv/CSVReader.java | 1 +
.../org/apache/nifi/csv/JacksonCSVRecordReader.java | 2 +-
.../apache/nifi/csv/TestJacksonCSVRecordReader.java | 19 +++++++++++++++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVReader.java
index b719711..8eeb422 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVReader.java
@@ -101,6 +101,7 @@ public class CSVReader extends SchemaRegistryService implements RecordReaderFact
properties.add(DateTimeUtils.TIMESTAMP_FORMAT);
properties.add(CSVUtils.CSV_FORMAT);
properties.add(CSVUtils.VALUE_SEPARATOR);
+ properties.add(CSVUtils.RECORD_SEPARATOR);
properties.add(CSVUtils.FIRST_LINE_IS_HEADER);
properties.add(CSVUtils.IGNORE_CSV_HEADER);
properties.add(CSVUtils.QUOTE_CHAR);
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/JacksonCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/JacksonCSVRecordReader.java
index 1af94b8..975a9b1 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/JacksonCSVRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/JacksonCSVRecordReader.java
@@ -60,7 +60,7 @@ public class JacksonCSVRecordReader extends AbstractCSVRecordReader {
CsvSchema.Builder csvSchemaBuilder = CsvSchema.builder()
.setColumnSeparator(csvFormat.getDelimiter())
- .setLineSeparator(csvFormat.getRecordSeparator())
+ .setLineSeparator((csvFormat.getRecordSeparator() == null) ? "\n" : csvFormat.getRecordSeparator())
// Can only use comments in Jackson CSV if the correct marker is set
.setAllowComments("#" .equals(CharUtils.toString(csvFormat.getCommentMarker())))
// The call to setUseHeader(false) in all code paths is due to the way Jackson does data binding/mapping. Missing or extra columns may not
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestJacksonCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestJacksonCSVRecordReader.java
index 935fe73..a9585ce 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestJacksonCSVRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestJacksonCSVRecordReader.java
@@ -49,6 +49,7 @@ import static org.junit.Assert.assertNull;
public class TestJacksonCSVRecordReader {
private final DataType doubleDataType = RecordFieldType.DOUBLE.getDataType();
private final CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().withQuote('"');
+ private final CSVFormat formatWithNullRecordSeparator = CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().withQuote('"').withRecordSeparator(null);
private List<RecordField> getDefaultFields() {
final List<RecordField> fields = new ArrayList<>();
@@ -391,4 +392,22 @@ public class TestJacksonCSVRecordReader {
assertNull(reader.nextRecord());
}
}
+
+ @Test
+ public void testNullRecordSeparator() throws IOException, MalformedRecordException {
+ final List<RecordField> fields = getDefaultFields();
+ fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f);
+
+ final RecordSchema schema = new SimpleRecordSchema(fields);
+
+ try (final InputStream fis = new FileInputStream(new File("src/test/resources/csv/single-bank-account.csv"));
+ final JacksonCSVRecordReader reader = createReader(fis, schema, formatWithNullRecordSeparator)) {
+
+ final Object[] record = reader.nextRecord().getValues();
+ final Object[] expectedValues = new Object[] {"1", "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA"};
+ Assert.assertArrayEquals(expectedValues, record);
+
+ assertNull(reader.nextRecord());
+ }
+ }
}