You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2024/11/12 21:27:42 UTC
(nifi) branch support/nifi-1.x updated: NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 41616773db NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)
41616773db is described below
commit 41616773dbb1e7dc9173170ecf3e46d6c88e5ea2
Author: dan-s1 <ds...@gmail.com>
AuthorDate: Tue Nov 12 16:27:36 2024 -0500
NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)
Signed-off-by: David Handermann <ex...@apache.org>
---
.../serialization/record/util/DataTypeUtils.java | 34 ++++++++--
.../serialization/record/TestDataTypeUtils.java | 12 ++++
.../apache/nifi/excel/TestExcelRecordReader.java | 76 ++++++++++++++--------
3 files changed, 89 insertions(+), 33 deletions(-)
diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
index fe457c75de..117fae95ad 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
@@ -1476,7 +1476,8 @@ public class DataTypeUtils {
if (value instanceof String) {
try {
- return new BigInteger((String) value);
+ final String string = (String)value;
+ return isBlank(string) ? null : new BigInteger((String) value);
} catch (NumberFormatException nfe) {
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigInteger for field " + fieldName
+ ", value is not a valid representation of BigInteger", nfe);
@@ -1562,7 +1563,8 @@ public class DataTypeUtils {
if (value instanceof String) {
try {
- return new BigDecimal((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : new BigDecimal(string);
} catch (NumberFormatException nfe) {
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigDecimal for field " + fieldName
+ ", value is not a valid representation of BigDecimal", nfe);
@@ -1582,7 +1584,8 @@ public class DataTypeUtils {
}
if (value instanceof String) {
- return Double.parseDouble((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : Double.parseDouble(string);
}
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Double for field " + fieldName);
@@ -1618,7 +1621,8 @@ public class DataTypeUtils {
}
if (value instanceof String) {
- return Float.parseFloat((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : Float.parseFloat(string);
}
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Float for field " + fieldName);
@@ -1684,7 +1688,8 @@ public class DataTypeUtils {
}
if (value instanceof String) {
- return Long.parseLong((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : Long.parseLong(string);
}
if (value instanceof java.util.Date) {
@@ -1774,7 +1779,8 @@ public class DataTypeUtils {
}
if (value instanceof String) {
- return Integer.parseInt((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : Integer.parseInt(string);
}
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Integer for field " + fieldName);
@@ -1803,7 +1809,8 @@ public class DataTypeUtils {
}
if (value instanceof String) {
- return Short.parseShort((String) value);
+ final String string = (String) value;
+ return isBlank(string) ? null : Short.parseShort(string);
}
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Short for field " + fieldName);
@@ -2386,4 +2393,17 @@ public class DataTypeUtils {
return false;
}
+
+ private static boolean isBlank(String string) {
+ final int strLen = string.length();
+ if (strLen == 0) {
+ return true;
+ }
+ for (int i = 0; i < strLen; i++) {
+ if (!Character.isWhitespace(string.charAt(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
index 439637eab0..87a40356ce 100644
--- a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
@@ -1110,6 +1110,18 @@ public class TestDataTypeUtils {
assertToLocalDateEquals(ISO_8601_YEAR_MONTH_DAY, epochMillis);
}
+ @Test
+ void testNumberParsingWhereStringBlank() {
+ final String fieldName = "someField";
+ assertNull(DataTypeUtils.toBigDecimal("", fieldName));
+ assertNull(DataTypeUtils.toBigInt("", fieldName));
+ assertNull(DataTypeUtils.toDouble("", fieldName));
+ assertNull(DataTypeUtils.toFloat("", fieldName));
+ assertNull(DataTypeUtils.toInteger("", fieldName));
+ assertNull(DataTypeUtils.toLong("", fieldName));
+ assertNull(DataTypeUtils.toShort("", fieldName));
+ }
+
private long toEpochMilliSystemDefaultZone(final LocalDate localDate) {
final LocalTime localTime = LocalTime.of(0, 0);
final Instant instantSystemDefaultZone = ZonedDateTime.of(localDate, localTime, SYSTEM_DEFAULT_ZONE_ID).toInstant();
diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
index e088d5761d..1f2b7e25e7 100644
--- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
+++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
@@ -53,6 +53,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -78,12 +79,7 @@ public class TestExcelRecordReader {
@BeforeAll
static void setUpBeforeAll() throws Exception {
//Generate an Excel file and populate it with data
- final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- try (XSSFWorkbook workbook = new XSSFWorkbook()) {
- final XSSFSheet sheet = workbook.createSheet("User Info");
- populateSheet(sheet);
- workbook.write(outputStream);
- }
+ final InputStream workbook = createWorkbook(DATA);
//Protect the Excel file with a password
try (POIFSFileSystem poifsFileSystem = new POIFSFileSystem()) {
@@ -91,7 +87,7 @@ public class TestExcelRecordReader {
Encryptor encryptor = encryptionInfo.getEncryptor();
encryptor.confirmPassword(PASSWORD);
- try (OPCPackage opc = OPCPackage.open(new ByteArrayInputStream(outputStream.toByteArray()));
+ try (OPCPackage opc = OPCPackage.open(workbook);
OutputStream os = encryptor.getDataStream(poifsFileSystem)) {
opc.save(os);
}
@@ -99,24 +95,6 @@ public class TestExcelRecordReader {
}
}
- private static void populateSheet(XSSFSheet sheet) {
- //Adding the data to the Excel worksheet
- int rowCount = 0;
- for (Object[] dataRow : DATA) {
- Row row = sheet.createRow(rowCount++);
- int columnCount = 0;
-
- for (Object field : dataRow) {
- Cell cell = row.createCell(columnCount++);
- if (field instanceof String) {
- cell.setCellValue((String) field);
- } else if (field instanceof Integer) {
- cell.setCellValue((Integer) field);
- }
- }
- }
- }
-
@Test
void testNonExcelFile() {
ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
@@ -190,7 +168,7 @@ public class TestExcelRecordReader {
List<Record> records = getRecords(recordReader, false, dropUnknownFields);
assertEquals(9, records.size());
- if(dropUnknownFields) {
+ if (dropUnknownFields) {
records.forEach(record -> assertEquals(fields.size(), record.getRawFieldNames().size()));
} else {
records.forEach(record -> {
@@ -335,4 +313,50 @@ public class TestExcelRecordReader {
return new SimpleRecordSchema(Arrays.asList(new RecordField("id", RecordFieldType.INT.getDataType()),
new RecordField("name", RecordFieldType.STRING.getDataType())));
}
+
+ @Test
+ void testWithNumberColumnWhoseValueIsEmptyString() throws Exception {
+ final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(new RecordField("first", RecordFieldType.STRING.getDataType()),
+ new RecordField("second", RecordFieldType.LONG.getDataType())));
+ final ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
+ .withSchema(schema)
+ .build();
+
+ final Object[][] data = {{"Manny", ""}};
+ final InputStream workbook = createWorkbook(data);
+ final ExcelRecordReader recordReader = new ExcelRecordReader(configuration, workbook, logger);
+
+ assertDoesNotThrow(() -> getRecords(recordReader, true, true));
+ }
+
+ private static InputStream createWorkbook(Object[][] data) throws Exception {
+ final ByteArrayOutputStream workbookOutputStream = new ByteArrayOutputStream();
+ try (XSSFWorkbook workbook = new XSSFWorkbook()) {
+ final XSSFSheet sheet = workbook.createSheet("SomeSheetName");
+ populateSheet(sheet, data);
+ workbook.write(workbookOutputStream);
+ }
+
+ return new ByteArrayInputStream(workbookOutputStream.toByteArray());
+ }
+
+ private static void populateSheet(XSSFSheet sheet, Object[][] data) {
+ //Adding the data to the Excel worksheet
+ int rowCount = 0;
+ for (Object[] dataRow : data) {
+ Row row = sheet.createRow(rowCount++);
+ int columnCount = 0;
+
+ for (Object field : dataRow) {
+ Cell cell = row.createCell(columnCount++);
+ if (field instanceof String) {
+ cell.setCellValue((String)field);
+ } else if (field instanceof Integer) {
+ cell.setCellValue(((Integer)field).doubleValue());
+ } else if (field instanceof Long) {
+ cell.setCellValue(((Long)field).doubleValue());
+ }
+ }
+ }
+ }
}