You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2024/11/12 21:27:42 UTC

(nifi) branch support/nifi-1.x updated: NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)

This is an automated email from the ASF dual-hosted git repository.

exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
     new 41616773db NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)
41616773db is described below

commit 41616773dbb1e7dc9173170ecf3e46d6c88e5ea2
Author: dan-s1 <ds...@gmail.com>
AuthorDate: Tue Nov 12 16:27:36 2024 -0500

    NIFI-13988 Adjusted Record number conversion to treat empty String as null (#9513)
    
    Signed-off-by: David Handermann <ex...@apache.org>
---
 .../serialization/record/util/DataTypeUtils.java   | 34 ++++++++--
 .../serialization/record/TestDataTypeUtils.java    | 12 ++++
 .../apache/nifi/excel/TestExcelRecordReader.java   | 76 ++++++++++++++--------
 3 files changed, 89 insertions(+), 33 deletions(-)

diff --git a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
index fe457c75de..117fae95ad 100644
--- a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
@@ -1476,7 +1476,8 @@ public class DataTypeUtils {
 
         if (value instanceof String) {
             try {
-                return new BigInteger((String) value);
+                final String string = (String)value;
+                return isBlank(string) ? null : new BigInteger((String) value);
             } catch (NumberFormatException nfe) {
                 throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigInteger for field " + fieldName
                         + ", value is not a valid representation of BigInteger", nfe);
@@ -1562,7 +1563,8 @@ public class DataTypeUtils {
 
         if (value instanceof String) {
             try {
-                return new BigDecimal((String) value);
+                final String string = (String) value;
+                return isBlank(string) ? null : new BigDecimal(string);
             } catch (NumberFormatException nfe) {
                 throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigDecimal for field " + fieldName
                         + ", value is not a valid representation of BigDecimal", nfe);
@@ -1582,7 +1584,8 @@ public class DataTypeUtils {
         }
 
         if (value instanceof String) {
-            return Double.parseDouble((String) value);
+            final String string = (String) value;
+            return isBlank(string) ? null : Double.parseDouble(string);
         }
 
         throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Double for field " + fieldName);
@@ -1618,7 +1621,8 @@ public class DataTypeUtils {
         }
 
         if (value instanceof String) {
-            return Float.parseFloat((String) value);
+            final String string = (String) value;
+            return isBlank(string) ? null : Float.parseFloat(string);
         }
 
         throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Float for field " + fieldName);
@@ -1684,7 +1688,8 @@ public class DataTypeUtils {
         }
 
         if (value instanceof String) {
-            return Long.parseLong((String) value);
+            final String string = (String) value;
+            return isBlank(string) ? null : Long.parseLong(string);
         }
 
         if (value instanceof java.util.Date) {
@@ -1774,7 +1779,8 @@ public class DataTypeUtils {
         }
 
         if (value instanceof String) {
-            return Integer.parseInt((String) value);
+            final String string = (String) value;
+            return isBlank(string) ? null : Integer.parseInt(string);
         }
 
         throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Integer for field " + fieldName);
@@ -1803,7 +1809,8 @@ public class DataTypeUtils {
         }
 
         if (value instanceof String) {
-            return Short.parseShort((String) value);
+            final String string = (String) value;
+            return isBlank(string) ? null : Short.parseShort(string);
         }
 
         throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Short for field " + fieldName);
@@ -2386,4 +2393,17 @@ public class DataTypeUtils {
 
         return false;
     }
+
+    private static boolean isBlank(String string) {
+        final int strLen = string.length();
+        if (strLen == 0) {
+            return true;
+        }
+        for (int i = 0; i < strLen; i++) {
+            if (!Character.isWhitespace(string.charAt(i))) {
+                return false;
+            }
+        }
+        return true;
+    }
 }
diff --git a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
index 439637eab0..87a40356ce 100644
--- a/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
+++ b/nifi-commons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
@@ -1110,6 +1110,18 @@ public class TestDataTypeUtils {
         assertToLocalDateEquals(ISO_8601_YEAR_MONTH_DAY, epochMillis);
     }
 
+    @Test
+    void testNumberParsingWhereStringBlank() {
+        final String fieldName = "someField";
+        assertNull(DataTypeUtils.toBigDecimal("", fieldName));
+        assertNull(DataTypeUtils.toBigInt("", fieldName));
+        assertNull(DataTypeUtils.toDouble("", fieldName));
+        assertNull(DataTypeUtils.toFloat("", fieldName));
+        assertNull(DataTypeUtils.toInteger("", fieldName));
+        assertNull(DataTypeUtils.toLong("", fieldName));
+        assertNull(DataTypeUtils.toShort("", fieldName));
+    }
+
     private long toEpochMilliSystemDefaultZone(final LocalDate localDate) {
         final LocalTime localTime = LocalTime.of(0, 0);
         final Instant instantSystemDefaultZone = ZonedDateTime.of(localDate, localTime, SYSTEM_DEFAULT_ZONE_ID).toInstant();
diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
index e088d5761d..1f2b7e25e7 100644
--- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
+++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
@@ -53,6 +53,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -78,12 +79,7 @@ public class TestExcelRecordReader {
     @BeforeAll
     static void setUpBeforeAll() throws Exception {
         //Generate an Excel file and populate it with data
-        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        try (XSSFWorkbook workbook = new XSSFWorkbook()) {
-            final XSSFSheet sheet = workbook.createSheet("User Info");
-            populateSheet(sheet);
-            workbook.write(outputStream);
-        }
+        final InputStream workbook = createWorkbook(DATA);
 
         //Protect the Excel file with a password
         try (POIFSFileSystem poifsFileSystem = new POIFSFileSystem()) {
@@ -91,7 +87,7 @@ public class TestExcelRecordReader {
             Encryptor encryptor = encryptionInfo.getEncryptor();
             encryptor.confirmPassword(PASSWORD);
 
-            try (OPCPackage opc = OPCPackage.open(new ByteArrayInputStream(outputStream.toByteArray()));
+            try (OPCPackage opc = OPCPackage.open(workbook);
                  OutputStream os = encryptor.getDataStream(poifsFileSystem)) {
                 opc.save(os);
             }
@@ -99,24 +95,6 @@ public class TestExcelRecordReader {
         }
     }
 
-    private static void populateSheet(XSSFSheet sheet) {
-        //Adding the data to the Excel worksheet
-        int rowCount = 0;
-        for (Object[] dataRow : DATA) {
-            Row row = sheet.createRow(rowCount++);
-            int columnCount = 0;
-
-            for (Object field : dataRow) {
-                Cell cell = row.createCell(columnCount++);
-                if (field instanceof String) {
-                    cell.setCellValue((String) field);
-                } else if (field instanceof Integer) {
-                    cell.setCellValue((Integer) field);
-                }
-            }
-        }
-    }
-
     @Test
     void testNonExcelFile() {
         ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
@@ -190,7 +168,7 @@ public class TestExcelRecordReader {
         List<Record> records = getRecords(recordReader, false, dropUnknownFields);
 
         assertEquals(9, records.size());
-        if(dropUnknownFields) {
+        if (dropUnknownFields) {
             records.forEach(record -> assertEquals(fields.size(), record.getRawFieldNames().size()));
         } else {
             records.forEach(record -> {
@@ -335,4 +313,50 @@ public class TestExcelRecordReader {
         return new SimpleRecordSchema(Arrays.asList(new RecordField("id", RecordFieldType.INT.getDataType()),
                 new RecordField("name", RecordFieldType.STRING.getDataType())));
     }
+
+    @Test
+    void testWithNumberColumnWhoseValueIsEmptyString() throws Exception {
+        final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(new RecordField("first", RecordFieldType.STRING.getDataType()),
+                new RecordField("second", RecordFieldType.LONG.getDataType())));
+        final ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
+                .withSchema(schema)
+                .build();
+
+        final Object[][] data = {{"Manny", ""}};
+        final InputStream workbook = createWorkbook(data);
+        final ExcelRecordReader recordReader = new ExcelRecordReader(configuration, workbook, logger);
+
+        assertDoesNotThrow(() -> getRecords(recordReader, true, true));
+    }
+
+    private static InputStream createWorkbook(Object[][] data) throws Exception {
+        final ByteArrayOutputStream workbookOutputStream = new ByteArrayOutputStream();
+        try (XSSFWorkbook workbook = new XSSFWorkbook()) {
+            final XSSFSheet sheet = workbook.createSheet("SomeSheetName");
+            populateSheet(sheet, data);
+            workbook.write(workbookOutputStream);
+        }
+
+        return new ByteArrayInputStream(workbookOutputStream.toByteArray());
+    }
+
+    private static void populateSheet(XSSFSheet sheet, Object[][] data) {
+        //Adding the data to the Excel worksheet
+        int rowCount = 0;
+        for (Object[] dataRow : data) {
+            Row row = sheet.createRow(rowCount++);
+            int columnCount = 0;
+
+            for (Object field : dataRow) {
+                Cell cell = row.createCell(columnCount++);
+                if (field instanceof String) {
+                    cell.setCellValue((String)field);
+                } else if (field instanceof Integer) {
+                    cell.setCellValue(((Integer)field).doubleValue());
+                } else if (field instanceof Long) {
+                    cell.setCellValue(((Long)field).doubleValue());
+                }
+            }
+        }
+    }
 }