You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 09:51:24 UTC

(seatunnel) 01/01: Fix Parquet INT96 writes for normalized field names

This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a commit to branch davidzollo_republish_10943_parquet_int96
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit b0c082b3d69afe216919a7b38f629b6914fed7cc
Author: Asish Kumar <of...@gmail.com>
AuthorDate: Mon May 25 08:47:03 2026 +0530

    Fix Parquet INT96 writes for normalized field names
    
    (cherry picked from commit 824dac5619d818e88e193d2f4bc860ec4a3b2795)
---
 .../file/sink/writer/ParquetWriteStrategy.java       |  9 +++++----
 .../file/writer/ParquetWriteStrategyTest.java        | 20 ++++++++++++++++----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
index e72436338f..b95cb8cdf8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
@@ -127,10 +127,11 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
         GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
         for (Integer integer : sinkColumnsIndexInRow) {
             String fieldName = seaTunnelRowType.getFieldName(integer);
+            String parquetFieldName = normalizeFieldName(fieldName);
             Object field = getFieldSafe(seaTunnelRow, integer);
             recordBuilder.set(
-                    fieldName.toLowerCase(),
-                    resolveObject(fieldName, field, seaTunnelRowType.getFieldType(integer)));
+                    parquetFieldName,
+                    resolveObject(parquetFieldName, field, seaTunnelRowType.getFieldType(integer)));
         }
         GenericData.Record record = recordBuilder.build();
         try {
@@ -307,7 +308,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
                 GenericRecordBuilder recordBuilder = new GenericRecordBuilder(recordSchema);
                 for (int i = 0; i < fieldNames.length; i++) {
                     recordBuilder.set(
-                            fieldNames[i].toLowerCase(),
+                            normalizeFieldName(fieldNames[i]),
                             resolveObject(fieldNames[i], seaTunnelRow.getField(i), fieldTypes[i]));
                 }
                 return recordBuilder.build();
@@ -468,7 +469,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
                 index -> {
                     Type type =
                             seaTunnelDataType2ParquetDataType(
-                                    fieldNames[index].toLowerCase(), fieldTypes[index]);
+                                    normalizeFieldName(fieldNames[index]), fieldTypes[index]);
                     types.add(type);
                 });
         MessageType seaTunnelRow =
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
index 7c1a71eef2..3599a06392 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
@@ -111,15 +111,16 @@ public class ParquetWriteStrategyTest {
         writeConfig.put("path", "file:///tmp/seatunnel/parquet/int96");
         writeConfig.put("file_format_type", FileFormat.PARQUET.name());
         writeConfig.put("parquet_avro_write_timestamp_as_int96", "true");
-        writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("f3_bytes"));
+        writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("F3_Bytes"));
 
         SeaTunnelRowType writeRowType =
                 new SeaTunnelRowType(
-                        new String[] {"f1_text", "f2_timestamp", "f3_bytes"},
+                        new String[] {"f1_text", "f2_timestamp", "F3_Bytes", "createTime"},
                         new SeaTunnelDataType[] {
                             BasicType.STRING_TYPE,
                             LocalTimeType.LOCAL_DATE_TIME_TYPE,
-                            PrimitiveByteArrayType.INSTANCE
+                            PrimitiveByteArrayType.INSTANCE,
+                            LocalTimeType.LOCAL_DATE_TIME_TYPE
                         });
         FileSinkConfig writeSinkConfig =
                 new FileSinkConfig(ReadonlyConfig.fromMap(writeConfig), writeRowType);
@@ -131,7 +132,10 @@ public class ParquetWriteStrategyTest {
         writeStrategy.init(hadoopConf, "test1", "test1", 0);
         writeStrategy.beginTransaction(1L);
         writeStrategy.write(
-                new SeaTunnelRow(new Object[] {"test", LocalDateTime.now(), new byte[12]}));
+                new SeaTunnelRow(
+                        new Object[] {
+                            "test", LocalDateTime.now(), new byte[12], LocalDateTime.now()
+                        }));
         writeStrategy.finishAndCloseFile();
         writeStrategy.close();
 
@@ -161,6 +165,10 @@ public class ParquetWriteStrategyTest {
             Assertions.assertEquals(
                     PrimitiveType.PrimitiveTypeName.INT96,
                     f3Type.asPrimitiveType().getPrimitiveTypeName());
+            Type createTimeType = metadata.getSchema().getType("createtime");
+            Assertions.assertEquals(
+                    PrimitiveType.PrimitiveTypeName.INT96,
+                    createTimeType.asPrimitiveType().getPrimitiveTypeName());
         }
 
         SeaTunnelRowType readRowType = readStrategy.getSeaTunnelRowTypeInfo(readFilePath);
@@ -172,6 +180,9 @@ public class ParquetWriteStrategyTest {
         Assertions.assertEquals(
                 LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
                 readRowType.getFieldType(2).getSqlType());
+        Assertions.assertEquals(
+                LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
+                readRowType.getFieldType(3).getSqlType());
         List<SeaTunnelRow> readRows = new ArrayList<>();
         Collector<SeaTunnelRow> readCollector =
                 new Collector<SeaTunnelRow>() {
@@ -180,6 +191,7 @@ public class ParquetWriteStrategyTest {
                         Assertions.assertTrue(record.getField(0) instanceof String);
                         Assertions.assertTrue(record.getField(1) instanceof LocalDateTime);
                         Assertions.assertTrue(record.getField(2) instanceof LocalDateTime);
+                        Assertions.assertTrue(record.getField(3) instanceof LocalDateTime);
                         readRows.add(record);
                     }