You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 09:51:23 UTC

(seatunnel) branch davidzollo_republish_10943_parquet_int96 created (now b0c082b3d6)

This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a change to branch davidzollo_republish_10943_parquet_int96
in repository https://gitbox.apache.org/repos/asf/seatunnel.git


      at b0c082b3d6 Fix Parquet INT96 writes for normalized field names

This branch includes the following new commits:

     new b0c082b3d6 Fix Parquet INT96 writes for normalized field names

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(seatunnel) 01/01: Fix Parquet INT96 writes for normalized field names

Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a commit to branch davidzollo_republish_10943_parquet_int96
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit b0c082b3d69afe216919a7b38f629b6914fed7cc
Author: Asish Kumar <of...@gmail.com>
AuthorDate: Mon May 25 08:47:03 2026 +0530

    Fix Parquet INT96 writes for normalized field names
    
    (cherry picked from commit 824dac5619d818e88e193d2f4bc860ec4a3b2795)
---
 .../file/sink/writer/ParquetWriteStrategy.java       |  9 +++++----
 .../file/writer/ParquetWriteStrategyTest.java        | 20 ++++++++++++++++----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
index e72436338f..b95cb8cdf8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
@@ -127,10 +127,11 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
         GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
         for (Integer integer : sinkColumnsIndexInRow) {
             String fieldName = seaTunnelRowType.getFieldName(integer);
+            String parquetFieldName = normalizeFieldName(fieldName);
             Object field = getFieldSafe(seaTunnelRow, integer);
             recordBuilder.set(
-                    fieldName.toLowerCase(),
-                    resolveObject(fieldName, field, seaTunnelRowType.getFieldType(integer)));
+                    parquetFieldName,
+                    resolveObject(parquetFieldName, field, seaTunnelRowType.getFieldType(integer)));
         }
         GenericData.Record record = recordBuilder.build();
         try {
@@ -307,7 +308,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
                 GenericRecordBuilder recordBuilder = new GenericRecordBuilder(recordSchema);
                 for (int i = 0; i < fieldNames.length; i++) {
                     recordBuilder.set(
-                            fieldNames[i].toLowerCase(),
+                            normalizeFieldName(fieldNames[i]),
                             resolveObject(fieldNames[i], seaTunnelRow.getField(i), fieldTypes[i]));
                 }
                 return recordBuilder.build();
@@ -468,7 +469,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
                 index -> {
                     Type type =
                             seaTunnelDataType2ParquetDataType(
-                                    fieldNames[index].toLowerCase(), fieldTypes[index]);
+                                    normalizeFieldName(fieldNames[index]), fieldTypes[index]);
                     types.add(type);
                 });
         MessageType seaTunnelRow =
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
index 7c1a71eef2..3599a06392 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
@@ -111,15 +111,16 @@ public class ParquetWriteStrategyTest {
         writeConfig.put("path", "file:///tmp/seatunnel/parquet/int96");
         writeConfig.put("file_format_type", FileFormat.PARQUET.name());
         writeConfig.put("parquet_avro_write_timestamp_as_int96", "true");
-        writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("f3_bytes"));
+        writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("F3_Bytes"));
 
         SeaTunnelRowType writeRowType =
                 new SeaTunnelRowType(
-                        new String[] {"f1_text", "f2_timestamp", "f3_bytes"},
+                        new String[] {"f1_text", "f2_timestamp", "F3_Bytes", "createTime"},
                         new SeaTunnelDataType[] {
                             BasicType.STRING_TYPE,
                             LocalTimeType.LOCAL_DATE_TIME_TYPE,
-                            PrimitiveByteArrayType.INSTANCE
+                            PrimitiveByteArrayType.INSTANCE,
+                            LocalTimeType.LOCAL_DATE_TIME_TYPE
                         });
         FileSinkConfig writeSinkConfig =
                 new FileSinkConfig(ReadonlyConfig.fromMap(writeConfig), writeRowType);
@@ -131,7 +132,10 @@ public class ParquetWriteStrategyTest {
         writeStrategy.init(hadoopConf, "test1", "test1", 0);
         writeStrategy.beginTransaction(1L);
         writeStrategy.write(
-                new SeaTunnelRow(new Object[] {"test", LocalDateTime.now(), new byte[12]}));
+                new SeaTunnelRow(
+                        new Object[] {
+                            "test", LocalDateTime.now(), new byte[12], LocalDateTime.now()
+                        }));
         writeStrategy.finishAndCloseFile();
         writeStrategy.close();
 
@@ -161,6 +165,10 @@ public class ParquetWriteStrategyTest {
             Assertions.assertEquals(
                     PrimitiveType.PrimitiveTypeName.INT96,
                     f3Type.asPrimitiveType().getPrimitiveTypeName());
+            Type createTimeType = metadata.getSchema().getType("createtime");
+            Assertions.assertEquals(
+                    PrimitiveType.PrimitiveTypeName.INT96,
+                    createTimeType.asPrimitiveType().getPrimitiveTypeName());
         }
 
         SeaTunnelRowType readRowType = readStrategy.getSeaTunnelRowTypeInfo(readFilePath);
@@ -172,6 +180,9 @@ public class ParquetWriteStrategyTest {
         Assertions.assertEquals(
                 LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
                 readRowType.getFieldType(2).getSqlType());
+        Assertions.assertEquals(
+                LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
+                readRowType.getFieldType(3).getSqlType());
         List<SeaTunnelRow> readRows = new ArrayList<>();
         Collector<SeaTunnelRow> readCollector =
                 new Collector<SeaTunnelRow>() {
@@ -180,6 +191,7 @@ public class ParquetWriteStrategyTest {
                         Assertions.assertTrue(record.getField(0) instanceof String);
                         Assertions.assertTrue(record.getField(1) instanceof LocalDateTime);
                         Assertions.assertTrue(record.getField(2) instanceof LocalDateTime);
+                        Assertions.assertTrue(record.getField(3) instanceof LocalDateTime);
                         readRows.add(record);
                     }