You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 09:51:24 UTC
(seatunnel) 01/01: Fix Parquet INT96 writes for normalized field names
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch davidzollo_republish_10943_parquet_int96
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit b0c082b3d69afe216919a7b38f629b6914fed7cc
Author: Asish Kumar <of...@gmail.com>
AuthorDate: Mon May 25 08:47:03 2026 +0530
Fix Parquet INT96 writes for normalized field names
(cherry picked from commit 824dac5619d818e88e193d2f4bc860ec4a3b2795)
---
.../file/sink/writer/ParquetWriteStrategy.java | 9 +++++----
.../file/writer/ParquetWriteStrategyTest.java | 20 ++++++++++++++++----
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
index e72436338f..b95cb8cdf8 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/writer/ParquetWriteStrategy.java
@@ -127,10 +127,11 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
for (Integer integer : sinkColumnsIndexInRow) {
String fieldName = seaTunnelRowType.getFieldName(integer);
+ String parquetFieldName = normalizeFieldName(fieldName);
Object field = getFieldSafe(seaTunnelRow, integer);
recordBuilder.set(
- fieldName.toLowerCase(),
- resolveObject(fieldName, field, seaTunnelRowType.getFieldType(integer)));
+ parquetFieldName,
+ resolveObject(parquetFieldName, field, seaTunnelRowType.getFieldType(integer)));
}
GenericData.Record record = recordBuilder.build();
try {
@@ -307,7 +308,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
GenericRecordBuilder recordBuilder = new GenericRecordBuilder(recordSchema);
for (int i = 0; i < fieldNames.length; i++) {
recordBuilder.set(
- fieldNames[i].toLowerCase(),
+ normalizeFieldName(fieldNames[i]),
resolveObject(fieldNames[i], seaTunnelRow.getField(i), fieldTypes[i]));
}
return recordBuilder.build();
@@ -468,7 +469,7 @@ public class ParquetWriteStrategy extends AbstractWriteStrategy<ParquetWriter<Ge
index -> {
Type type =
seaTunnelDataType2ParquetDataType(
- fieldNames[index].toLowerCase(), fieldTypes[index]);
+ normalizeFieldName(fieldNames[index]), fieldTypes[index]);
types.add(type);
});
MessageType seaTunnelRow =
diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
index 7c1a71eef2..3599a06392 100644
--- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
+++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/writer/ParquetWriteStrategyTest.java
@@ -111,15 +111,16 @@ public class ParquetWriteStrategyTest {
writeConfig.put("path", "file:///tmp/seatunnel/parquet/int96");
writeConfig.put("file_format_type", FileFormat.PARQUET.name());
writeConfig.put("parquet_avro_write_timestamp_as_int96", "true");
- writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("f3_bytes"));
+ writeConfig.put("parquet_avro_write_fixed_as_int96", Arrays.asList("F3_Bytes"));
SeaTunnelRowType writeRowType =
new SeaTunnelRowType(
- new String[] {"f1_text", "f2_timestamp", "f3_bytes"},
+ new String[] {"f1_text", "f2_timestamp", "F3_Bytes", "createTime"},
new SeaTunnelDataType[] {
BasicType.STRING_TYPE,
LocalTimeType.LOCAL_DATE_TIME_TYPE,
- PrimitiveByteArrayType.INSTANCE
+ PrimitiveByteArrayType.INSTANCE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE
});
FileSinkConfig writeSinkConfig =
new FileSinkConfig(ReadonlyConfig.fromMap(writeConfig), writeRowType);
@@ -131,7 +132,10 @@ public class ParquetWriteStrategyTest {
writeStrategy.init(hadoopConf, "test1", "test1", 0);
writeStrategy.beginTransaction(1L);
writeStrategy.write(
- new SeaTunnelRow(new Object[] {"test", LocalDateTime.now(), new byte[12]}));
+ new SeaTunnelRow(
+ new Object[] {
+ "test", LocalDateTime.now(), new byte[12], LocalDateTime.now()
+ }));
writeStrategy.finishAndCloseFile();
writeStrategy.close();
@@ -161,6 +165,10 @@ public class ParquetWriteStrategyTest {
Assertions.assertEquals(
PrimitiveType.PrimitiveTypeName.INT96,
f3Type.asPrimitiveType().getPrimitiveTypeName());
+ Type createTimeType = metadata.getSchema().getType("createtime");
+ Assertions.assertEquals(
+ PrimitiveType.PrimitiveTypeName.INT96,
+ createTimeType.asPrimitiveType().getPrimitiveTypeName());
}
SeaTunnelRowType readRowType = readStrategy.getSeaTunnelRowTypeInfo(readFilePath);
@@ -172,6 +180,9 @@ public class ParquetWriteStrategyTest {
Assertions.assertEquals(
LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
readRowType.getFieldType(2).getSqlType());
+ Assertions.assertEquals(
+ LocalTimeType.LOCAL_DATE_TIME_TYPE.getSqlType(),
+ readRowType.getFieldType(3).getSqlType());
List<SeaTunnelRow> readRows = new ArrayList<>();
Collector<SeaTunnelRow> readCollector =
new Collector<SeaTunnelRow>() {
@@ -180,6 +191,7 @@ public class ParquetWriteStrategyTest {
Assertions.assertTrue(record.getField(0) instanceof String);
Assertions.assertTrue(record.getField(1) instanceof LocalDateTime);
Assertions.assertTrue(record.getField(2) instanceof LocalDateTime);
+ Assertions.assertTrue(record.getField(3) instanceof LocalDateTime);
readRows.add(record);
}