You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/01/15 02:11:59 UTC
(incubator-paimon) 15/20: [parquet] Fix timestamp type write parquet file not have logicaltype (#2558)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.6
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 99c2200e0c082abc26214184a73a4382d58d2a8a
Author: leoyy0316 <10...@users.noreply.github.com>
AuthorDate: Mon Dec 25 15:55:15 2023 +0800
[parquet] Fix timestamp type write parquet file not have logicaltype (#2558)
---
.../format/parquet/ParquetSchemaConverter.java | 23 ++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
index 4ebc3e1bf..24160df56 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
@@ -114,16 +114,12 @@ public class ParquetSchemaConverter {
return Types.primitive(INT32, repetition).as(OriginalType.TIME_MILLIS).named(name);
case TIMESTAMP_WITHOUT_TIME_ZONE:
TimestampType timestampType = (TimestampType) type;
- return timestampType.getPrecision() <= 6
- ? Types.primitive(INT64, repetition).named(name)
- : Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, repetition)
- .named(name);
+ return createTimestampWithLogicalType(
+ name, timestampType.getPrecision(), repetition);
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
LocalZonedTimestampType localZonedTimestampType = (LocalZonedTimestampType) type;
- return localZonedTimestampType.getPrecision() <= 6
- ? Types.primitive(INT64, repetition).named(name)
- : Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, repetition)
- .named(name);
+ return createTimestampWithLogicalType(
+ name, localZonedTimestampType.getPrecision(), repetition);
case ARRAY:
ArrayType arrayType = (ArrayType) type;
return ConversionPatterns.listOfElements(
@@ -154,6 +150,17 @@ public class ParquetSchemaConverter {
}
}
+ private static Type createTimestampWithLogicalType(
+ String name, int precision, Type.Repetition repetition) {
+ if (precision <= 3) {
+ return Types.primitive(INT64, repetition).as(OriginalType.TIMESTAMP_MILLIS).named(name);
+ } else if (precision > 6) {
+ return Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, repetition).named(name);
+ } else {
+ return Types.primitive(INT64, repetition).as(OriginalType.TIMESTAMP_MICROS).named(name);
+ }
+ }
+
private static List<Type> convertToParquetTypes(RowType rowType) {
List<Type> types = new ArrayList<>(rowType.getFieldCount());
for (int i = 0; i < rowType.getFieldCount(); i++) {