You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/07/10 02:51:55 UTC

(paimon) branch master updated: [hotfix] Fix deprecated apis in paimon-format related to parquet (#3709)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 4c064b56d [hotfix] Fix deprecated apis in paimon-format related to parquet (#3709)
4c064b56d is described below

commit 4c064b56dba5ddd4563a9ada254d8ab8bd14aac7
Author: Wei Guo <gu...@gmail.com>
AuthorDate: Wed Jul 10 10:51:49 2024 +0800

    [hotfix] Fix deprecated apis in paimon-format related to parquet (#3709)
---
 .../format/parquet/ParquetSchemaConverter.java      | 21 +++++++++++++++------
 .../parquet/reader/ParquetSplitReaderUtil.java      | 11 +++++++----
 .../parquet/writer/PositionOutputStreamAdapter.java |  5 ++---
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
index 61e0e1d6b..345c9944c 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java
@@ -32,7 +32,6 @@ import org.apache.parquet.schema.ConversionPatterns;
 import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.OriginalType;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
 import org.apache.parquet.schema.Types;
@@ -70,7 +69,7 @@ public class ParquetSchemaConverter {
             case CHAR:
             case VARCHAR:
                 return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
-                        .as(OriginalType.UTF8)
+                        .as(LogicalTypeAnnotation.stringType())
                         .named(name);
             case BOOLEAN:
                 return Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition)
@@ -97,9 +96,13 @@ public class ParquetSchemaConverter {
                             .named(name);
                 }
             case TINYINT:
-                return Types.primitive(INT32, repetition).as(OriginalType.INT_8).named(name);
+                return Types.primitive(INT32, repetition)
+                        .as(LogicalTypeAnnotation.intType(8, true))
+                        .named(name);
             case SMALLINT:
-                return Types.primitive(INT32, repetition).as(OriginalType.INT_16).named(name);
+                return Types.primitive(INT32, repetition)
+                        .as(LogicalTypeAnnotation.intType(16, true))
+                        .named(name);
             case INTEGER:
                 return Types.primitive(INT32, repetition).named(name);
             case BIGINT:
@@ -111,9 +114,15 @@ public class ParquetSchemaConverter {
                 return Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition)
                         .named(name);
             case DATE:
-                return Types.primitive(INT32, repetition).as(OriginalType.DATE).named(name);
+                return Types.primitive(INT32, repetition)
+                        .as(LogicalTypeAnnotation.dateType())
+                        .named(name);
             case TIME_WITHOUT_TIME_ZONE:
-                return Types.primitive(INT32, repetition).as(OriginalType.TIME_MILLIS).named(name);
+                return Types.primitive(INT32, repetition)
+                        .as(
+                                LogicalTypeAnnotation.timeType(
+                                        true, LogicalTypeAnnotation.TimeUnit.MILLIS))
+                        .named(name);
             case TIMESTAMP_WITHOUT_TIME_ZONE:
                 TimestampType timestampType = (TimestampType) type;
                 return createTimestampWithLogicalType(
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java
index 8a362961d..90abaa992 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java
@@ -58,7 +58,7 @@ import org.apache.parquet.io.PrimitiveColumnIO;
 import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.schema.InvalidSchemaException;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
 
@@ -233,7 +233,8 @@ public class ParquetSplitReaderUtil {
                     checkArgument(
                             (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
                                             || typeName == PrimitiveType.PrimitiveTypeName.INT32)
-                                    && primitiveType.getOriginalType() == OriginalType.DECIMAL,
+                                    && primitiveType.getLogicalTypeAnnotation()
+                                            instanceof DecimalLogicalTypeAnnotation,
                             "Unexpected type: %s",
                             typeName);
                     return new HeapIntVector(batchSize);
@@ -241,7 +242,8 @@ public class ParquetSplitReaderUtil {
                     checkArgument(
                             (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
                                             || typeName == PrimitiveType.PrimitiveTypeName.INT64)
-                                    && primitiveType.getOriginalType() == OriginalType.DECIMAL,
+                                    && primitiveType.getLogicalTypeAnnotation()
+                                            instanceof DecimalLogicalTypeAnnotation,
                             "Unexpected type: %s",
                             typeName);
                     return new HeapLongVector(batchSize);
@@ -249,7 +251,8 @@ public class ParquetSplitReaderUtil {
                     checkArgument(
                             (typeName == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
                                             || typeName == PrimitiveType.PrimitiveTypeName.BINARY)
-                                    && primitiveType.getOriginalType() == OriginalType.DECIMAL,
+                                    && primitiveType.getLogicalTypeAnnotation()
+                                            instanceof DecimalLogicalTypeAnnotation,
                             "Unexpected type: %s",
                             typeName);
                     return new HeapBytesVector(batchSize);
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/PositionOutputStreamAdapter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/PositionOutputStreamAdapter.java
index 72fda99e4..39e69e255 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/PositionOutputStreamAdapter.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/PositionOutputStreamAdapter.java
@@ -21,8 +21,7 @@ package org.apache.paimon.format.parquet.writer;
 import org.apache.parquet.io.PositionOutputStream;
 
 import java.io.IOException;
-
-import static org.apache.parquet.Preconditions.checkNotNull;
+import java.util.Objects;
 
 /** An adapter to turn Paimon's {@link PositionOutputStream} into a {@link PositionOutputStream}. */
 class PositionOutputStreamAdapter extends PositionOutputStream {
@@ -36,7 +35,7 @@ class PositionOutputStreamAdapter extends PositionOutputStream {
      * @param out The Paimon stream written to.
      */
     PositionOutputStreamAdapter(org.apache.paimon.fs.PositionOutputStream out) {
-        this.out = checkNotNull(out, "out");
+        this.out = Objects.requireNonNull(out, "out should not be null");
     }
 
     @Override