You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2020/03/02 09:06:46 UTC
[flink] branch release-1.10 updated: [FLINK-16360][orc] Flink
STRING data type should map to ORC STRING type (#11277)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new ec4155a [FLINK-16360][orc] Flink STRING data type should map to ORC STRING type (#11277)
ec4155a is described below
commit ec4155a298eaf3012bbbb139082713e86a4e60b4
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Mar 2 17:02:54 2020 +0800
[FLINK-16360][orc] Flink STRING data type should map to ORC STRING type (#11277)
Hive 2.0 ORC not support schema evolution from STRING to VARCHAR.
We need produce STRING in ORC for VarcharType(MAX_LENGHT) in Flink.
---
.../src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java | 7 ++++++-
.../src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java | 3 ++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
index f3f6e91..e23bd72 100644
--- a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
+++ b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
@@ -129,7 +129,12 @@ public class OrcSplitReaderUtil {
case CHAR:
return TypeDescription.createChar().withMaxLength(((CharType) type).getLength());
case VARCHAR:
- return TypeDescription.createVarchar().withMaxLength(((VarCharType) type).getLength());
+ int len = ((VarCharType) type).getLength();
+ if (len == VarCharType.MAX_LENGTH) {
+ return TypeDescription.createString();
+ } else {
+ return TypeDescription.createVarchar().withMaxLength(len);
+ }
case BOOLEAN:
return TypeDescription.createBoolean();
case VARBINARY:
diff --git a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
index d6f1459..7b84f1e 100644
--- a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
+++ b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
@@ -36,6 +36,7 @@ public class OrcSplitReaderUtilTest {
test("boolean", DataTypes.BOOLEAN());
test("char(123)", DataTypes.CHAR(123));
test("varchar(123)", DataTypes.VARCHAR(123));
+ test("string", DataTypes.STRING());
test("binary", DataTypes.BYTES());
test("tinyint", DataTypes.TINYINT());
test("smallint", DataTypes.SMALLINT());
@@ -47,7 +48,7 @@ public class OrcSplitReaderUtilTest {
test("timestamp", DataTypes.TIMESTAMP());
test("array<float>", DataTypes.ARRAY(DataTypes.FLOAT()));
test("map<float,bigint>", DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT()));
- test("struct<int0:int,str1:varchar(2147483647),double2:double,row3:struct<int0:int,int1:int>>",
+ test("struct<int0:int,str1:string,double2:double,row3:struct<int0:int,int1:int>>",
DataTypes.ROW(
DataTypes.FIELD("int0", DataTypes.INT()),
DataTypes.FIELD("str1", DataTypes.STRING()),