You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2020/03/02 09:06:46 UTC

[flink] branch release-1.10 updated: [FLINK-16360][orc] Flink STRING data type should map to ORC STRING type (#11277)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.10 by this push:
     new ec4155a  [FLINK-16360][orc] Flink STRING data type should map to ORC STRING type (#11277)
ec4155a is described below

commit ec4155a298eaf3012bbbb139082713e86a4e60b4
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Mar 2 17:02:54 2020 +0800

    [FLINK-16360][orc] Flink STRING data type should map to ORC STRING type (#11277)
    
    Hive 2.0 ORC not support schema evolution from STRING to VARCHAR.
    We need produce STRING in ORC for VarcharType(MAX_LENGHT) in Flink.
---
 .../src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java     | 7 ++++++-
 .../src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java | 3 ++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
index f3f6e91..e23bd72 100644
--- a/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
+++ b/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java
@@ -129,7 +129,12 @@ public class OrcSplitReaderUtil {
 			case CHAR:
 				return TypeDescription.createChar().withMaxLength(((CharType) type).getLength());
 			case VARCHAR:
-				return TypeDescription.createVarchar().withMaxLength(((VarCharType) type).getLength());
+				int len = ((VarCharType) type).getLength();
+				if (len == VarCharType.MAX_LENGTH) {
+					return TypeDescription.createString();
+				} else {
+					return TypeDescription.createVarchar().withMaxLength(len);
+				}
 			case BOOLEAN:
 				return TypeDescription.createBoolean();
 			case VARBINARY:
diff --git a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
index d6f1459..7b84f1e 100644
--- a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
+++ b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcSplitReaderUtilTest.java
@@ -36,6 +36,7 @@ public class OrcSplitReaderUtilTest {
 		test("boolean", DataTypes.BOOLEAN());
 		test("char(123)", DataTypes.CHAR(123));
 		test("varchar(123)", DataTypes.VARCHAR(123));
+		test("string", DataTypes.STRING());
 		test("binary", DataTypes.BYTES());
 		test("tinyint", DataTypes.TINYINT());
 		test("smallint", DataTypes.SMALLINT());
@@ -47,7 +48,7 @@ public class OrcSplitReaderUtilTest {
 		test("timestamp", DataTypes.TIMESTAMP());
 		test("array<float>", DataTypes.ARRAY(DataTypes.FLOAT()));
 		test("map<float,bigint>", DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT()));
-		test("struct<int0:int,str1:varchar(2147483647),double2:double,row3:struct<int0:int,int1:int>>",
+		test("struct<int0:int,str1:string,double2:double,row3:struct<int0:int,int1:int>>",
 				DataTypes.ROW(
 						DataTypes.FIELD("int0", DataTypes.INT()),
 						DataTypes.FIELD("str1", DataTypes.STRING()),