You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2020/06/11 06:52:05 UTC
[flink] 01/02: [FLINK-18030][hive] Hive UDF doesn't accept empty
string literal parameters
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 820b662b94f8362ce829fc237fd413d033f492d1
Author: Rui Li <li...@apache.org>
AuthorDate: Thu Jun 11 14:43:29 2020 +0800
[FLINK-18030][hive] Hive UDF doesn't accept empty string literal parameters
This closes #12403
---
.../table/catalog/hive/util/HiveTypeUtil.java | 32 +++++++++++++++-------
.../catalog/hive/HiveCatalogDataTypeTest.java | 2 --
.../flink/table/module/hive/HiveModuleTest.java | 17 ++++++++++++
3 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTypeUtil.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTypeUtil.java
index 7e7ef81..0678c17 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTypeUtil.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTypeUtil.java
@@ -182,11 +182,17 @@ public class HiveTypeUtil {
@Override
public TypeInfo visit(CharType charType) {
- if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH) {
- throw new CatalogException(
- String.format("HiveCatalog doesn't support char type with length of '%d'. " +
- "The maximum length is %d",
- charType.getLength(), HiveChar.MAX_CHAR_LENGTH));
+ // Flink and Hive have different length limit for CHAR. Promote it to STRING if it exceeds the limits of
+ // Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
+ if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH || charType.getLength() < 1) {
+ if (checkPrecision) {
+ throw new CatalogException(
+ String.format("HiveCatalog doesn't support char type with length of '%d'. " +
+ "The supported length is [%d, %d]",
+ charType.getLength(), 1, HiveChar.MAX_CHAR_LENGTH));
+ } else {
+ return TypeInfoFactory.stringTypeInfo;
+ }
}
return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
@@ -199,11 +205,17 @@ public class HiveTypeUtil {
if (varCharType.getLength() == Integer.MAX_VALUE) {
return TypeInfoFactory.stringTypeInfo;
}
- if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH) {
- throw new CatalogException(
- String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
- "The maximum length is %d",
- varCharType.getLength(), HiveVarchar.MAX_VARCHAR_LENGTH));
+ // Flink and Hive have different length limit for VARCHAR. Promote it to STRING if it exceeds the limits of
+ // Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
+ if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH || varCharType.getLength() < 1) {
+ if (checkPrecision) {
+ throw new CatalogException(
+ String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
+ "The supported length is [%d, %d]",
+ varCharType.getLength(), 1, HiveVarchar.MAX_VARCHAR_LENGTH));
+ } else {
+ return TypeInfoFactory.stringTypeInfo;
+ }
}
return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogDataTypeTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogDataTypeTest.java
index 5208d53..d24380f 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogDataTypeTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogDataTypeTest.java
@@ -154,7 +154,6 @@ public class HiveCatalogDataTypeTest {
};
exception.expect(CatalogException.class);
- exception.expectMessage("HiveCatalog doesn't support char type with length of '256'. The maximum length is 255");
verifyDataTypes(types);
}
@@ -165,7 +164,6 @@ public class HiveCatalogDataTypeTest {
};
exception.expect(CatalogException.class);
- exception.expectMessage("HiveCatalog doesn't support varchar type with length of '65536'. The maximum length is 65535");
verifyDataTypes(types);
}
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/module/hive/HiveModuleTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/module/hive/HiveModuleTest.java
index 374984a..343d818 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/module/hive/HiveModuleTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/module/hive/HiveModuleTest.java
@@ -168,4 +168,21 @@ public class HiveModuleTest {
assertEquals("[{a=1, b=2, c=3}]", results.toString());
}
+
+ @Test
+ public void testEmptyStringLiteralParameters() {
+ TableEnvironment tableEnv = HiveTestUtils.createTableEnvWithBlinkPlannerBatchMode();
+
+ tableEnv.unloadModule("core");
+ tableEnv.loadModule("hive", new HiveModule());
+
+ // UDF
+ List<Row> results = Lists.newArrayList(
+ tableEnv.sqlQuery("select regexp_replace('foobar','oo|ar','')").execute().collect());
+ assertEquals("[fb]", results.toString());
+
+ // GenericUDF
+ results = Lists.newArrayList(tableEnv.sqlQuery("select length('')").execute().collect());
+ assertEquals("[0]", results.toString());
+ }
}