You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2019/08/09 13:02:52 UTC
[flink] 04/04: [FLINK-13547][table-planner-blink] Remove LENGTH(),
JSONVALUE(), KEYVALUE(), SUBSTR() builtin functions which are not standard.
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7e13f5b43f417b8e29e1964aa109a463b152991a
Author: Zhenghua Gao <do...@gmail.com>
AuthorDate: Thu Aug 8 11:40:24 2019 +0800
[FLINK-13547][table-planner-blink] Remove LENGTH(), JSONVALUE(), KEYVALUE(), SUBSTR() builtin functions which are not standard.
LENGTH, SUBSTR, KEYVALUE can be covered by existing functions, e.g. CHAR_LENGTH, SUBSTRING, STR_TO_MAP(str)[key].
---
.../functions/sql/FlinkSqlOperatorTable.java | 79 ++++------------------
.../planner/codegen/calls/StringCallGen.scala | 8 +--
.../planner/expressions/ScalarFunctionsTest.scala | 43 +-----------
.../planner/expressions/SqlExpressionTest.scala | 6 --
.../planner/expressions/TemporalTypesTest.scala | 2 +-
5 files changed, 16 insertions(+), 122 deletions(-)
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
index 04c867b..a1228fc 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
@@ -263,14 +263,6 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
OperandTypes.family(SqlTypeFamily.STRING)),
SqlFunctionCategory.NUMERIC);
- public static final SqlFunction JSONVALUE = new SqlFunction(
- "JSONVALUE",
- SqlKind.OTHER_FUNCTION,
- VARCHAR_2000_NULLABLE,
- null,
- OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER),
- SqlFunctionCategory.STRING);
-
public static final SqlFunction STR_TO_MAP = new SqlFunction(
"STR_TO_MAP",
SqlKind.OTHER_FUNCTION,
@@ -398,18 +390,6 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
OperandTypes.STRING_STRING),
SqlFunctionCategory.STRING);
- public static final SqlFunction KEYVALUE = new SqlFunction(
- "KEYVALUE",
- SqlKind.OTHER_FUNCTION,
- VARCHAR_2000_NULLABLE,
- null,
- OperandTypes.family(
- SqlTypeFamily.STRING,
- SqlTypeFamily.STRING,
- SqlTypeFamily.STRING,
- SqlTypeFamily.STRING),
- SqlFunctionCategory.STRING);
-
public static final SqlFunction HASH_CODE = new SqlFunction(
"HASH_CODE",
SqlKind.OTHER_FUNCTION,
@@ -429,9 +409,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA1 = new SqlFunction(
@@ -439,9 +417,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA224 = new SqlFunction(
@@ -449,9 +425,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA256 = new SqlFunction(
@@ -459,9 +433,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA384 = new SqlFunction(
@@ -469,9 +441,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA512 = new SqlFunction(
@@ -479,9 +449,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)),
+ OperandTypes.family(SqlTypeFamily.STRING),
SqlFunctionCategory.STRING);
public static final SqlFunction SHA2 = new SqlFunction(
@@ -489,9 +457,8 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
VARCHAR_2000_NULLABLE,
null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.INTEGER)),
+ OperandTypes.sequence("'SHA2(DATA, HASH_LENGTH)'",
+ OperandTypes.STRING, OperandTypes.NUMERIC_INTEGER),
SqlFunctionCategory.STRING);
public static final SqlFunction DATE_FORMAT = new SqlFunction(
@@ -625,9 +592,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
SqlKind.OTHER_FUNCTION,
ReturnTypes.VARCHAR_2000,
null,
- OperandTypes.or(
- OperandTypes.NILADIC,
- OperandTypes.ANY),
+ OperandTypes.NILADIC,
SqlFunctionCategory.STRING) {
@Override
@@ -647,19 +612,8 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
ARG0_VARCHAR_FORCE_NULLABLE,
null,
OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)
- ),
- SqlFunctionCategory.STRING);
-
- public static final SqlFunction SUBSTR = new SqlFunction(
- "SUBSTR",
- SqlKind.OTHER_FUNCTION,
- ARG0_VARCHAR_FORCE_NULLABLE,
- null,
- OperandTypes.or(
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER),
- OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)
+ OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER),
+ OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)
),
SqlFunctionCategory.STRING);
@@ -717,21 +671,12 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.INTEGER)),
SqlFunctionCategory.NUMERIC);
- public static final SqlFunction LENGTH = new SqlFunction(
- "LENGTH",
- SqlKind.OTHER_FUNCTION,
- ReturnTypes.INTEGER_NULLABLE,
- null,
- OperandTypes.family(SqlTypeFamily.STRING),
- SqlFunctionCategory.NUMERIC);
-
-
public static final SqlFunction ASCII = new SqlFunction(
"ASCII",
SqlKind.OTHER_FUNCTION,
ReturnTypes.INTEGER_NULLABLE,
null,
- OperandTypes.family(SqlTypeFamily.STRING),
+ OperandTypes.family(SqlTypeFamily.CHARACTER),
SqlFunctionCategory.NUMERIC);
public static final SqlFunction ENCODE = new SqlFunction(
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/StringCallGen.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/StringCallGen.scala
index 7c4a129..0e7c5d1 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/StringCallGen.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/StringCallGen.scala
@@ -63,13 +63,13 @@ object StringCallGen {
case NOT_LIKE =>
generateNot(ctx, new LikeCallGen().generate(ctx, operands, new BooleanType()))
- case SUBSTRING | SUBSTR => generateSubString(ctx, operands)
+ case SUBSTRING => generateSubString(ctx, operands)
case LEFT => generateLeft(ctx, operands.head, operands(1))
case RIGHT => generateRight(ctx, operands.head, operands(1))
- case CHAR_LENGTH | CHARACTER_LENGTH | LENGTH => generateCharLength(ctx, operands)
+ case CHAR_LENGTH | CHARACTER_LENGTH => generateCharLength(ctx, operands)
case SIMILAR_TO => generateSimilarTo(ctx, operands)
@@ -109,8 +109,6 @@ object StringCallGen {
case SPLIT_INDEX => generateSplitIndex(ctx, operands)
- case KEYVALUE => generateKeyValue(ctx, operands)
-
case HASH_CODE if isCharacterString(operands.head.resultType) =>
generateHashCode(ctx, operands)
@@ -138,8 +136,6 @@ object StringCallGen {
case REGEXP => generateRegExp(ctx, operands)
- case JSONVALUE => generateJsonValue(ctx, operands)
-
case BIN => generateBin(ctx, operands)
case CONCAT_FUNCTION =>
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
index 9411a75..921b2ac 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
@@ -518,21 +518,6 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
}
@Test
- def testLength(): Unit = {
- testSqlApi(
- "LENGTH(f0)",
- "22")
-
- testSqlApi(
- "LENGTH(f0)",
- "22")
-
- testSqlApi(
- "length(uuid())",
- "36")
- }
-
- @Test
def testUpperCase(): Unit = {
testAllApis(
'f0.upperCase(),
@@ -953,7 +938,7 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
@Test
def testSubString(): Unit = {
- Array("substring", "substr").foreach {
+ Array("substring").foreach {
substr =>
testSqlApi(s"$substr(f0, 2, 3)", "his")
testSqlApi(s"$substr(f0, 2, 100)", "his is a test String.")
@@ -1160,21 +1145,6 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
}
@Test
- def testKeyValue(): Unit = {
- // NOTE: Spark has str_to_map
- testSqlApi("keyValue('a=1,b=2,c=3', ',', '=', 'a')", "1")
- testSqlApi("keyValue('a=1,b=2,c=3', ',', '=', 'b')", "2")
- testSqlApi("keyValue('a=1,b=2,c=3', ',', '=', 'c')", "3")
- testSqlApi("keyValue('', ',', '=', 'c')", "null")
- testSqlApi("keyValue(f40, ',', '=', 'c')", "null")
- testSqlApi("keyValue(CAST(null as VARCHAR), ',', '=', 'c')", "null")
- testSqlApi("keyValue('a=1,b=2,c=3', ',', '=', 'd')", "null")
- testSqlApi("keyValue('a=1,b=2,c=3', CAST(null as VARCHAR), '=', 'a')", "null")
- testSqlApi("keyValue('a=1,b=2,c=3', ',', CAST(null as VARCHAR), 'a')", "null")
- testSqlApi("keyValue('a=1,b=2,c=3', ',', '=', CAST(null as VARCHAR))", "null")
- }
-
- @Test
def testHashCode(): Unit = {
testSqlApi("hash_code('abc')", "96354")
testSqlApi("hash_code(f35)", "97")
@@ -1202,17 +1172,6 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
}
@Test
- def testJsonValue(): Unit = {
- testSqlApi("jsonValue('[10, 20, [30, 40]]', '$[2][*]')", "[30,40]")
- testSqlApi("jsonValue('[10, 20, [30, [40, 50, 60]]]', '$[2][*][1][*]')", "[30,[40,50,60]]")
- testSqlApi("jsonValue(f40, '$[2][*][1][*]')", "null")
- testSqlApi("jsonValue('[10, 20, [30, [40, 50, 60]]]', '')", "null")
- testSqlApi("jsonValue('', '$[2][*][1][*]')", "null")
- testSqlApi("jsonValue(CAST(null as VARCHAR), '$[2][*][1][*]')", "null")
- testSqlApi("jsonValue('[10, 20, [30, [40, 50, 60]]]', CAST(null as VARCHAR))", "null")
- }
-
- @Test
def testHex(): Unit = {
testAllApis(
100.hex(),
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/SqlExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/SqlExpressionTest.scala
index d31c937..42de09d 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/SqlExpressionTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/SqlExpressionTest.scala
@@ -158,10 +158,6 @@ class SqlExpressionTest extends ExpressionTestBase {
testSqlApi("SUBSTRING('hello world', 2)", "ello world")
testSqlApi("SUBSTRING('hello world', 2, 3)", "ell")
testSqlApi("SUBSTRING('hello world', 2, 300)", "ello world")
- testSqlApi("SUBSTR('hello world', 2, 3)", "ell")
- testSqlApi("SUBSTR('hello world', 2)", "ello world")
- testSqlApi("SUBSTR('hello world', 2, 300)", "ello world")
- testSqlApi("SUBSTR('hello world', 0, 3)", "hel")
testSqlApi("INITCAP('hello world')", "Hello World")
testSqlApi("REGEXP_REPLACE('foobar', 'oo|ar', '')", "fb")
testSqlApi("REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2)", "bar")
@@ -177,8 +173,6 @@ class SqlExpressionTest extends ExpressionTestBase {
testSqlApi("CASE WHEN 1 = 2 THEN 2 WHEN 1 = 1 THEN 3 ELSE 3 END", "3")
testSqlApi("NULLIF(1, 1)", "null")
testSqlApi("COALESCE(NULL, 5)", "5")
- testSqlApi("COALESCE(keyvalue('', ';', ':', 'isB2C'), '5')", "5")
- testSqlApi("COALESCE(jsonvalue('xx', '$x'), '5')", "5")
}
@Test
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
index 27965f9..4e1bb8d 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala
@@ -589,7 +589,7 @@ class TemporalTypesTest extends ExpressionTestBase {
)
testSqlApi(
- "TO_TIMESTAMP(SUBSTR('', 2, -1))",
+ "TO_TIMESTAMP(SUBSTRING('', 2, -1))",
"null"
)