You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2021/04/21 14:30:41 UTC
[flink] 01/03: [FLINK-22354][table-planner] Fix the timestamp
function return value precision isn't matching with declared DataType
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch release-1.13
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 74c6a062063e0dd7dbf5f9036fd4c810471da1a5
Author: Leonard Xu <xb...@gmail.com>
AuthorDate: Wed Apr 21 12:22:19 2021 +0800
[FLINK-22354][table-planner] Fix the timestamp function return value precision isn't matching with declared DataType
This closes #15689
---
docs/data/sql_functions.yml | 8 ++--
.../delegation/hive/SqlFunctionConverter.java | 4 +-
.../functions/sql/FlinkSqlOperatorTable.java | 18 ++++---
...unction.java => FlinkSqlTimestampFunction.java} | 27 +++++++++--
.../expressions/NonDeterministicTests.scala | 10 ++--
.../planner/expressions/ScalarFunctionsTest.scala | 56 +++++++++++++---------
.../plan/stream/sql/WindowTableFunctionTest.scala | 2 +-
7 files changed, 79 insertions(+), 46 deletions(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index 30b6eb6..6b7caa15 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -417,10 +417,10 @@ temporal:
description: Creates an interval of NUMERIC milliseconds.
- sql: LOCALTIME
table: localTime()
- description: Returns the current SQL time in the local time zone. It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
+ description: Returns the current SQL time in the local time zone, the return type is TIME(0). It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
- sql: LOCALTIMESTAMP
table: localTimestamp()
- description: Returns the current SQL timestamp in local time zone, the return type is TIMESTAMP WITHOUT ITME ZONE. It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
+ description: Returns the current SQL timestamp in local time zone, the return type is TIMESTAMP(3). It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
- sql: CURRENT_TIME
table: currentTime()
description: Returns the current SQL time in the local time zone, this is a synonym of LOCAL_TIME.
@@ -429,11 +429,11 @@ temporal:
description: Returns the current SQL date in the local time zone. It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
- sql: CURRENT_TIMESTAMP
table: currentTimestamp()
- description: Returns the current SQL timestamp in the local time zone, the return type is TIMESTAMP WITH LOCAL TIME ZONE. It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
+ description: Returns the current SQL timestamp in the local time zone, the return type is TIMESTAMP_LTZ(3). It is evaluated for each record in streaming mode. But in batch mode, it is evaluated once as the query starts and uses the same result for every row.
- sql: NOW()
description: Returns the current SQL timestamp in the local time zone, this is a synonym of CURRENT_TIMESTAMP.
- sql: CURRENT_ROW_TIMESTAMP()
- description: Returns the current SQL timestamp in the local time zone, the return type is TIMESTAMP WITH LOCAL TIME ZONE. It is evaluated for each record no matter in batch or streaming mode.
+ description: Returns the current SQL timestamp in the local time zone, the return type is TIMESTAMP_LTZ(3). It is evaluated for each record no matter in batch or streaming mode.
- sql: EXTRACT(timeinteravlunit FROM temporal)
table: TEMPORAL.extract(TIMEINTERVALUNIT)
description: Returns a long value extracted from the timeintervalunit part of temporal. E.g., EXTRACT(DAY FROM DATE '2006-06-05') returns 5.
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/planner/delegation/hive/SqlFunctionConverter.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/planner/delegation/hive/SqlFunctionConverter.java
index 0f20662..5465ebf 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/planner/delegation/hive/SqlFunctionConverter.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/planner/delegation/hive/SqlFunctionConverter.java
@@ -21,7 +21,7 @@ package org.apache.flink.table.planner.delegation.hive;
import org.apache.flink.connectors.hive.FlinkHiveException;
import org.apache.flink.table.planner.delegation.hive.copy.HiveParserBetween;
import org.apache.flink.table.planner.delegation.hive.copy.HiveParserSqlFunctionConverter;
-import org.apache.flink.table.planner.functions.sql.SqlCurrentTimestampFunction;
+import org.apache.flink.table.planner.functions.sql.FlinkSqlTimestampFunction;
import org.apache.flink.util.Preconditions;
import org.apache.calcite.plan.RelOptCluster;
@@ -90,7 +90,7 @@ public class SqlFunctionConverter extends RexShuttle {
RelDataType type = call.getType();
return builder.makeCall(type, convertedOp, visitList(operands, update));
} else {
- if (convertedOp instanceof SqlCurrentTimestampFunction) {
+ if (convertedOp instanceof FlinkSqlTimestampFunction) {
// flink's current_timestamp has different type from hive's, convert it to a literal
Timestamp currentTS =
((HiveParser.HiveParserSessionState) SessionState.get())
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
index 60c7bb3..5cb3e87 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
@@ -561,19 +561,24 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.ANY),
SqlFunctionCategory.STRING);
+ // Flink timestamp functions
+ public static final SqlFunction LOCALTIMESTAMP =
+ new FlinkSqlTimestampFunction("LOCALTIMESTAMP", SqlTypeName.TIMESTAMP, 3);
+
+ public static final SqlFunction CURRENT_TIMESTAMP =
+ new FlinkSqlTimestampFunction(
+ "CURRENT_TIMESTAMP", SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 3);
+
public static final SqlFunction NOW =
- new SqlCurrentTimestampFunction("NOW") {
+ new FlinkSqlTimestampFunction("NOW", SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 3) {
@Override
public SqlSyntax getSyntax() {
return SqlSyntax.FUNCTION;
}
};
-
- public static final SqlFunction CURRENT_TIMESTAMP =
- new SqlCurrentTimestampFunction("CURRENT_TIMESTAMP");
-
public static final SqlFunction CURRENT_ROW_TIMESTAMP =
- new SqlCurrentTimestampFunction("CURRENT_ROW_TIMESTAMP") {
+ new FlinkSqlTimestampFunction(
+ "CURRENT_ROW_TIMESTAMP", SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 3) {
@Override
public SqlSyntax getSyntax() {
@@ -1103,7 +1108,6 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
public static final SqlFunction FLOOR = SqlStdOperatorTable.FLOOR;
public static final SqlFunction CEIL = SqlStdOperatorTable.CEIL;
public static final SqlFunction LOCALTIME = SqlStdOperatorTable.LOCALTIME;
- public static final SqlFunction LOCALTIMESTAMP = SqlStdOperatorTable.LOCALTIMESTAMP;
public static final SqlFunction CURRENT_TIME = SqlStdOperatorTable.CURRENT_TIME;
public static final SqlFunction CURRENT_DATE = SqlStdOperatorTable.CURRENT_DATE;
public static final SqlFunction CAST = SqlStdOperatorTable.CAST;
diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/SqlCurrentTimestampFunction.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlTimestampFunction.java
similarity index 54%
rename from flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/SqlCurrentTimestampFunction.java
rename to flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlTimestampFunction.java
index 17e552d..f938ee2 100644
--- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/SqlCurrentTimestampFunction.java
+++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlTimestampFunction.java
@@ -18,18 +18,35 @@
package org.apache.flink.table.planner.functions.sql;
+import org.apache.flink.annotation.Internal;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.fun.SqlAbstractTimeFunction;
import org.apache.calcite.sql.type.SqlTypeName;
/**
- * Function that returns current timestamp, the function return type is {@link
- * SqlTypeName#TIMESTAMP_WITH_LOCAL_TIME_ZONE}.
+ * Function that used to define SQL time function like LOCALTIMESTAMP, CURRENT_TIMESTAMP,
+ * CURRENT_ROW_TIMESTAMP(), NOW() in Flink, the function support configuring the return type and the
+ * precision of return type.
*/
-public class SqlCurrentTimestampFunction extends SqlAbstractTimeFunction {
+@Internal
+public class FlinkSqlTimestampFunction extends SqlAbstractTimeFunction {
+
+ private final SqlTypeName returnTypeName;
+ private final int precision;
- public SqlCurrentTimestampFunction(String name) {
+ public FlinkSqlTimestampFunction(
+ String functionName, SqlTypeName returnTypeName, int precision) {
// access protected constructor
- super(name, SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE);
+ super(functionName, returnTypeName);
+ this.returnTypeName = returnTypeName;
+ this.precision = precision;
+ }
+
+ @Override
+ public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+ return opBinding.getTypeFactory().createSqlType(returnTypeName, precision);
}
@Override
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala
index a88135b..ed87e33 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala
@@ -81,9 +81,9 @@ class NonDeterministicTests extends ExpressionTestBase {
config.setLocalTimeZone(zoneId)
config.getConfiguration.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH)
- config.getConfiguration.setLong(InternalConfigOptions.TABLE_QUERY_START_EPOCH_TIME, 1000L)
+ config.getConfiguration.setLong(InternalConfigOptions.TABLE_QUERY_START_EPOCH_TIME, 1123L)
config.getConfiguration.setLong(InternalConfigOptions.TABLE_QUERY_START_LOCAL_TIME,
- 1000L + TimeZone.getTimeZone(zoneId).getOffset(1000L))
+ 1123L + TimeZone.getTimeZone(zoneId).getOffset(1123L))
val temporalFunctions = getCodeGenFunctions(List(
"CURRENT_DATE",
@@ -96,10 +96,10 @@ class NonDeterministicTests extends ExpressionTestBase {
val expected = mutable.MutableList[String](
"1970-01-01",
"08:00:01",
- "1970-01-01 08:00:01",
- "1970-01-01 08:00:01",
+ "1970-01-01 08:00:01.123",
+ "1970-01-01 08:00:01.123",
"08:00:01",
- "1970-01-01 08:00:01")
+ "1970-01-01 08:00:01.123")
val result = evaluateFunctionResult(temporalFunctions)
assertEquals(expected.toList.sorted, result.sorted)
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
index 859c869..e97ce27 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
@@ -3027,39 +3027,51 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
// we just test the format of the output
// manual test can be found in NonDeterministicTests
+ // e.g: 2021-04-19
testAllApis(
- currentDate().cast(DataTypes.STRING).charLength() >= 5,
- "currentDate().cast(STRING).charLength() >= 5",
- "CHAR_LENGTH(CAST(CURRENT_DATE AS VARCHAR)) >= 5",
- "true")
+ currentDate().cast(DataTypes.STRING).charLength(),
+ "currentDate().cast(STRING).charLength()",
+ "CHAR_LENGTH(CAST(CURRENT_DATE AS VARCHAR))",
+ "10")
+ // e.g: 12:13:43
+ // TODO
testAllApis(
- currentTime().cast(DataTypes.STRING).charLength() >= 5,
- "currentTime().cast(STRING).charLength() >= 5",
- "CHAR_LENGTH(CAST(CURRENT_TIME AS VARCHAR)) >= 5",
- "true")
+ localTime().cast(DataTypes.STRING).charLength(),
+ "localTime().cast(STRING).charLength()",
+ "CHAR_LENGTH(CAST(LOCALTIME AS VARCHAR))",
+ "8")
+ // e.g: 12:13:43
testAllApis(
- currentTimestamp().cast(DataTypes.STRING).charLength() >= 12,
- "currentTimestamp().cast(STRING).charLength() >= 12",
- "CHAR_LENGTH(CAST(CURRENT_TIMESTAMP AS VARCHAR)) >= 12",
- "true")
+ currentTime().cast(DataTypes.STRING).charLength(),
+ "currentTime().cast(STRING).charLength()",
+ "CHAR_LENGTH(CAST(CURRENT_TIME AS VARCHAR))",
+ "8")
+ // e.g: 2021-04-19 12:13:43.678
testAllApis(
- localTimestamp().cast(DataTypes.STRING).charLength() >= 12,
- "localTimestamp().cast(STRING).charLength() >= 12",
- "CHAR_LENGTH(CAST(LOCALTIMESTAMP AS VARCHAR)) >= 12",
- "true")
+ localTimestamp().cast(DataTypes.STRING).charLength(),
+ "localTimestamp().cast(STRING).charLength()",
+ "CHAR_LENGTH(CAST(LOCALTIMESTAMP AS VARCHAR))",
+ "23")
+ // e.g: 2021-04-19 12:13:43.678
testAllApis(
- localTime().cast(DataTypes.STRING).charLength() >= 5,
- "localTime().cast(STRING).charLength() >= 5",
- "CHAR_LENGTH(CAST(LOCALTIME AS VARCHAR)) >= 5",
- "true")
+ currentTimestamp().cast(DataTypes.STRING).charLength(),
+ "currentTimestamp().cast(STRING).charLength()",
+ "CHAR_LENGTH(CAST(CURRENT_TIMESTAMP AS VARCHAR))",
+ "23")
+ // e.g: 2021-04-19 12:13:43.678
testSqlApi(
- "CHAR_LENGTH(CAST(NOW() AS VARCHAR)) >= 12",
- "true")
+ "CHAR_LENGTH(CAST(NOW() AS VARCHAR))",
+ "23")
+
+ // e.g: 2021-04-19 12:13:43.678
+ testSqlApi(
+ "CHAR_LENGTH(CAST(CURRENT_ROW_TIMESTAMP() AS VARCHAR))",
+ "23")
// comparisons are deterministic
testAllApis(
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.scala
index 82bcc62..d3f93c8 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/WindowTableFunctionTest.scala
@@ -99,7 +99,7 @@ class WindowTableFunctionTest extends TableTestBase {
|FROM TABLE(
| TUMBLE(TABLE v1, DESCRIPTOR(cur_time), INTERVAL '15' MINUTE))
|""".stripMargin
- thrown.expectMessage("requires the timecol is a time attribute type, but is TIMESTAMP(0)")
+ thrown.expectMessage("requires the timecol is a time attribute type, but is TIMESTAMP(3)")
thrown.expect(classOf[ValidationException])
util.verifyRelPlan(sql)
}