You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/01/14 10:36:44 UTC
[doris] branch master updated: [Bug](datev2) Fix bugs for datev2 (#15860)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0a57f12578 [Bug](datev2) Fix bugs for datev2 (#15860)
0a57f12578 is described below
commit 0a57f1257897dd06f67561e332af1e40f3859391
Author: Gabriel <ga...@gmail.com>
AuthorDate: Sat Jan 14 18:36:36 2023 +0800
[Bug](datev2) Fix bugs for datev2 (#15860)
These bugs are found when I run regression test with enable_date_conversion on
---
.../org/apache/doris/analysis/DateLiteral.java | 11 ++++++
.../apache/doris/analysis/FunctionCallExpr.java | 2 +-
.../org/apache/doris/analysis/StringLiteral.java | 2 +-
.../main/java/org/apache/doris/catalog/Type.java | 7 ++--
.../analysis/CreateTableAsSelectStmtTest.java | 3 +-
.../org/apache/doris/analysis/SelectStmtTest.java | 4 +--
.../apache/doris/nereids/types/DataTypeTest.java | 9 +++--
.../org/apache/doris/planner/QueryPlanTest.java | 40 +++++++++++++---------
.../doris/rewrite/RewriteDateLiteralRuleTest.java | 13 ++-----
gensrc/script/doris_builtins_functions.py | 4 +--
10 files changed, 55 insertions(+), 40 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
index 229769e6f2..82e80a766f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -1647,6 +1647,17 @@ public class DateLiteral extends LiteralExpr {
type = ScalarType.getDefaultDateType(Type.DATE);
} else {
type = ScalarType.getDefaultDateType(Type.DATETIME);
+ if (type.isDatetimeV2() && microsecond != 0) {
+ int scale = 6;
+ for (int i = 0; i < 6; i++) {
+ if (microsecond % Math.pow(10.0, i + 1) > 0) {
+ break;
+ } else {
+ scale -= 1;
+ }
+ }
+ type = ScalarType.createDatetimeV2Type(scale);
+ }
}
if (checkRange() || checkDate()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 5aeb1daf47..6ac5e05b4e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1329,7 +1329,7 @@ public class FunctionCallExpr extends Expr {
&& children.get(0).getType().isDecimalV3() && args[ix].isDecimalV3()) {
continue;
} else if (!argTypes[i].matchesType(args[ix]) && !(
- argTypes[i].isDateType() && args[ix].isDateType())
+ argTypes[i].isDateOrDateTime() && args[ix].isDateOrDateTime())
&& (!fn.getReturnType().isDecimalV3()
|| (argTypes[i].isValid() && !argTypes[i].isDecimalV3() && args[ix].isDecimalV3()))) {
uncheckedCastChild(args[ix], i);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
index 9fa1690e44..bd8867b810 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
@@ -182,7 +182,7 @@ public class StringLiteral extends LiteralExpr {
public LiteralExpr convertToDate(Type targetType) throws AnalysisException {
LiteralExpr newLiteral = null;
try {
- newLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(targetType));
+ newLiteral = new DateLiteral(value, targetType);
} catch (AnalysisException e) {
if (targetType.isScalarType(PrimitiveType.DATETIME)) {
newLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATE));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index d80aa4fc22..a07023a28f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -368,6 +368,10 @@ public abstract class Type {
|| isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2);
}
+ public boolean isDateOrDateTime() {
+ return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME);
+ }
+
public boolean isDatetime() {
return isScalarType(PrimitiveType.DATETIME);
}
@@ -1673,6 +1677,7 @@ public abstract class Type {
case DATE:
case DATEV2:
case DATETIME:
+ case DATETIMEV2:
return Type.BIGINT;
case LARGEINT:
return Type.LARGEINT;
@@ -1684,8 +1689,6 @@ public abstract class Type {
case STRING:
case HLL:
return Type.DOUBLE;
- case DATETIMEV2:
- return Type.DEFAULT_DATETIMEV2;
case TIMEV2:
return Type.DEFAULT_TIMEV2;
case DECIMALV2:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
index 2a6bc8e8c6..58fcff2c2a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
@@ -426,7 +426,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService {
ShowResultSet showResultSet = showCreateTableByName("test_default_timestamp");
Assertions.assertEquals("CREATE TABLE `test_default_timestamp` (\n"
+ " `userId` varchar(65533) NOT NULL,\n"
- + " `date` datetime NULL DEFAULT CURRENT_TIMESTAMP\n"
+ + " `date` " + (Config.enable_date_conversion ? "datetimev2(0)" : "datetime")
+ + " NULL DEFAULT CURRENT_TIMESTAMP\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n"
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
index 7e0137e6bd..ed4969d374 100755
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
@@ -494,12 +494,12 @@ public class SelectStmtTest {
Assert.assertTrue(dorisAssert
.query(sql3)
.explainQuery()
- .contains("`dt` = '2020-09-08 00:00:00'"));
+ .contains(Config.enable_date_conversion ? "`dt` = '2020-09-08'" : "`dt` = '2020-09-08 00:00:00'"));
String sql4 = "select count() from db1.date_partition_table where dt='2020-09-08'";
Assert.assertTrue(dorisAssert
.query(sql4)
.explainQuery()
- .contains("`dt` = '2020-09-08 00:00:00'"));
+ .contains(Config.enable_date_conversion ? "`dt` = '2020-09-08'" : "`dt` = '2020-09-08 00:00:00'"));
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java
index 21e9c89280..aa63e09f03 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.types;
+import org.apache.doris.common.Config;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -88,13 +90,16 @@ public class DataTypeTest {
Assertions.assertEquals(NullType.INSTANCE, DataType.convertFromString("null"));
Assertions.assertEquals(NullType.INSTANCE, DataType.convertFromString("null_type"));
// date
- Assertions.assertEquals(DateType.INSTANCE, DataType.convertFromString("date"));
+ Assertions.assertEquals(Config.enable_date_conversion ? DateV2Type.INSTANCE : DateType.INSTANCE,
+ DataType.convertFromString("date"));
// datev2
Assertions.assertEquals(DateV2Type.INSTANCE, DataType.convertFromString("datev2"));
// time
Assertions.assertEquals(TimeType.INSTANCE, DataType.convertFromString("time"));
// datetime
- Assertions.assertEquals(DateTimeType.INSTANCE, DataType.convertFromString("datetime"));
+ Assertions.assertEquals(Config.enable_date_conversion ? DateTimeV2Type.of(0) : DateTimeType.INSTANCE,
+ Config.enable_date_conversion ? DataType.convertFromString("datetimev2(0)")
+ : DataType.convertFromString("datetime"));
// datetimev2
Assertions.assertEquals(DateTimeV2Type.of(3), DataType.convertFromString("datetimev2(3)"));
// hll
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index 0b29176801..82ae56ca90 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -1604,11 +1604,14 @@ public class QueryPlanTest extends TestWithFeService {
//valid date
String sql = "select day from tbl_int_date where day in ('2020-10-30')";
String explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `day` IN ('2020-10-30 00:00:00')"));
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion ? "PREDICATES: `day` IN ('2020-10-30')"
+ : "PREDICATES: `day` IN ('2020-10-30 00:00:00')"));
//valid date
sql = "select day from tbl_int_date where day in ('2020-10-30','2020-10-29')";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `day` IN ('2020-10-30 00:00:00', '2020-10-29 00:00:00')"));
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion
+ ? "PREDICATES: `day` IN ('2020-10-30', '2020-10-29')"
+ : "PREDICATES: `day` IN ('2020-10-30 00:00:00', '2020-10-29 00:00:00')"));
//valid datetime
sql = "select day from tbl_int_date where date in ('2020-10-30 12:12:30')";
@@ -1678,7 +1681,8 @@ public class QueryPlanTest extends TestWithFeService {
//valid date
String sql = "select day from tbl_int_date where day = '2020-10-30'";
String explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'"));
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion ? "PREDICATES: `day` = '2020-10-30'"
+ : "PREDICATES: `day` = '2020-10-30 00:00:00'"));
sql = "select day from tbl_int_date where day = from_unixtime(1196440219)";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2007-12-01 00:30:19'"));
@@ -1688,19 +1692,19 @@ public class QueryPlanTest extends TestWithFeService {
//valid date
sql = "select day from tbl_int_date where day = 20201030";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'"));
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion ? "PREDICATES: `day` = '2020-10-30'"
+ : "PREDICATES: `day` = '2020-10-30 00:00:00'"));
//valid date
sql = "select day from tbl_int_date where day = '20201030'";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 00:00:00'"));
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion ? "PREDICATES: `day` = '2020-10-30'"
+ : "PREDICATES: `day` = '2020-10-30 00:00:00'"));
//valid date contains micro second
sql = "select day from tbl_int_date where day = '2020-10-30 10:00:01.111111'";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- if (Config.enable_date_conversion) {
- Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 10:00:01.111111'"));
- } else {
- Assert.assertTrue(explainString.contains("PREDICATES: `day` = '2020-10-30 10:00:01'"));
- }
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion
+ ? "PREDICATES: `day` = '2020-10-30 10:00:01.111111'"
+ : "PREDICATES: `day` = '2020-10-30 10:00:01'"));
//invalid date
sql = "select day from tbl_int_date where day = '2020-10-32'";
@@ -1754,11 +1758,8 @@ public class QueryPlanTest extends TestWithFeService {
//valid datetime contains micro second
sql = "select day from tbl_int_date where date = '2020-10-30 10:00:01.111111'";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- if (Config.enable_date_conversion) {
- Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 10:00:01.111111'"));
- } else {
- Assert.assertTrue(explainString.contains("PREDICATES: `date` = '2020-10-30 10:00:01'"));
- }
+ Assert.assertTrue(explainString.contains(Config.enable_date_conversion
+ ? "VEMPTYSET" : "PREDICATES: `date` = '2020-10-30 10:00:01'"));
//invalid datetime
sql = "select day from tbl_int_date where date = '2020-10-32'";
explainString = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
@@ -1890,8 +1891,13 @@ public class QueryPlanTest extends TestWithFeService {
+ " \"line_delimiter\" = \"\\n\","
+ " \"max_file_size\" = \"500MB\" );";
String explainStr = getSQLPlanOrErrorMsg("EXPLAIN " + sql);
- Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00',"
- + " `date` <= '2021-10-11 00:00:00'"));
+ if (Config.enable_date_conversion) {
+ Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07',"
+ + " `date` <= '2021-10-11'"));
+ } else {
+ Assert.assertTrue(explainStr.contains("PREDICATES: `date` >= '2021-10-07 00:00:00',"
+ + " `date` <= '2021-10-11 00:00:00'"));
+ }
}
// Fix: issue-#7929
diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
index 6306073e75..40bc9bb2c3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
@@ -18,7 +18,6 @@
package org.apache.doris.rewrite;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.DorisAssert;
@@ -77,11 +76,7 @@ public class RewriteDateLiteralRuleTest {
public void testWithStringFormatDate() throws Exception {
String query = "select * from " + DB_NAME + ".tb1 where k1 > '2021030112334455'";
String planString = dorisAssert.query(query).explainQuery();
- if (Config.enable_date_conversion) {
- Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44.550000'"));
- } else {
- Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44'"));
- }
+ Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44'"));
query = "select k1 > '20210301' from " + DB_NAME + ".tb1";
planString = dorisAssert.query(query).explainQuery();
@@ -89,11 +84,7 @@ public class RewriteDateLiteralRuleTest {
query = "select k1 > '20210301233234.34' from " + DB_NAME + ".tb1";
planString = dorisAssert.query(query).explainQuery();
- if (Config.enable_date_conversion) {
- Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34.340000'"));
- } else {
- Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34'"));
- }
+ Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34'"));
query = "select * from " + DB_NAME + ".tb1 where k1 > '2021-03-01'";
planString = dorisAssert.query(query).explainQuery();
diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
index 9b231881de..18380f8320 100755
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -2290,10 +2290,8 @@ visible_functions = [
[['ifnull', 'nvl'], 'DATETIME', ['DATETIME', 'DATETIME'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DATETIME', ['DATE', 'DATETIME'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DATETIME', ['DATETIME', 'DATE'], '', '', '', 'vec', 'CUSTOM'],
- [['ifnull', 'nvl'], 'DATEV2', ['DATE', 'DATE'], '', '', '', 'vec', 'CUSTOM'],
+ [['ifnull', 'nvl'], 'DATEV2', ['DATEV2', 'DATEV2'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DATETIMEV2', ['DATETIMEV2', 'DATETIMEV2'], '', '', '', 'vec', 'CUSTOM'],
- [['ifnull', 'nvl'], 'DATETIMEV2', ['DATEV2', 'DATETIMEV2'], '', '', '', 'vec', 'CUSTOM'],
- [['ifnull', 'nvl'], 'DATETIMEV2', ['DATETIMEV2', 'DATEV2'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DECIMAL32', ['DECIMAL32', 'DECIMAL32'], '', '', '', 'vec', 'CUSTOM'],
[['ifnull', 'nvl'], 'DECIMAL64', ['DECIMAL64', 'DECIMAL64'], '', '', '', 'vec', 'CUSTOM'],
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org