You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/11/27 12:43:39 UTC
(doris) branch branch-2.0 updated: [nereids] fix stats error when using dateTime type filter #27571 (#27577)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 2cbccdfa3c8 [nereids] fix stats error when using dateTime type filter #27571 (#27577)
2cbccdfa3c8 is described below
commit 2cbccdfa3c8060c299a69c3dec0fda4e9383d796
Author: xzj7019 <13...@users.noreply.github.com>
AuthorDate: Mon Nov 27 20:43:31 2023 +0800
[nereids] fix stats error when using dateTime type filter #27571 (#27577)
---
.../apache/doris/nereids/types/DateTimeType.java | 21 ++++++++++
.../apache/doris/nereids/types/DateTimeV2Type.java | 20 +++++++++
.../org/apache/doris/nereids/types/DateType.java | 21 ++++++++++
.../org/apache/doris/nereids/types/DateV2Type.java | 21 ++++++++++
.../doris/nereids/types/coercion/DateLikeType.java | 32 +++++++-------
.../explain/test_datetime_filter_stats0.groovy | 49 ++++++++++++++++++++++
6 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java
index b59e47d3eb2..f57e29eab93 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeType.java
@@ -20,6 +20,10 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
+import java.time.DateTimeException;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+
/**
* Datetime type in Nereids.
*/
@@ -46,4 +50,21 @@ public class DateTimeType extends DateLikeType {
public int width() {
return WIDTH;
}
+
+ @Override
+ public double rangeLength(double high, double low) {
+ if (high == low) {
+ return 0;
+ }
+ if (Double.isInfinite(high) || Double.isInfinite(low)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ try {
+ LocalDateTime to = toLocalDateTime(high);
+ LocalDateTime from = toLocalDateTime(low);
+ return ChronoUnit.SECONDS.between(from, to);
+ } catch (DateTimeException e) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
index 12b8be5959a..77891ed3486 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
@@ -27,6 +27,9 @@ import org.apache.doris.nereids.types.coercion.IntegralType;
import com.google.common.base.Preconditions;
+import java.time.DateTimeException;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
import java.util.Objects;
/**
@@ -127,4 +130,21 @@ public class DateTimeV2Type extends DateLikeType {
public int getScale() {
return scale;
}
+
+ @Override
+ public double rangeLength(double high, double low) {
+ if (high == low) {
+ return 0;
+ }
+ if (Double.isInfinite(high) || Double.isInfinite(low)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ try {
+ LocalDateTime to = toLocalDateTime(high);
+ LocalDateTime from = toLocalDateTime(low);
+ return ChronoUnit.SECONDS.between(from, to);
+ } catch (DateTimeException e) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java
index a33fec9d1c3..a1d16246f6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateType.java
@@ -20,6 +20,10 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
+
/**
* Date type in Nereids.
*/
@@ -41,5 +45,22 @@ public class DateType extends DateLikeType {
public int width() {
return WIDTH;
}
+
+ @Override
+ public double rangeLength(double high, double low) {
+ if (high == low) {
+ return 0;
+ }
+ if (Double.isInfinite(high) || Double.isInfinite(low)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ try {
+ LocalDate to = toLocalDate(high);
+ LocalDate from = toLocalDate(low);
+ return ChronoUnit.DAYS.between(from, to);
+ } catch (DateTimeException e) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java
index 9cf5efdbb6c..0437fb0365a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateV2Type.java
@@ -20,6 +20,10 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
+
/**
* Date type in Nereids.
*/
@@ -41,5 +45,22 @@ public class DateV2Type extends DateLikeType {
public int width() {
return WIDTH;
}
+
+ @Override
+ public double rangeLength(double high, double low) {
+ if (high == low) {
+ return 0;
+ }
+ if (Double.isInfinite(high) || Double.isInfinite(low)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ try {
+ LocalDate to = toLocalDate(high);
+ LocalDate from = toLocalDate(low);
+ return ChronoUnit.DAYS.between(from, to);
+ } catch (DateTimeException e) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
index acbce88e5e2..22ea99f00bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java
@@ -27,31 +27,35 @@ import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
-import java.time.temporal.ChronoUnit;
-import java.util.Calendar;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
/**
* date like type.
*/
public abstract class DateLikeType extends PrimitiveType {
- private Calendar toCalendar(double d) {
- //d = (year * 10000 + month * 100 + day) * 1000000L;
+
+ protected LocalDate toLocalDate(double d) {
+ // d = (year * 10000 + month * 100 + day) * 1000000L;
int date = (int) (d / 1000000);
int day = date % 100;
int month = (date / 100) % 100;
int year = date / 10000;
- Calendar calendar = Calendar.getInstance();
- calendar.set(Calendar.YEAR, year);
- calendar.set(Calendar.MONTH, month);
- calendar.set(Calendar.DAY_OF_MONTH, day);
- return calendar;
+ return LocalDate.of(year, month, day);
}
- @Override
- public double rangeLength(double high, double low) {
- Calendar to = toCalendar(high);
- Calendar from = toCalendar(low);
- return ChronoUnit.DAYS.between(from.toInstant(), to.toInstant());
+ protected LocalDateTime toLocalDateTime(double d) {
+ // d = (year * 10000 + month * 100 + day) * 1000000L + time
+ // time = (hour * 10000 + minute * 100 + second);
+ int date = (int) (d / 1000000);
+ int day = date % 100;
+ int month = (date / 100) % 100;
+ int year = date / 10000;
+ int time = (int) (d % 1000000);
+ int second = time % 100;
+ int minute = (time / 100) % 100;
+ int hour = time / 10000;
+ return LocalDateTime.of(year, month, day, hour, minute, second);
}
/**
diff --git a/regression-test/suites/nereids_p0/explain/test_datetime_filter_stats0.groovy b/regression-test/suites/nereids_p0/explain/test_datetime_filter_stats0.groovy
new file mode 100644
index 00000000000..317645e89d9
--- /dev/null
+++ b/regression-test/suites/nereids_p0/explain/test_datetime_filter_stats0.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_datetime_filter_stats0") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ sql "DROP TABLE IF EXISTS test_datetime_filter_stats0"
+ sql """ CREATE TABLE `test_datetime_filter_stats0` (
+ `id` int(11),
+ `is_delete` int,
+ `company_id` int,
+ `book_time` DATETIMEV2
+ )ENGINE=OLAP
+ unique key (id)
+ distributed by hash(id) buckets 10
+ properties(
+ "replication_allocation" = "tag.location.default: 1"
+ );"""
+
+ sql """ alter table test_datetime_filter_stats0 modify column id set stats('row_count'='52899687', 'ndv'='52899687', 'num_nulls'='0', 'min_value'='1', 'max_value'='52899687', 'data_size'='4'); """
+ sql """ alter table test_datetime_filter_stats0 modify column book_time set stats('row_count'='52899687', 'ndv'='23622730', 'num_nulls'='0', 'min_value'='2002-01-01 00:45:39', 'max_value'='2027-09-25 23:03:00', 'data_size'='10'); """
+ sql """ alter table test_datetime_filter_stats0 modify column is_delete set stats('row_count'='52899687', 'ndv'='2', 'num_nulls'='0', 'min_value'='0', 'max_value'='1', 'data_size'='4'); """
+ sql """ alter table test_datetime_filter_stats0 modify column company_id set stats('row_count'='52899687', 'ndv'='7559', 'num_nulls'='0', 'min_value'='2', 'max_value'='876981', 'data_size'='4'); """
+
+ explain {
+ sql("physical plan select count(1) from test_datetime_filter_stats0 o where o.book_time >= '2020-03-01 00:00:00.0' and o.book_time <= '2020-03-01 23:59:59.0';");
+ notContains"stats=2.24"
+ }
+
+ explain {
+ sql("physical plan select count(1) from test_datetime_filter_stats0 o where o.book_time >= '2020-03-01 00:00:00.0' and o.book_time <= '2020-03-01 00:00:01.0';");
+ notContains"stats=2.24"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org