You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/07 08:57:22 UTC
[incubator-doris] branch dev-1.0.1 updated (a4252467c4 -> c975be38b2)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
from a4252467c4 [fix] fix grammar of ADMIN SHOW TABLET STORAGE FORMAT stmt (#9938)
new 5d3f07feb0 [hotfix] fix bloomfilter disable threshold
new ed04506425 [Bug] Fix timestamp_diff issue when timeunit is year and month (#9574)
new 9dd6d3263f [improvement] set default jvm memory to 8G for FE (#9683)
new 5a1d1faba9 [fix] group by with two NULL rows after left join (#9688)
new 8ff7994913 [fix](function) fix bug in time_round function (#9712)
new c975be38b2 [feature](priv) Support grant node_priv to other user. (#9951)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.h | 4 -
be/src/exprs/timestamp_functions.cpp | 55 ++++++---
be/src/olap/bloom_filter_predicate.h | 16 ---
be/src/runtime/datetime_value.h | 14 ++-
be/src/vec/columns/column_nullable.cpp | 5 +-
.../vec/functions/function_datetime_floor_ceil.cpp | 63 ++++++++---
be/src/vec/runtime/vdatetime_value.h | 14 ++-
conf/fe.conf | 4 +-
docs/en/administrator-guide/privilege.md | 10 +-
.../java/org/apache/doris/analysis/GrantStmt.java | 22 ++--
.../org/apache/doris/mysql/privilege/AuthTest.java | 124 ++++++++++++++++++++-
.../data/correctness/test_time_round.out | 85 ++++++++++++++
.../suites/correctness/test_time_round.groovy | 70 ++++++++++++
.../query/aggregate/aggregate_groupby_null.groovy | 58 ++++++++++
14 files changed, 469 insertions(+), 75 deletions(-)
create mode 100644 regression-test/data/correctness/test_time_round.out
create mode 100644 regression-test/suites/correctness/test_time_round.groovy
create mode 100644 regression-test/suites/query/aggregate/aggregate_groupby_null.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/06: [improvement] set default jvm memory to 8G for FE (#9683)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 9dd6d3263fbd1d2191d55300e2234562dbbf8d16
Author: jiafeng.zhang <zh...@gmail.com>
AuthorDate: Mon Jun 6 08:59:20 2022 +0800
[improvement] set default jvm memory to 8G for FE (#9683)
---
conf/fe.conf | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/conf/fe.conf b/conf/fe.conf
index 723221de86..c91313b5dc 100644
--- a/conf/fe.conf
+++ b/conf/fe.conf
@@ -25,10 +25,10 @@
LOG_DIR = ${DORIS_HOME}/log
DATE = `date +%Y%m%d-%H%M%S`
-JAVA_OPTS="-Xmx4096m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
+JAVA_OPTS="-Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
+JAVA_OPTS_FOR_JDK_9="-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
##
## the lowercase properties are read by main program.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/06: [hotfix] fix bloomfilter disable threshold
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 5d3f07feb0be5d1b136242242b9da3485e799233
Author: morningman <mo...@163.com>
AuthorDate: Tue Jun 7 09:49:50 2022 +0800
[hotfix] fix bloomfilter disable threshold
This issue is introduced from #9792, that config may cause some query slow
---
be/src/common/config.h | 4 ----
be/src/olap/bloom_filter_predicate.h | 16 ----------------
2 files changed, 20 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 90faf93006..48582beabe 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -718,10 +718,6 @@ CONF_Int32(object_pool_buffer_size, "100");
// ParquetReaderWrap prefetch buffer size
CONF_Int32(parquet_reader_max_buffer_size, "50");
-// When the rows number reached this limit, will check the filter rate the of bloomfilter
-// if it is lower than a specific threshold, the predicate will be disabled.
-CONF_mInt32(bloom_filter_predicate_check_row_num, "1000");
-
} // namespace config
} // namespace doris
diff --git a/be/src/olap/bloom_filter_predicate.h b/be/src/olap/bloom_filter_predicate.h
index a7671f2724..f7ba4653f6 100644
--- a/be/src/olap/bloom_filter_predicate.h
+++ b/be/src/olap/bloom_filter_predicate.h
@@ -71,9 +71,6 @@ public:
private:
std::shared_ptr<IBloomFilterFuncBase> _filter;
SpecificFilter* _specific_filter; // owned by _filter
- mutable uint64_t _evaluated_rows = 1;
- mutable uint64_t _passed_rows = 0;
- mutable bool _enable_pred = true;
};
// bloom filter column predicate do not support in segment v1
@@ -116,9 +113,6 @@ void BloomFilterColumnPredicate<T>::evaluate(vectorized::IColumn& column, uint16
uint16_t* size) const {
uint16_t new_size = 0;
using FT = typename PredicatePrimitiveTypeTraits<T>::PredicateFieldType;
- if (!_enable_pred) {
- return;
- }
if (column.is_nullable()) {
auto* nullable_col = vectorized::check_and_get_column<vectorized::ColumnNullable>(column);
auto& null_map_data = nullable_col->get_null_map_column().get_data();
@@ -163,16 +157,6 @@ void BloomFilterColumnPredicate<T>::evaluate(vectorized::IColumn& column, uint16
new_size += _specific_filter->find_olap_engine(cell_value);
}
}
- // If the pass rate is very high, for example > 50%, then the bloomfilter is useless.
- // Some bloomfilter is useless, for example ssb 4.3, it consumes a lot of cpu but it is
- // useless.
- _evaluated_rows += *size;
- _passed_rows += new_size;
- if (_evaluated_rows > config::bloom_filter_predicate_check_row_num) {
- if (_passed_rows / (_evaluated_rows * 1.0) > 0.5) {
- _enable_pred = false;
- }
- }
*size = new_size;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 05/06: [fix](function) fix bug in time_round function (#9712)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 8ff79949131728ed54ae76c91e8c2cd665e87b36
Author: zhengshengjun <zh...@apache.org>
AuthorDate: Mon Jun 6 08:58:22 2022 +0800
[fix](function) fix bug in time_round function (#9712)
---
be/src/exprs/timestamp_functions.cpp | 55 ++++++++++----
.../vec/functions/function_datetime_floor_ceil.cpp | 63 +++++++++++-----
.../data/correctness/test_time_round.out | 85 ++++++++++++++++++++++
.../suites/correctness/test_time_round.groovy | 70 ++++++++++++++++++
4 files changed, 242 insertions(+), 31 deletions(-)
diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp
index 44bc34bccb..ec968583e5 100644
--- a/be/src/exprs/timestamp_functions.cpp
+++ b/be/src/exprs/timestamp_functions.cpp
@@ -581,47 +581,74 @@ DateTimeVal TimestampFunctions::time_round(FunctionContext* ctx, const DateTimeV
DateTimeValue ts1 = DateTimeValue::from_datetime_val(origin);
DateTimeValue ts2 = DateTimeValue::from_datetime_val(ts_val);
int64_t diff;
+ int64_t trivial_part_ts1;
+ int64_t trivial_part_ts2;
switch (unit) {
case YEAR: {
- int year = (ts2.year() - ts1.year());
- diff = year - (ts2.to_int64() % 10000000000 < ts1.to_int64() % 10000000000);
+ diff = (ts2.year() - ts1.year());
+ trivial_part_ts2 = ts2.to_int64() % 10000000000;
+ trivial_part_ts1 = ts1.to_int64() % 10000000000;
break;
}
case MONTH: {
- int month = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
- diff = month - (ts2.to_int64() % 100000000 < ts1.to_int64() % 100000000);
+ diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
+ trivial_part_ts2 = ts2.to_int64() % 100000000;
+ trivial_part_ts1 = ts1.to_int64() % 100000000;
break;
}
case WEEK: {
- int week = ts2.daynr() / 7 - ts1.daynr() / 7;
- diff = week - (ts2.daynr() % 7 < ts1.daynr() % 7 + (ts2.time_part_diff(ts1) < 0));
+ diff = ts2.daynr() / 7 - ts1.daynr() / 7;
+ trivial_part_ts2 =
+ ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second();
+ trivial_part_ts1 =
+ ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second();
break;
}
case DAY: {
- int day = ts2.daynr() - ts1.daynr();
- diff = day - (ts2.time_part_diff(ts1) < 0);
+ diff = ts2.daynr() - ts1.daynr();
+ trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second();
+ trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second();
break;
}
case HOUR: {
- int hour = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour());
- diff = hour - ((ts2.minute() * 60 + ts2.second()) < (ts1.minute() * 60 - ts1.second()));
+ diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour());
+ trivial_part_ts2 = ts2.minute() * 60 + ts2.second();
+ trivial_part_ts1 = ts1.minute() * 60 + ts1.second();
break;
}
case MINUTE: {
- int minute = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 +
- (ts2.minute() - ts1.minute());
- diff = minute - (ts2.second() < ts1.second());
+ diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 +
+ (ts2.minute() - ts1.minute());
+ trivial_part_ts2 = ts2.second();
+ trivial_part_ts1 = ts1.second();
break;
}
case SECOND: {
diff = ts2.second_diff(ts1);
+ trivial_part_ts1 = 0;
+ trivial_part_ts2 = 0;
break;
}
default:
return DateTimeVal::null();
}
+
+ //round down/up to specific time-unit(HOUR/DAY/MONTH...) by increase/decrease diff variable
+ if (type == CEIL) {
+ //e.g. hour_ceil(ts: 00:00:40, origin: 00:00:30), ts should be rounded to 01:00:30
+ diff += trivial_part_ts2 > trivial_part_ts1;
+ } else if (type == FLOOR) {
+ //e.g. hour_floor(ts: 01:00:20, origin: 00:00:30), ts should be rounded to 00:00:30
+ diff -= trivial_part_ts2 < trivial_part_ts1;
+ }
+
+ //round down/up inside time period(several time-units)
int64_t count = period.val;
- int64_t step = diff - (diff % count + count) % count + (type == FLOOR ? 0 : count);
+ int64_t delta_inside_period = (diff % count + count) % count;
+ int64_t step = diff - delta_inside_period +
+ (type == FLOOR ? 0
+ : delta_inside_period == 0 ? 0
+ : count);
bool is_neg = step < 0;
TimeInterval interval(unit, is_neg ? -step : step, is_neg);
diff --git a/be/src/vec/functions/function_datetime_floor_ceil.cpp b/be/src/vec/functions/function_datetime_floor_ceil.cpp
index 79afa52f37..058af52a12 100644
--- a/be/src/vec/functions/function_datetime_floor_ceil.cpp
+++ b/be/src/vec/functions/function_datetime_floor_ceil.cpp
@@ -150,6 +150,8 @@ struct TimeRound {
static constexpr auto name = Impl::name;
static constexpr uint64_t FIRST_DAY = 19700101000000;
static constexpr uint64_t FIRST_SUNDAY = 19700104000000;
+ static constexpr int8_t FLOOR = 0;
+ static constexpr int8_t CEIL = 1;
static void time_round(const doris::vectorized::VecDateTimeValue& ts2, Int32 period,
doris::vectorized::VecDateTimeValue& ts1, UInt8& is_null) {
@@ -159,43 +161,70 @@ struct TimeRound {
}
int64_t diff;
+ int64_t trivial_part_ts1;
+ int64_t trivial_part_ts2;
if constexpr (Impl::Unit == YEAR) {
- int year = (ts2.year() - ts1.year());
- diff = year - (ts2.to_int64() % 10000000000 < ts1.to_int64() % 10000000000);
+ diff = (ts2.year() - ts1.year());
+ trivial_part_ts2 = ts2.to_int64() % 10000000000;
+ trivial_part_ts1 = ts1.to_int64() % 10000000000;
}
if constexpr (Impl::Unit == MONTH) {
- int month = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
- diff = month - (ts2.to_int64() % 100000000 < ts1.to_int64() % 100000000);
+ diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
+ trivial_part_ts2 = ts2.to_int64() % 100000000;
+ trivial_part_ts1 = ts1.to_int64() % 100000000;
}
if constexpr (Impl::Unit == MONTH) {
- int month = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
- diff = month - (ts2.to_int64() % 100000000 < ts1.to_int64() % 100000000);
+ diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month());
+ trivial_part_ts2 = ts2.to_int64() % 100000000;
+ trivial_part_ts1 = ts1.to_int64() % 100000000;
}
if constexpr (Impl::Unit == WEEK) {
- int week = ts2.daynr() / 7 - ts1.daynr() / 7;
- diff = week - (ts2.daynr() % 7 < ts1.daynr() % 7 + (ts2.time_part_diff(ts1) < 0));
+ diff = ts2.daynr() / 7 - ts1.daynr() / 7;
+ trivial_part_ts2 = ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + ts2.minute() * 60 +
+ ts2.second();
+ trivial_part_ts1 = ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + ts1.minute() * 60 +
+ ts1.second();
}
if constexpr (Impl::Unit == DAY) {
- int day = ts2.daynr() - ts1.daynr();
- diff = day - (ts2.time_part_diff(ts1) < 0);
+ diff = ts2.daynr() - ts1.daynr();
+ trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second();
+ trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second();
}
if constexpr (Impl::Unit == HOUR) {
- int hour = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour());
- diff = hour - ((ts2.minute() * 60 + ts2.second()) < (ts1.minute() * 60 - ts1.second()));
+ diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour());
+ trivial_part_ts2 = ts2.minute() * 60 + ts2.second();
+ trivial_part_ts1 = ts1.minute() * 60 + ts1.second();
}
if constexpr (Impl::Unit == MINUTE) {
- int minute = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 +
- (ts2.minute() - ts1.minute());
- diff = minute - (ts2.second() < ts1.second());
+ diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 +
+ (ts2.minute() - ts1.minute());
+ trivial_part_ts2 = ts2.second();
+ trivial_part_ts1 = ts1.second();
}
if constexpr (Impl::Unit == SECOND) {
diff = ts2.second_diff(ts1);
+ trivial_part_ts1 = 0;
+ trivial_part_ts2 = 0;
}
+ //round down/up to specific time-unit(HOUR/DAY/MONTH...) by increase/decrease diff variable
+ if constexpr (Impl::Type == CEIL) {
+ //e.g. hour_ceil(ts: 00:00:40, origin: 00:00:30), ts should be rounded to 01:00:30
+ diff += trivial_part_ts2 > trivial_part_ts1;
+ }
+ if constexpr (Impl::Type == FLOOR) {
+ //e.g. hour_floor(ts: 01:00:20, origin: 00:00:30), ts should be rounded to 00:00:30
+ diff -= trivial_part_ts2 < trivial_part_ts1;
+ }
+
+ //round down/up inside time period(several time-units)
int64_t count = period;
- int64_t step = diff - (diff % count + count) % count + (Impl::Type == 0 ? 0 : count);
+ int64_t delta_inside_period = (diff % count + count) % count;
+ int64_t step = diff - delta_inside_period +
+ (Impl::Type == FLOOR ? 0
+ : delta_inside_period == 0 ? 0
+ : count);
bool is_neg = step < 0;
-
TimeInterval interval(Impl::Unit, is_neg ? -step : step, is_neg);
is_null = !ts1.date_add_interval(interval, Impl::Unit);
return;
diff --git a/regression-test/data/correctness/test_time_round.out b/regression-test/data/correctness/test_time_round.out
new file mode 100644
index 0000000000..a3d0aa9ebd
--- /dev/null
+++ b/regression-test/data/correctness/test_time_round.out
@@ -0,0 +1,85 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+1970-01-01T01:00:30
+
+-- !select --
+1970-01-01T00:00:30
+
+-- !select --
+2022-05-25T00:00
+
+-- !select --
+2022-05-01T00:00
+
+-- !select --
+2022-05-25T00:06
+
+-- !select --
+2022-05-25T00:05
+
+-- !select --
+2022-05-26T00:00
+
+-- !select --
+2022-05-23T00:00
+
+-- !select --
+2022-05-29T00:00
+
+-- !select --
+2022-05-22T00:00
+
+-- !select --
+2022-06-01T00:00
+
+-- !select --
+2022-05-01T00:00
+
+-- !select --
+2023-01-01T00:00
+
+-- !select --
+2022-01-01T00:00
+
+-- !select --
+1970-01-01T01:00:30
+
+-- !select --
+1970-01-01T00:00:30
+
+-- !select --
+2022-05-25T00:00
+
+-- !select --
+2022-05-01T00:00
+
+-- !select --
+2022-05-25T00:06
+
+-- !select --
+2022-05-25T00:05
+
+-- !select --
+2022-05-26T00:00
+
+-- !select --
+2022-05-23T00:00
+
+-- !select --
+2022-05-29T00:00
+
+-- !select --
+2022-05-22T00:00
+
+-- !select --
+2022-06-01T00:00
+
+-- !select --
+2022-05-01T00:00
+
+-- !select --
+2023-01-01T00:00
+
+-- !select --
+2022-01-01T00:00
+
diff --git a/regression-test/suites/correctness/test_time_round.groovy b/regression-test/suites/correctness/test_time_round.groovy
new file mode 100644
index 0000000000..7e3b7d1891
--- /dev/null
+++ b/regression-test/suites/correctness/test_time_round.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_time_round") {
+
+ sql """ set enable_vectorized_engine = false """
+ // fix by issues/9711, expect: '1970-01-01T01:00:30'
+ qt_select "select hour_ceil('1970-01-01 01:00:10', 1, '1970-01-01 00:00:30')"
+
+ // fix by issues/9711, expect: '1970-01-01T00:00:30'
+ qt_select "select hour_floor('1970-01-01 01:00:10', 1, '1970-01-01 00:00:30')"
+
+ // fix by issues/9711, expect: '2022-05-25'
+ qt_select "select day_ceil('2022-05-25')"
+
+ // fix by issues/9711, expect: '2022-05-01'
+ qt_select "select month_ceil('2022-05-01')"
+
+ qt_select "select minute_ceil('2022-05-25 00:05:10')"
+ qt_select "select minute_floor('2022-05-25 00:05:10')"
+ qt_select "select day_ceil('2022-05-25 02:00:00', 3, '2022-05-20 00:00:00')"
+ qt_select "select day_floor('2022-05-25 02:00:00', 3, '2022-05-20 00:00:00')"
+ qt_select "select week_ceil('2022-05-25 00:00:00')"
+ qt_select "select week_floor('2022-05-25 00:00:00')"
+ qt_select "select month_ceil('2022-05-25 00:00:00')"
+ qt_select "select month_floor('2022-05-25 00:00:00')"
+ qt_select "select year_ceil('2022-05-25 00:00:00')"
+ qt_select "select year_floor('2022-05-25 00:00:00')"
+
+
+ sql """ set enable_vectorized_engine = true """
+ // fix by issues/9711, expect: '1970-01-01T01:00:30'
+ qt_select "select hour_ceil('1970-01-01 01:00:10', 1, '1970-01-01 00:00:30')"
+
+ // fix by issues/9711, expect: '1970-01-01T00:00:30'
+ qt_select "select hour_floor('1970-01-01 01:00:10', 1, '1970-01-01 00:00:30')"
+
+ // fix by issues/9711, expect: '2022-05-25'
+ qt_select "select day_ceil('2022-05-25')"
+
+ // fix by issues/9711, expect: '2022-05-01'
+ qt_select "select month_ceil('2022-05-01')"
+
+ qt_select "select minute_ceil('2022-05-25 00:05:10')"
+ qt_select "select minute_floor('2022-05-25 00:05:10')"
+ qt_select "select day_ceil('2022-05-25 02:00:00', 3, '2022-05-20 00:00:00')"
+ qt_select "select day_floor('2022-05-25 02:00:00', 3, '2022-05-20 00:00:00')"
+ qt_select "select week_ceil('2022-05-25 00:00:00')"
+ qt_select "select week_floor('2022-05-25 00:00:00')"
+ qt_select "select month_ceil('2022-05-25 00:00:00')"
+ qt_select "select month_floor('2022-05-25 00:00:00')"
+ qt_select "select year_ceil('2022-05-25 00:00:00')"
+ qt_select "select year_floor('2022-05-25 00:00:00')"
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 06/06: [feature](priv) Support grant node_priv to other user. (#9951)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit c975be38b2d6fe2371f02a9077197289925cca9b
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon Jun 6 11:04:20 2022 +0800
[feature](priv) Support grant node_priv to other user. (#9951)
Currently, only the root user has node_priv privileges.
That is, only the root user can operate the addition and deletion of nodes.
In the original design of Doris, there is an Operator role. This role can have node_priv for node operations.
This PR supports assigning node_priv to users other than root.
However, only users who have both grant_priv and node_priv can assign node_priv to other users.
This ensures that only the root user has this permission, and users who are given node_priv
cannot continue to expand this permission outward.
---
docs/en/administrator-guide/privilege.md | 10 +-
.../java/org/apache/doris/analysis/GrantStmt.java | 22 ++--
.../org/apache/doris/mysql/privilege/AuthTest.java | 124 ++++++++++++++++++++-
3 files changed, 147 insertions(+), 9 deletions(-)
diff --git a/docs/en/administrator-guide/privilege.md b/docs/en/administrator-guide/privilege.md
index f20316b1f7..d7e004a7b3 100644
--- a/docs/en/administrator-guide/privilege.md
+++ b/docs/en/administrator-guide/privilege.md
@@ -73,10 +73,18 @@ Doris currently supports the following permissions
Nodes change permissions. Including FE, BE, BROKER node addition, deletion, offline operations. Currently, this permission can only be granted to Root users.
+ The root user has this permission by default.
+
+ Users who have both Grant_priv and Node_priv can grant this privilege to other users.
+
+ This permission can only be granted to the Global level.
+
2. Grant_priv
Permissions change permissions. Allow the execution of operations including authorization, revocation, add/delete/change user/role, etc.
+ However, a user with this permission can not grant node_priv permission to other users, unless the user itself has node_priv permission.
+
3. Select_priv
Read-only access to databases and tables.
@@ -149,7 +157,7 @@ ADMIN\_PRIV and GRANT\_PRIV have the authority of **"grant authority"** at the s
2. It is not supported to delete or change the permissions of default created roles or users.
-3. The user of the operator role has one and only one user. Users of admin roles can create multiple.
+3. The user of the operator role has one and only one user, that is, root. Users of admin roles can create multiple.
4. Operational instructions for possible conflicts
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
index be915b9d27..0c18564962 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
@@ -126,26 +126,34 @@ public class GrantStmt extends DdlStmt {
}
}
- /*
+ /**
* Rules:
- * 1. Can not grant/revoke NODE_PRIV to/from any other user.
- * 2. ADMIN_PRIV can only be granted/revoked on GLOBAL level
+ * 1. ADMIN_PRIV and NODE_PRIV can only be granted/revoked on GLOBAL level
+ * 2. Only the user with NODE_PRIV can grant NODE_PRIV to other user
* 3. Privileges can not be granted/revoked to/from ADMIN and OPERATOR role
* 4. Only user with GLOBAL level's GRANT_PRIV can grant/revoke privileges to/from roles.
* 5.1 User should has GLOBAL level GRANT_PRIV
* 5.2 or user has DATABASE/TABLE level GRANT_PRIV if grant/revoke to/from certain database or table.
* 5.3 or user should has 'resource' GRANT_PRIV if grant/revoke to/from certain 'resource'
+ *
+ * @param analyzer
+ * @param privileges
+ * @param role
+ * @param tblPattern
+ * @throws AnalysisException
*/
public static void checkPrivileges(Analyzer analyzer, List<PaloPrivilege> privileges,
String role, TablePattern tblPattern) throws AnalysisException {
// Rule 1
- if (privileges.contains(PaloPrivilege.NODE_PRIV)) {
- throw new AnalysisException("Can not grant NODE_PRIV to any other users or roles");
+ if (tblPattern.getPrivLevel() != PrivLevel.GLOBAL && (privileges.contains(PaloPrivilege.ADMIN_PRIV)
+ || privileges.contains(PaloPrivilege.NODE_PRIV))) {
+ throw new AnalysisException("ADMIN_PRIV and NODE_PRIV can only be granted on *.*");
}
// Rule 2
- if (tblPattern.getPrivLevel() != PrivLevel.GLOBAL && privileges.contains(PaloPrivilege.ADMIN_PRIV)) {
- throw new AnalysisException("ADMIN_PRIV privilege can only be granted on *.*");
+ if (privileges.contains(PaloPrivilege.NODE_PRIV) && !Catalog.getCurrentCatalog().getAuth()
+ .checkGlobalPriv(ConnectContext.get(), PrivPredicate.OPERATOR)) {
+ throw new AnalysisException("Only the user with NODE_PRIV can grant NODE_PRIV to other user");
}
if (role != null) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
index 91182dc4cf..1d06a5b276 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
@@ -143,7 +143,8 @@ public class AuthTest {
}
@Test
- public void test() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+ public void test()
+ throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, UserException {
// 1. create cmy@%
UserIdentity userIdentity = new UserIdentity("cmy", "%");
UserDesc userDesc = new UserDesc(userIdentity, "12345", true);
@@ -1207,6 +1208,127 @@ public class AuthTest {
} catch (UserException e) {
e.printStackTrace();
}
+
+ // 40. create new user and grant node_priv to it
+ final UserIdentity opUser = new UserIdentity("op_user", "%");
+ userDesc = new UserDesc(opUser, "12345", true);
+ createUserStmt = new CreateUserStmt(false, userDesc, null);
+ createUserStmt.analyze(analyzer);
+ auth.createUser(createUserStmt);
+
+ privileges = Lists.newArrayList(AccessPrivilege.NODE_PRIV);
+ // 40.1 grant to non-global level, which is not allowed
+ grantStmt = new GrantStmt(opUser, null, new TablePattern("db1", "*"), privileges);
+ try {
+ grantStmt.analyze(analyzer);
+ Assert.fail();
+ } catch (UserException e) {
+ e.printStackTrace();
+ }
+
+ grantStmt = new GrantStmt(opUser, null, new TablePattern("db1", "tbl"), privileges);
+ try {
+ grantStmt.analyze(analyzer);
+ Assert.fail();
+ } catch (UserException e) {
+ e.printStackTrace();
+ }
+ // 40.2 grant to global level
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 1;
+ result = opUser;
+ }
+ };
+ Assert.assertFalse(auth.checkGlobalPriv(ctx, PrivPredicate.OPERATOR));
+ grantStmt = new GrantStmt(opUser, null, new TablePattern("*", "*"), privileges);
+ // first, use op_user itself to grant node_priv, which is not allowed
+ try {
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 1;
+ result = opUser;
+ }
+ };
+ grantStmt.analyze(analyzer);
+ Assert.fail();
+ } catch (UserException e) {
+ e.printStackTrace();
+ }
+ // second, use root to grant node_priv
+ try {
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 1;
+ result = UserIdentity.ROOT;
+ }
+ };
+ grantStmt.analyze(analyzer);
+ auth.grant(grantStmt);
+ } catch (UserException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ // switch to op_user to check it has node_priv
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 2;
+ result = opUser;
+ }
+ };
+ Assert.assertTrue(auth.checkGlobalPriv(ctx, PrivPredicate.OPERATOR));
+ // Now, op_user only has node_priv, it can not grant node_priv to other user.
+ // create otherOpUser first
+ UserIdentity otherOpUser = new UserIdentity("other_op_user", "%");
+ userDesc = new UserDesc(otherOpUser, "12345", true);
+ createUserStmt = new CreateUserStmt(false, userDesc, null);
+ createUserStmt.analyze(analyzer);
+ auth.createUser(createUserStmt);
+ // try grant, it should fail
+ grantStmt = new GrantStmt(otherOpUser, null, new TablePattern("*", "*"), privileges);
+ try {
+ grantStmt.analyze(analyzer);
+ Assert.fail();
+ } catch (UserException e) {
+ e.printStackTrace();
+ }
+ // Now, we grant grant_priv to opUser, and check if it can than grant node_priv to other user
+ privileges = Lists.newArrayList(AccessPrivilege.GRANT_PRIV);
+ grantStmt = new GrantStmt(opUser, null, new TablePattern("*", "*"), privileges);
+ try {
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 2;
+ result = UserIdentity.ROOT;
+ }
+ };
+ grantStmt.analyze(analyzer);
+ auth.grant(grantStmt);
+ } catch (UserException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ // grant node_priv to other_op_user
+ grantStmt = new GrantStmt(otherOpUser, null, new TablePattern("*", "*"), privileges);
+ try {
+ new Expectations() {
+ {
+ ctx.getCurrentUserIdentity();
+ minTimes = 1;
+ result = opUser;
+ }
+ };
+ grantStmt.analyze(analyzer);
+ auth.grant(grantStmt);
+ } catch (UserException e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/06: [fix] group by with two NULL rows after left join (#9688)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 5a1d1faba9964c8e8595628293d1278031eca079
Author: camby <10...@qq.com>
AuthorDate: Wed May 25 16:43:55 2022 +0800
[fix] group by with two NULL rows after left join (#9688)
Co-authored-by: cambyzju <zh...@baidu.com>
---
be/src/vec/columns/column_nullable.cpp | 5 +-
.../query/aggregate/aggregate_groupby_null.groovy | 58 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp
index 69634effb4..58f81fdf5e 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -102,7 +102,10 @@ StringRef ColumnNullable::serialize_value_into_arena(size_t n, Arena& arena,
static constexpr auto s = sizeof(arr[0]);
auto pos = arena.alloc_continue(s, begin);
- memcpy(pos, &arr[n], s);
+ // Value of `NULL` may be 1 or JOIN_NULL_HINT, we serialize both to 1.
+ // Because we need same key for both `NULL` values while processing `group by`.
+ UInt8* val = reinterpret_cast<UInt8*>(pos);
+ *val = (arr[n] ? 1 : 0);
if (arr[n]) return StringRef(pos, s);
diff --git a/regression-test/suites/query/aggregate/aggregate_groupby_null.groovy b/regression-test/suites/query/aggregate/aggregate_groupby_null.groovy
new file mode 100644
index 0000000000..dc3742c5fe
--- /dev/null
+++ b/regression-test/suites/query/aggregate/aggregate_groupby_null.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("aggregate_groupby_null") {
+ def leftTable = "agg_groupby_null_left"
+ sql """ DROP TABLE IF EXISTS ${leftTable} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${leftTable} (
+ id INT NULL,
+ device_id STRING NULL
+ )
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+ sql """ INSERT INTO ${leftTable} VALUES (1,'1'),(2,'2'),(3,'3'),(4,'4') """
+
+ def rightTable = "agg_groupby_null_right"
+ sql """ DROP TABLE IF EXISTS ${rightTable} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${rightTable} (
+ id INT NULL,
+ device_name STRING NULL
+ )
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+ sql """ INSERT INTO ${rightTable} VALUES (1,'name'),(3,null) """
+
+ sql """ set enable_vectorized_engine=false """
+ qt_groupby_null """ SELECT rt.device_name, COUNT(${leftTable}.id) FROM ${leftTable}
+ LEFT JOIN ${rightTable} rt ON ${leftTable}.id = rt.id
+ WHERE rt.device_name is NULL group by rt.device_name """
+
+ sql """ set enable_vectorized_engine=true """
+ qt_groupby_null """ SELECT rt.device_name, COUNT(${leftTable}.id) FROM ${leftTable}
+ LEFT JOIN ${rightTable} rt ON ${leftTable}.id = rt.id
+ WHERE rt.device_name is NULL group by rt.device_name """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/06: [Bug] Fix timestamp_diff issue when timeunit is year and month (#9574)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit ed04506425139151f760a1871596f6ff336b6c6e
Author: huangzhaowei <ca...@gmail.com>
AuthorDate: Thu May 19 21:24:43 2022 +0800
[Bug] Fix timestamp_diff issue when timeunit is year and month (#9574)
---
be/src/runtime/datetime_value.h | 14 ++++++++------
be/src/vec/runtime/vdatetime_value.h | 14 ++++++++------
2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h
index 4f3a310e48..b4c7c58638 100644
--- a/be/src/runtime/datetime_value.h
+++ b/be/src/runtime/datetime_value.h
@@ -288,11 +288,11 @@ public:
case YEAR: {
int year = (ts_value2.year() - ts_value1.year());
if (year > 0) {
- year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) <
- 0;
+ year -= (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) < 0;
} else if (year < 0) {
- year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) >
- 0;
+ year += (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) > 0;
}
return year;
}
@@ -300,9 +300,11 @@ public:
int month = (ts_value2.year() - ts_value1.year()) * 12 +
(ts_value2.month() - ts_value1.month());
if (month > 0) {
- month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0;
+ month -= (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) < 0;
} else if (month < 0) {
- month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0;
+ month += (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) > 0;
}
return month;
}
diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h
index a51be2e41f..99a6dee5c6 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -288,11 +288,11 @@ public:
case YEAR: {
int year = (ts_value2.year() - ts_value1.year());
if (year > 0) {
- year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) <
- 0;
+ year -= (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) < 0;
} else if (year < 0) {
- year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) >
- 0;
+ year += (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) > 0;
}
return year;
}
@@ -300,9 +300,11 @@ public:
int month = (ts_value2.year() - ts_value1.year()) * 12 +
(ts_value2.month() - ts_value1.month());
if (month > 0) {
- month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0;
+ month -= (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) < 0;
} else if (month < 0) {
- month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0;
+ month += (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) > 0;
}
return month;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org