You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/25 14:37:57 UTC
[doris] branch branch-2.0 updated (e5e4efe088 -> fe51e6ffdb)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from e5e4efe088 [test](regression) update some case in p2 (#21094)
new 35538ec669 [fix](stats) Fix jdbc timeout with multiple FE when execute analyze table (#21115)
new e173d309db [fix](nereids) check table privilege when it's needed (#21130)
new 1fa23d25a9 [bugfix](workloadgroup) could not upgrade from 2.0 alpha (#21149)
new c1c0689821 [Fix](inverted index) fix inverted query cache for chinese tokenizer (#21106)
new fe51e6ffdb [fix](nereids)scan node's smap should use materiazlied slots and project list as left and right expr list (#21142)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../rowset/segment_v2/inverted_index_cache.cpp | 6 ++
.../olap/rowset/segment_v2/inverted_index_cache.h | 12 +++-
.../rowset/segment_v2/inverted_index_query_type.h | 71 ++++++++++++++++++++++
.../rowset/segment_v2/inverted_index_reader.cpp | 1 +
.../olap/rowset/segment_v2/inverted_index_reader.h | 13 +---
.../org/apache/doris/analysis/AnalyzeStmt.java | 7 ++-
.../org/apache/doris/analysis/AnalyzeTblStmt.java | 5 --
.../org/apache/doris/analysis/RedirectStatus.java | 4 +-
.../main/java/org/apache/doris/catalog/Env.java | 14 -----
.../nereids/rules/analysis/UserAuthentication.java | 31 +++++++---
.../java/org/apache/doris/planner/ScanNode.java | 7 ++-
.../main/java/org/apache/doris/qe/DdlExecutor.java | 6 --
.../java/org/apache/doris/qe/StmtExecutor.java | 11 ++++
.../resource/workloadgroup/WorkloadGroup.java | 10 ++-
.../apache/doris/statistics/AnalysisManager.java | 25 ++++++--
.../inverted_index_p0/test_chinese_analyzer.out | 38 ++++++++++++
.../test_inlineview_with_project.groovy | 61 +++++++++++++++++++
.../inverted_index_p0/test_chinese_analyzer.groovy | 33 ++++++++++
18 files changed, 294 insertions(+), 61 deletions(-)
create mode 100644 be/src/olap/rowset/segment_v2/inverted_index_query_type.h
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/05: [fix](nereids)scan node's smap should use materiazlied slots and project list as left and right expr list (#21142)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fe51e6ffdb99a08a5f70a72bfe4cc2a8cdc26f6c
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Sun Jun 25 22:34:43 2023 +0800
[fix](nereids)scan node's smap should use materiazlied slots and project list as left and right expr list (#21142)
---
.../java/org/apache/doris/planner/ScanNode.java | 7 ++-
.../test_inlineview_with_project.groovy | 61 ++++++++++++++++++++++
2 files changed, 66 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 7dc0a7b3de..84b95c5a06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -533,8 +533,11 @@ public abstract class ScanNode extends PlanNode {
// this happens if the olap table is in the most inner sub-query block in the cascades sub-queries
// create a tmpSmap for the later setOutputSmap call
ExprSubstitutionMap tmpSmap = new ExprSubstitutionMap(
- Lists.newArrayList(outputTupleDesc.getSlots().stream().map(slot -> new SlotRef(slot)).collect(
- Collectors.toList())), Lists.newArrayList(projectList));
+ Lists.newArrayList(outputTupleDesc.getSlots().stream()
+ .filter(slot -> slot.isMaterialized())
+ .map(slot -> new SlotRef(slot))
+ .collect(Collectors.toList())),
+ Lists.newArrayList(projectList));
Set<SlotId> allOutputSlotIds = outputTupleDesc.getSlots().stream().map(slot -> slot.getId())
.collect(Collectors.toSet());
List<Expr> newRhs = Lists.newArrayList();
diff --git a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy
index 6eb09cf047..a155514041 100644
--- a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy
+++ b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy
@@ -16,6 +16,7 @@
// under the License.
suite("test_inlineview_with_project") {
+ sql "set enable_nereids_planner=false"
sql """
drop table if exists cir_1756_t1;
"""
@@ -321,4 +322,64 @@ suite("test_inlineview_with_project") {
sql """
drop table if exists ods_table4;
"""
+
+ sql """
+ drop table if exists cir2824_table;
+ """
+
+ sql """
+ CREATE TABLE `cir2824_table` (
+ `id` BIGINT(20) NULL,
+ `create_user` BIGINT(20) NULL,
+ `event_content` TEXT NULL,
+ `dest_farm_id` BIGINT(20) NULL,
+ `weight` DOUBLE NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 48
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "function_column.sequence_type" = "BIGINT",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ drop view if exists cir2824_view;
+ """
+
+ sql """
+ CREATE VIEW `cir2824_view` COMMENT 'VIEW' AS
+ select `ev`.`id` AS `id`,
+ CAST(`ev`.`create_user` AS BIGINT) AS `create_user`,
+ `ev`.`event_content` AS `event_content`,
+ `ev`.`dest_farm_id` AS `dest_farm_id`
+ FROM `cir2824_table` ev;
+ """
+
+ explain {
+ sql("""
+ WITH cir2824_temp1 AS( SELECT
+ CASE
+ WHEN dest_farm_id IS NULL
+ AND get_json_string(t.event_content,'\$.destFarmId') != '' THEN
+ 0
+ ELSE 1
+ END AS is_trans
+ FROM cir2824_view t )
+ SELECT 1
+ FROM cir2824_temp1;
+ """)
+ }
+
+ sql """
+ drop view if exists cir2824_view;
+ """
+
+ sql """
+ drop table if exists cir2824_table;
+ """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/05: [Fix](inverted index) fix inverted query cache for chinese tokenizer (#21106)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c1c068982107852c3be5431c41fd5993a09a5e45
Author: airborne12 <ai...@gmail.com>
AuthorDate: Sun Jun 25 22:04:02 2023 +0800
[Fix](inverted index) fix inverted query cache for chinese tokenizer (#21106)
1. query cache for chinese tokenizer is confusing when just converting w_char to char.
2. seperate query_type from inverted_index_reader to clean code.
---
.../rowset/segment_v2/inverted_index_cache.cpp | 6 ++
.../olap/rowset/segment_v2/inverted_index_cache.h | 12 +++-
.../rowset/segment_v2/inverted_index_query_type.h | 71 ++++++++++++++++++++++
.../rowset/segment_v2/inverted_index_reader.cpp | 1 +
.../olap/rowset/segment_v2/inverted_index_reader.h | 13 +---
.../inverted_index_p0/test_chinese_analyzer.out | 38 ++++++++++++
.../inverted_index_p0/test_chinese_analyzer.groovy | 33 ++++++++++
7 files changed, 159 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
index 7db99e26e9..243a738da9 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
@@ -238,6 +238,9 @@ Cache::Handle* InvertedIndexSearcherCache::_insert(const InvertedIndexSearcherCa
InvertedIndexQueryCache* InvertedIndexQueryCache::_s_instance = nullptr;
bool InvertedIndexQueryCache::lookup(const CacheKey& key, InvertedIndexQueryCacheHandle* handle) {
+ if (key.encode().empty()) {
+ return false;
+ }
auto lru_handle = _cache->lookup(key.encode());
if (lru_handle == nullptr) {
return false;
@@ -257,6 +260,9 @@ void InvertedIndexQueryCache::insert(const CacheKey& key, std::shared_ptr<roarin
cache_value_ptr->last_visit_time = UnixMillis();
cache_value_ptr->bitmap = bitmap;
cache_value_ptr->size = bitmap->getSizeInBytes();
+ if (key.encode().empty()) {
+ return;
+ }
auto lru_handle = _cache->insert(key.encode(), (void*)cache_value_ptr.release(),
bitmap->getSizeInBytes(), deleter, CachePriority::NORMAL);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.h b/be/src/olap/rowset/segment_v2/inverted_index_cache.h
index 7f9b7e8c14..0ab2b18af4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.h
@@ -18,6 +18,7 @@
#pragma once
#include <CLucene.h> // IWYU pragma: keep
+#include <CLucene/config/repl_wchar.h>
#include <CLucene/util/Misc.h>
#include <butil/macros.h>
#include <glog/logging.h>
@@ -35,6 +36,7 @@
#include "io/fs/file_system.h"
#include "io/fs/path.h"
#include "olap/lru_cache.h"
+#include "olap/rowset/segment_v2/inverted_index_query_type.h"
#include "runtime/memory/mem_tracker.h"
#include "util/slice.h"
#include "util/time.h"
@@ -183,7 +185,6 @@ private:
DISALLOW_COPY_AND_ASSIGN(InvertedIndexCacheHandle);
};
-enum class InvertedIndexQueryType;
class InvertedIndexQueryCacheHandle;
class InvertedIndexQueryCache {
@@ -201,9 +202,14 @@ public:
key_buf.append("/");
key_buf.append(column_name);
key_buf.append("/");
- key_buf.append(1, static_cast<char>(query_type));
+ auto query_type_str = InvertedIndexQueryType_toString(query_type);
+ if (query_type_str.empty()) {
+ return "";
+ }
+ key_buf.append(query_type_str);
key_buf.append("/");
- key_buf.append(lucene::util::Misc::toString(value.c_str()));
+ auto str = lucene_wcstoutf8string(value.c_str(), value.length());
+ key_buf.append(str);
return key_buf;
}
};
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_query_type.h b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
new file mode 100644
index 0000000000..1ebfe63591
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+namespace doris {
+namespace segment_v2 {
+
+enum class InvertedIndexQueryType {
+ UNKNOWN_QUERY = -1,
+ EQUAL_QUERY = 0,
+ LESS_THAN_QUERY = 1,
+ LESS_EQUAL_QUERY = 2,
+ GREATER_THAN_QUERY = 3,
+ GREATER_EQUAL_QUERY = 4,
+ MATCH_ANY_QUERY = 5,
+ MATCH_ALL_QUERY = 6,
+ MATCH_PHRASE_QUERY = 7,
+};
+
+inline std::string InvertedIndexQueryType_toString(InvertedIndexQueryType query_type) {
+ switch (query_type) {
+ case InvertedIndexQueryType::UNKNOWN_QUERY: {
+ return "UNKNOWN";
+ }
+ case InvertedIndexQueryType::EQUAL_QUERY: {
+ return "EQ";
+ }
+ case InvertedIndexQueryType::LESS_THAN_QUERY: {
+ return "LT";
+ }
+ case InvertedIndexQueryType::LESS_EQUAL_QUERY: {
+ return "LE";
+ }
+ case InvertedIndexQueryType::GREATER_THAN_QUERY: {
+ return "GT";
+ }
+ case InvertedIndexQueryType::GREATER_EQUAL_QUERY: {
+ return "GE";
+ }
+ case InvertedIndexQueryType::MATCH_ANY_QUERY: {
+ return "MANY";
+ }
+ case InvertedIndexQueryType::MATCH_ALL_QUERY: {
+ return "MALL";
+ }
+ case InvertedIndexQueryType::MATCH_PHRASE_QUERY: {
+ return "MPHRASE";
+ }
+ default:
+ return "";
+ }
+}
+} // namespace segment_v2
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index c311bcec8f..fae74fc883 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -344,6 +344,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, const std::string
InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name,
InvertedIndexQueryType::EQUAL_QUERY,
token_ws};
+ VLOG_DEBUG << "cache_key:" << cache_key.encode();
InvertedIndexQueryCacheHandle cache_handle;
if (cache->lookup(cache_key, &cache_handle)) {
stats->inverted_index_query_cache_hit++;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index ac7280ed54..1b30286a15 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -30,6 +30,7 @@
#include "io/fs/path.h"
#include "olap/inverted_index_parser.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_query_type.h"
#include "olap/tablet_schema.h"
namespace lucene {
@@ -63,18 +64,6 @@ enum class InvertedIndexReaderType {
BKD = 2,
};
-enum class InvertedIndexQueryType {
- UNKNOWN_QUERY = -1,
- EQUAL_QUERY = 0,
- LESS_THAN_QUERY = 1,
- LESS_EQUAL_QUERY = 2,
- GREATER_THAN_QUERY = 3,
- GREATER_EQUAL_QUERY = 4,
- MATCH_ANY_QUERY = 5,
- MATCH_ALL_QUERY = 6,
- MATCH_PHRASE_QUERY = 7,
-};
-
class InvertedIndexReader {
public:
explicit InvertedIndexReader(io::FileSystemSPtr fs, const std::string& path,
diff --git a/regression-test/data/inverted_index_p0/test_chinese_analyzer.out b/regression-test/data/inverted_index_p0/test_chinese_analyzer.out
index dfb1cd2ccb..91045120ab 100644
--- a/regression-test/data/inverted_index_p0/test_chinese_analyzer.out
+++ b/regression-test/data/inverted_index_p0/test_chinese_analyzer.out
@@ -27,3 +27,41 @@
-- !sql --
3 人民可以得到更多实惠
+-- !sql --
+2 我爱你中国
+
+-- !sql --
+2 我爱你中国
+
+-- !sql --
+1 我来到北京清华大学
+
+-- !sql --
+1 我来到北京清华大学
+
+-- !sql --
+1 我来到北京清华大学
+
+-- !sql --
+3 人民可以得到更多实惠
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
+-- !sql --
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
+-- !sql --
+4 陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.
+
diff --git a/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy b/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy
index 1acf8ffa6a..dd375f3894 100644
--- a/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy
+++ b/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy
@@ -77,4 +77,37 @@ suite("test_chinese_analyzer"){
qt_sql "SELECT * FROM $indexTblName2 WHERE c MATCH '大学' ORDER BY id;"
qt_sql "SELECT * FROM $indexTblName2 WHERE c MATCH '清华大学' ORDER BY id;"
qt_sql "SELECT * FROM $indexTblName2 WHERE c MATCH '人民' ORDER BY id;"
+
+ def indexTblName3 = "chinese_analyzer_test3"
+
+ sql "DROP TABLE IF EXISTS ${indexTblName3}"
+ // create 1 replica table
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTblName3}(
+ `id`int(11)NULL,
+ `c` text NULL,
+ INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="unicode", "support_phrase"="true") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES(
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql "INSERT INTO $indexTblName3 VALUES (1, '我来到北京清华大学'), (2, '我爱你中国'), (3, '人民可以得到更多实惠'), (4, '陕西省西安市高新区创业大厦A座，我的手机号码是12345678901,邮箱是12345678@qq.com，,ip是1.1.1.1，this information is created automatically.');"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_PHRASE '我爱你' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL'我爱你' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '清华' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '大学' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '清华大学' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '人民' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '陕西' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '12345678901' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_ALL '12345678' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_PHRASE '1.1.1.1' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_PHRASE '陕西西安' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_PHRASE '陕西省西安市' ORDER BY id;"
+ qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH 'information' ORDER BY id;"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/05: [fix](nereids) check table privilege when it's needed (#21130)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e173d309db63eed01912fae588302c7db9da3c14
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Sun Jun 25 18:35:39 2023 +0800
[fix](nereids) check table privilege when it's needed (#21130)
check privilege on LogicalOlapScan, LogicalEsScan, LogicalFileScan and LogicalSchemaScan
---
.../nereids/rules/analysis/UserAuthentication.java | 31 ++++++++++++++++------
1 file changed, 23 insertions(+), 8 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java
index 48356ea3a4..b88108cbc9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java
@@ -23,13 +23,23 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.collect.Sets;
+
+import java.util.Set;
+
/**
* Check whether a user is permitted to scan specific tables.
*/
public class UserAuthentication extends OneAnalysisRuleFactory {
+ Set<Class<?>> relationsToCheck = Sets.newHashSet(LogicalOlapScan.class, LogicalEsScan.class,
+ LogicalFileScan.class, LogicalSchemaScan.class);
@Override
public Rule build() {
@@ -43,15 +53,20 @@ public class UserAuthentication extends OneAnalysisRuleFactory {
if (connectContext.getSessionVariable().isPlayNereidsDump()) {
return relation;
}
- String dbName = !relation.getQualifier().isEmpty() ? relation.getQualifier().get(0) : null;
- String tableName = relation.getTable().getName();
- if (!connectContext.getEnv().getAccessManager()
- .checkTblPriv(connectContext, dbName, tableName, PrivPredicate.SELECT)) {
- String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SELECT",
- ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
- dbName + ": " + tableName);
- throw new AnalysisException(message);
+
+ if (relationsToCheck.contains(relation.getClass())) {
+ String dbName =
+ !relation.getQualifier().isEmpty() ? relation.getQualifier().get(0) : null;
+ String tableName = relation.getTable().getName();
+ if (!connectContext.getEnv().getAccessManager().checkTblPriv(connectContext, dbName,
+ tableName, PrivPredicate.SELECT)) {
+ String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SELECT",
+ ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
+ dbName + ": " + tableName);
+ throw new AnalysisException(message);
+ }
}
+
return relation;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/05: [bugfix](workloadgroup) could not upgrade from 2.0 alpha (#21149)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1fa23d25a9936ceb6af2698717e69df3899cd47e
Author: yiguolei <67...@qq.com>
AuthorDate: Sun Jun 25 22:02:53 2023 +0800
[bugfix](workloadgroup) could not upgrade from 2.0 alpha (#21149)
---------
Co-authored-by: yiguolei <yi...@gmail.com>
---
.../org/apache/doris/resource/workloadgroup/WorkloadGroup.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
index 1276f54db8..cb2c151215 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java
@@ -281,8 +281,14 @@ public class WorkloadGroup implements Writable, GsonPostProcessable {
@Override
public void gsonPostProcess() throws IOException {
- String memoryLimitString = properties.get(MEMORY_LIMIT);
- this.memoryLimitPercent = Double.parseDouble(memoryLimitString.substring(0, memoryLimitString.length() - 1));
+ if (properties.containsKey(MEMORY_LIMIT)) {
+ String memoryLimitString = properties.get(MEMORY_LIMIT);
+ this.memoryLimitPercent = Double.parseDouble(memoryLimitString.substring(0,
+ memoryLimitString.length() - 1));
+ } else {
+ this.memoryLimitPercent = 100;
+ this.properties.put(MEMORY_LIMIT, "100%");
+ }
this.initQueryQueue();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/05: [fix](stats) Fix jdbc timeout with multiple FE when execute analyze table (#21115)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 35538ec6694f5ccd08548d9b57abe631b4bebd26
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Sun Jun 25 16:49:36 2023 +0900
[fix](stats) Fix jdbc timeout with multiple FE when execute analyze table (#21115)
SQL may forward to master to execute when connecting to follower node, the result should be set to `StmtExecutor#proxyResultSet`
Before this PR, in above scenario , submit analyze sql by mysql client/jdbc whould return get malformed packet/ Communication failed.
---
.../org/apache/doris/analysis/AnalyzeStmt.java | 7 +++++-
.../org/apache/doris/analysis/AnalyzeTblStmt.java | 5 -----
.../org/apache/doris/analysis/RedirectStatus.java | 4 ++--
.../main/java/org/apache/doris/catalog/Env.java | 14 ------------
.../main/java/org/apache/doris/qe/DdlExecutor.java | 6 ------
.../java/org/apache/doris/qe/StmtExecutor.java | 11 ++++++++++
.../apache/doris/statistics/AnalysisManager.java | 25 ++++++++++++++++------
7 files changed, 38 insertions(+), 34 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
index 202a870f12..6f1f7c64d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
@@ -25,7 +25,7 @@ import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import java.util.Map;
-public class AnalyzeStmt extends DdlStmt {
+public class AnalyzeStmt extends StatementBase {
protected AnalyzeProperties analyzeProperties;
@@ -81,4 +81,9 @@ public class AnalyzeStmt extends DdlStmt {
public AnalyzeProperties getAnalyzeProperties() {
return analyzeProperties;
}
+
+ @Override
+ public RedirectStatus getRedirectStatus() {
+ return RedirectStatus.FORWARD_WITH_SYNC;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index 71d25c97f6..03681bdb36 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -252,11 +252,6 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
return table instanceof HMSExternalTable && table.getPartitionNames().size() > partNum;
}
- @Override
- public RedirectStatus getRedirectStatus() {
- return RedirectStatus.FORWARD_NO_SYNC;
- }
-
private void checkAnalyzePriv(String dbName, String tblName) throws AnalysisException {
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(ConnectContext.get(), dbName, tblName, PrivPredicate.SELECT)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RedirectStatus.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RedirectStatus.java
index 478f9fdf07..628d5a3080 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RedirectStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RedirectStatus.java
@@ -28,7 +28,7 @@ public class RedirectStatus {
public RedirectStatus(boolean isForwardToMaster, boolean needToWaitJournalSync) {
this.isForwardToMaster = isForwardToMaster;
- this.needToWaitJournalSync = needToWaitJournalSync;
+ this.needToWaitJournalSync = needToWaitJournalSync;
}
public boolean isForwardToMaster() {
@@ -47,7 +47,7 @@ public class RedirectStatus {
this.needToWaitJournalSync = needToWaitJournalSync;
}
- public static RedirectStatus FORWARD_NO_SYNC = new RedirectStatus(true, false);
+ public static RedirectStatus FORWARD_NO_SYNC = new RedirectStatus(true, false);
public static RedirectStatus FORWARD_WITH_SYNC = new RedirectStatus(true, true);
public static RedirectStatus NO_FORWARD = new RedirectStatus(false, false);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 90e4d06118..933b7714a6 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -39,7 +39,6 @@ import org.apache.doris.analysis.AlterMaterializedViewStmt;
import org.apache.doris.analysis.AlterSystemStmt;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterViewStmt;
-import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.BackupStmt;
import org.apache.doris.analysis.CancelAlterSystemStmt;
import org.apache.doris.analysis.CancelAlterTableStmt;
@@ -211,7 +210,6 @@ import org.apache.doris.resource.Tag;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.statistics.AnalysisManager;
-import org.apache.doris.statistics.AnalysisTaskScheduler;
import org.apache.doris.statistics.StatisticsAutoAnalyzer;
import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.StatisticsCleaner;
@@ -5345,18 +5343,6 @@ public class Env {
return count;
}
- public AnalysisTaskScheduler getAnalysisJobScheduler() {
- return analysisManager.taskScheduler;
- }
-
- // TODO:
- // 1. handle partition level analysis statement properly
- // 2. support sample job
- // 3. support period job
- public void createAnalysisJob(AnalyzeTblStmt analyzeTblStmt) throws DdlException {
- analysisManager.createAnalysisJob(analyzeTblStmt);
- }
-
public AnalysisManager getAnalysisManager() {
return analysisManager;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index faff8fdfb4..15b7bcc883 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -43,8 +43,6 @@ import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.AlterUserStmt;
import org.apache.doris.analysis.AlterViewStmt;
import org.apache.doris.analysis.AlterWorkloadGroupStmt;
-import org.apache.doris.analysis.AnalyzeDBStmt;
-import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.BackupStmt;
import org.apache.doris.analysis.CancelAlterSystemStmt;
import org.apache.doris.analysis.CancelAlterTableStmt;
@@ -297,8 +295,6 @@ public class DdlExecutor {
env.getRefreshManager().handleRefreshTable((RefreshTableStmt) ddlStmt);
} else if (ddlStmt instanceof RefreshDbStmt) {
env.getRefreshManager().handleRefreshDb((RefreshDbStmt) ddlStmt);
- } else if (ddlStmt instanceof AnalyzeTblStmt) {
- env.createAnalysisJob((AnalyzeTblStmt) ddlStmt);
} else if (ddlStmt instanceof AlterResourceStmt) {
env.getResourceMgr().alterResource((AlterResourceStmt) ddlStmt);
} else if (ddlStmt instanceof AlterWorkloadGroupStmt) {
@@ -337,8 +333,6 @@ public class DdlExecutor {
env.getAnalysisManager().dropStats((DropStatsStmt) ddlStmt);
} else if (ddlStmt instanceof KillAnalysisJobStmt) {
env.getAnalysisManager().handleKillAnalyzeStmt((KillAnalysisJobStmt) ddlStmt);
- } else if (ddlStmt instanceof AnalyzeDBStmt) {
- env.getAnalysisManager().createAnalysisJobs((AnalyzeDBStmt) ddlStmt);
} else if (ddlStmt instanceof CleanQueryStatsStmt) {
CleanQueryStatsStmt stmt = (CleanQueryStatsStmt) ddlStmt;
CleanQueryStatsInfo cleanQueryStatsInfo = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index ed68e03889..9195738b75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -736,6 +736,8 @@ public class StmtExecutor {
handleLockTablesStmt();
} else if (parsedStmt instanceof UnsupportedStmt) {
handleUnsupportedStmt();
+ } else if (parsedStmt instanceof AnalyzeStmt) {
+ handleAnalyzeStmt();
} else {
context.getState().setError(ErrorCode.ERR_NOT_SUPPORTED_YET, "Do not support this query.");
}
@@ -1899,6 +1901,10 @@ public class StmtExecutor {
context.getState().setOk();
}
+ private void handleAnalyzeStmt() throws DdlException {
+ context.env.getAnalysisManager().createAnalyze((AnalyzeStmt) parsedStmt, isProxy);
+ }
+
// Process switch catalog
private void handleSwitchStmt() throws AnalysisException {
SwitchStmt switchStmt = (SwitchStmt) parsedStmt;
@@ -2537,5 +2543,10 @@ public class StmtExecutor {
public void setProfileType(ProfileType profileType) {
this.profileType = profileType;
}
+
+
+ public void setProxyResultSet(ShowResultSet proxyResultSet) {
+ this.proxyResultSet = proxyResultSet;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index fc29affadb..c4443e5733 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AnalyzeDBStmt;
+import org.apache.doris.analysis.AnalyzeStmt;
import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.DropAnalyzeJobStmt;
import org.apache.doris.analysis.DropStatsStmt;
@@ -151,7 +152,15 @@ public class AnalysisManager extends Daemon implements Writable {
return statisticsCache;
}
- public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt) throws DdlException {
+ public void createAnalyze(AnalyzeStmt analyzeStmt, boolean proxy) throws DdlException {
+ if (analyzeStmt instanceof AnalyzeDBStmt) {
+ createAnalysisJobs((AnalyzeDBStmt) analyzeStmt, proxy);
+ } else if (analyzeStmt instanceof AnalyzeTblStmt) {
+ createAnalysisJob((AnalyzeTblStmt) analyzeStmt, proxy);
+ }
+ }
+
+ public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws DdlException {
DatabaseIf<TableIf> db = analyzeDBStmt.getDb();
List<TableIf> tbls = db.getTables();
List<AnalysisInfo> analysisInfos = new ArrayList<>();
@@ -179,7 +188,7 @@ public class AnalysisManager extends Daemon implements Writable {
analysisInfos.add(buildAndAssignJob(analyzeTblStmt));
}
if (!analyzeDBStmt.isSync()) {
- sendJobId(analysisInfos);
+ sendJobId(analysisInfos, proxy);
}
} finally {
db.readUnlock();
@@ -188,12 +197,12 @@ public class AnalysisManager extends Daemon implements Writable {
}
// Each analyze stmt corresponding to an analysis job.
- public void createAnalysisJob(AnalyzeTblStmt stmt) throws DdlException {
+ public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlException {
AnalysisInfo jobInfo = buildAndAssignJob(stmt);
if (jobInfo == null) {
return;
}
- sendJobId(ImmutableList.of(jobInfo));
+ sendJobId(ImmutableList.of(jobInfo), proxy);
}
@Nullable
@@ -259,7 +268,7 @@ public class AnalysisManager extends Daemon implements Writable {
analysisTaskInfos.values().forEach(taskScheduler::schedule);
}
- private void sendJobId(List<AnalysisInfo> analysisInfos) {
+ private void sendJobId(List<AnalysisInfo> analysisInfos, boolean proxy) {
List<Column> columns = new ArrayList<>();
columns.add(new Column("Catalog_Name", ScalarType.createVarchar(1024)));
columns.add(new Column("DB_Name", ScalarType.createVarchar(1024)));
@@ -279,7 +288,11 @@ public class AnalysisManager extends Daemon implements Writable {
}
ShowResultSet commonResultSet = new ShowResultSet(commonResultSetMetaData, resultRows);
try {
- ConnectContext.get().getExecutor().sendResultSet(commonResultSet);
+ if (!proxy) {
+ ConnectContext.get().getExecutor().sendResultSet(commonResultSet);
+ } else {
+ ConnectContext.get().getExecutor().setProxyResultSet(commonResultSet);
+ }
} catch (Throwable t) {
LOG.warn("Failed to send job id to user", t);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org