You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/30 10:03:15 UTC
[doris] branch dev-1.0.1 updated (6d98f6a43e -> 3948ef23b5)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 6d98f6a43e fix bug (#10509)
omit 589b44eddf [hotfix](dev-1.0.1) fix send fragment stuck bug
omit 73a7545982 [hotfix](dev-1.0.1) modify stop_be and disable meta_tool build by default
omit 9b6d585e75 [log] add more error info for hdfs reader writer (#10475)
omit 758eb455d4 [hotfix](dev-1.0.1) remove incubating
omit f2ac55d6ae [bugfix]fix bitmap function bug (#10477)
omit 7a2228b76b [hotfix] change hive, broker, iceberg table query to non-vec engine
omit 8cce760493 [hotfix](dev-1.0.1) fix agg node produce nullable value bug (#10430)
omit f0ec2e3639 [improvement]change shard num of page cache to 1024 to reduce lock contention (#10449)
omit cda74f67fb [hotfix] add rm -f to avoid build failed
omit 07d5672116 [fix](backup) fix mkdir failed (#10422) (#10423)
omit f0927d9755 [fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
omit 1df9ae0aaa [fix](doe) fix doe on es v8 (#10391)
omit cedf2f240a [bug] fix window function nullable type bug
omit c6fb643a32 Revert "[fix](vectorized) Support outer join for vectorized exec engine (#10323)"
new ef176deafa [fix](doe) fix doe on es v8 (#10391)
new d51a8d2131 [fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
new ade515225c [fix](backup) fix mkdir failed (#10422) (#10423)
new 1b03e68d11 [hotfix] add rm -f to avoid build failed
new 4ff62b32b4 [improvement]change shard num of page cache to 1024 to reduce lock contention (#10449)
new 36bbe02f96 [hotfix] change hive, broker, iceberg table query to non-vec engine
new 40a5673d4c [hotfix](dev-1.0.1) remove incubating
new 926138c6a0 [log] add more error info for hdfs reader writer (#10475)
new 7caa1eb6ab [hotfix](dev-1.0.1) modify stop_be and disable meta_tool build by default
new 6f649184be [hotfix](dev-1.0.1) fix send fragment stuck bug
new 108085330a [hotfix](dev-1.0.1) fix inline view bug for vec engine
new 9177ccd30a [hotfix](dev-1.0.1) BE prevent core by nullable not suit in hash join node
new 3948ef23b5 [bugfix]fix bitmap function bug (#10477)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6d98f6a43e)
\
N -- N -- N refs/heads/dev-1.0.1 (3948ef23b5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/exec_node.cpp | 4 +-
be/src/exec/exec_node.h | 2 +-
be/src/vec/columns/column_nullable.cpp | 7 +
be/src/vec/columns/column_nullable.h | 1 +
be/src/vec/core/block.cpp | 4 +-
be/src/vec/exec/join/vhash_join_node.cpp | 422 +++++++++++++--------
be/src/vec/exec/join/vhash_join_node.h | 37 +-
be/src/vec/exec/vaggregation_node.cpp | 73 ++--
be/src/vec/exec/vaggregation_node.h | 7 +-
be/src/vec/exec/vanalytic_eval_node.cpp | 11 +-
be/src/vec/exprs/vslot_ref.h | 1 -
be/src/vec/functions/functions_logical.cpp | 18 +-
be/src/vec/sink/vtablet_sink.cpp | 12 +-
.../java/org/apache/doris/analysis/Analyzer.java | 71 +---
.../org/apache/doris/analysis/DescriptorTable.java | 17 +
.../apache/doris/analysis/ExprSubstitutionMap.java | 107 +++++-
.../java/org/apache/doris/analysis/FromClause.java | 23 +-
.../org/apache/doris/analysis/InlineViewRef.java | 1 +
.../java/org/apache/doris/analysis/SelectStmt.java | 8 -
.../java/org/apache/doris/analysis/TableRef.java | 5 -
.../org/apache/doris/planner/AggregationNode.java | 17 +-
.../org/apache/doris/planner/HashJoinNode.java | 317 ++++++++++++++--
.../org/apache/doris/planner/OlapScanNode.java | 1 -
.../java/org/apache/doris/planner/PlanNode.java | 15 +-
.../org/apache/doris/planner/ProjectPlanner.java | 3 +-
.../org/apache/doris/planner/SetOperationNode.java | 3 +-
.../apache/doris/planner/SingleNodePlanner.java | 24 +-
.../java/org/apache/doris/planner/SortNode.java | 2 +-
.../doris/planner/ProjectPlannerFunctionTest.java | 4 +-
.../org/apache/doris/planner/QueryPlanTest.java | 9 +-
gensrc/thrift/PlanNodes.thrift | 5 +
.../test_outer_join_with_window_function.out | 4 -
.../test_outer_join_with_window_function.groovy | 289 --------------
33 files changed, 867 insertions(+), 657 deletions(-)
delete mode 100644 regression-test/data/correctness/test_outer_join_with_window_function.out
delete mode 100644 regression-test/suites/correctness/test_outer_join_with_window_function.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/13: [fix](doe) fix doe on es v8 (#10391)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ef176deafa31ba55b89a962f0083ccd408fd65c2
Author: Stalary <st...@163.com>
AuthorDate: Sun Jun 26 09:51:29 2022 +0800
[fix](doe) fix doe on es v8 (#10391)
doris on es8 can not work, because type change. The use of type is no longer recommended in es7,
and support for type has been removed from es8.
1. /_mapping not support include_type_name
2. /_search not support use type
---
be/src/exec/es/es_scan_reader.cpp | 32 +++++++++----
docs/en/extending-doris/doris-on-es.md | 14 +-----
docs/zh-CN/extending-doris/doris-on-es.md | 14 +-----
.../java/org/apache/doris/catalog/Catalog.java | 12 ++---
.../java/org/apache/doris/catalog/EsTable.java | 53 +++++++++++-----------
.../external/elasticsearch/EsMajorVersion.java | 7 +--
.../doris/external/elasticsearch/EsRestClient.java | 5 +-
.../doris/external/elasticsearch/MappingPhase.java | 20 ++------
.../java/org/apache/doris/planner/EsScanNode.java | 4 +-
.../external/elasticsearch/MappingPhaseTest.java | 2 +-
10 files changed, 71 insertions(+), 92 deletions(-)
diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp
index 87131a7647..5d67d995cf 100644
--- a/be/src/exec/es/es_scan_reader.cpp
+++ b/be/src/exec/es/es_scan_reader.cpp
@@ -47,7 +47,9 @@ ESScanReader::ESScanReader(const std::string& target,
_doc_value_mode(doc_value_mode) {
_target = target;
_index = props.at(KEY_INDEX);
- _type = props.at(KEY_TYPE);
+ if (props.find(KEY_TYPE) != props.end()) {
+ _type = props.at(KEY_TYPE);
+ }
if (props.find(KEY_USER_NAME) != props.end()) {
_user_name = props.at(KEY_USER_NAME);
}
@@ -73,20 +75,32 @@ ESScanReader::ESScanReader(const std::string& target,
_exactly_once = true;
std::stringstream scratch;
// just send a normal search against the elasticsearch with additional terminate_after param to achieve terminate early effect when limit take effect
- scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
- << "/_search?"
- << "terminate_after=" << props.at(KEY_TERMINATE_AFTER) << REQUEST_PREFERENCE_PREFIX
- << _shards << "&" << filter_path;
+ if (_type.empty()) {
+ scratch << _target << REQUEST_SEPARATOR << _index << "/_search?"
+ << "terminate_after=" << props.at(KEY_TERMINATE_AFTER)
+ << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path;
+ } else {
+ scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
+ << "/_search?"
+ << "terminate_after=" << props.at(KEY_TERMINATE_AFTER)
+ << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path;
+ }
_search_url = scratch.str();
} else {
_exactly_once = false;
std::stringstream scratch;
// scroll request for scanning
// add terminate_after for the first scroll to avoid decompress all postings list
- scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
- << "/_search?"
- << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards << "&"
- << filter_path << "&terminate_after=" << batch_size_str;
+ if (_type.empty()) {
+ scratch << _target << REQUEST_SEPARATOR << _index << "/_search?"
+ << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards
+ << "&" << filter_path << "&terminate_after=" << batch_size_str;
+ } else {
+ scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
+ << "/_search?"
+ << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards
+ << "&" << filter_path << "&terminate_after=" << batch_size_str;
+ }
_init_scroll_url = scratch.str();
_next_scroll_url = _target + REQUEST_SEARCH_SCROLL_PATH + "?" + filter_path;
}
diff --git a/docs/en/extending-doris/doris-on-es.md b/docs/en/extending-doris/doris-on-es.md
index a653986c35..8a3a104b4a 100644
--- a/docs/en/extending-doris/doris-on-es.md
+++ b/docs/en/extending-doris/doris-on-es.md
@@ -119,7 +119,6 @@ PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
"type" = "doc",
-
"user" = "root",
"password" = "root"
);
@@ -131,7 +130,7 @@ Parameter | Description
---|---
**hosts** | ES Cluster Connection Address, maybe one or more node, load-balance is also accepted
**index** | the related ES index name, alias is supported, and if you use doc_value, you need to use the real name
-**type** | the type for this index, If not specified, `_doc` will be used
+**type** | the type for this index, ES 7.x and later versions do not pass this parameter
**user** | username for ES
**password** | password for the user
@@ -188,10 +187,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_docvalue_scan" = "true"
);
```
@@ -229,10 +226,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_keyword_sniff" = "true"
);
```
@@ -341,10 +336,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"nodes_discovery" = "true"
);
```
@@ -370,10 +363,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"http_ssl_enabled" = "true"
);
```
@@ -561,8 +552,7 @@ PROPERTIES (
"hosts" = "http://127.0.0.1:8200",
"user" = "root",
"password" = "root",
-"index" = "doe",
-"type" = "doc"
+"index" = "doe"
}
```
`Notice`:
diff --git a/docs/zh-CN/extending-doris/doris-on-es.md b/docs/zh-CN/extending-doris/doris-on-es.md
index 7840b6be27..1ca84bf4c5 100644
--- a/docs/zh-CN/extending-doris/doris-on-es.md
+++ b/docs/zh-CN/extending-doris/doris-on-es.md
@@ -117,7 +117,6 @@ PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
"type" = "doc",
-
"user" = "root",
"password" = "root"
);
@@ -129,7 +128,7 @@ PROPERTIES (
---|---
**hosts** | ES集群地址，可以是一个或多个，也可以是ES前端的负载均衡地址
**index** | 对应的ES的index名字，支持alias，如果使用doc_value，需要使用真实的名称
-**type** | index的type，不指定的情况会使用_doc
+**type** | index的type，ES 7.x及以后的版本不传此参数
**user** | ES集群用户名
**password** | 对应用户的密码信息
@@ -185,10 +184,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_docvalue_scan" = "true"
);
```
@@ -226,10 +223,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_keyword_sniff" = "true"
);
```
@@ -338,10 +333,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"nodes_discovery" = "true"
);
```
@@ -367,10 +360,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"http_ssl_enabled" = "true"
);
```
@@ -559,8 +550,7 @@ PROPERTIES (
"hosts" = "http://127.0.0.1:8200",
"user" = "root",
"password" = "root",
-"index" = "doe",
-"type" = "doc"
+"index" = "doe"
}
```
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 22e532f78f..69832b139a 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -4362,12 +4362,8 @@ public class Catalog {
if (partitionInfo.getType() == PartitionType.RANGE) {
sb.append("\n");
sb.append("PARTITION BY RANGE(");
- idx = 0;
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo;
for (Column column : rangePartitionInfo.getPartitionColumns()) {
- if (idx != 0) {
- sb.append(", ");
- }
sb.append("`").append(column.getName()).append("`");
}
sb.append(")\n()");
@@ -4379,12 +4375,14 @@ public class Catalog {
sb.append("\"user\" = \"").append(esTable.getUserName()).append("\",\n");
sb.append("\"password\" = \"").append(hidePassword ? "" : esTable.getPasswd()).append("\",\n");
sb.append("\"index\" = \"").append(esTable.getIndexName()).append("\",\n");
- sb.append("\"type\" = \"").append(esTable.getMappingType()).append("\",\n");
+ if (esTable.getMappingType() != null) {
+ sb.append("\"type\" = \"").append(esTable.getMappingType()).append("\",\n");
+ }
sb.append("\"transport\" = \"").append(esTable.getTransport()).append("\",\n");
sb.append("\"enable_docvalue_scan\" = \"").append(esTable.isDocValueScanEnable()).append("\",\n");
sb.append("\"max_docvalue_fields\" = \"").append(esTable.maxDocValueFields()).append("\",\n");
- sb.append("\"enable_keyword_sniff\" = \"").append(esTable.isKeywordSniffEnable()).append("\"\n");
- sb.append("\"nodes_discovery\" = \"").append(esTable.isNodesDiscovery()).append("\"\n");
+ sb.append("\"enable_keyword_sniff\" = \"").append(esTable.isKeywordSniffEnable()).append("\",\n");
+ sb.append("\"nodes_discovery\" = \"").append(esTable.isNodesDiscovery()).append("\",\n");
sb.append("\"http_ssl_enabled\" = \"").append(esTable.isHttpSslEnabled()).append("\"\n");
sb.append(")");
} else if (table.getType() == TableType.HIVE) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index df19a86d96..cc5d902080 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -73,8 +73,8 @@ public class EsTable extends Table {
// index name can be specific index、wildcard matched or alias.
private String indexName;
- // which type used for `indexName`, default to `_doc`
- private String mappingType = "_doc";
+ // which type used for `indexName`
+ private String mappingType = null;
private String transport = "http";
// only save the partition definition, save the partition key,
// partition list is got from es cluster dynamically and is saved in esTableState
@@ -116,8 +116,8 @@ public class EsTable extends Table {
super(TableType.ELASTICSEARCH);
}
- public EsTable(long id, String name, List<Column> schema,
- Map<String, String> properties, PartitionInfo partitionInfo) throws DdlException {
+ public EsTable(long id, String name, List<Column> schema, Map<String, String> properties,
+ PartitionInfo partitionInfo) throws DdlException {
super(id, name, TableType.ELASTICSEARCH, schema);
this.partitionInfo = partitionInfo;
validate(properties);
@@ -154,32 +154,29 @@ public class EsTable extends Table {
private void validate(Map<String, String> properties) throws DdlException {
if (properties == null) {
- throw new DdlException("Please set properties of elasticsearch table, "
- + "they are: hosts, user, password, index");
+ throw new DdlException(
+ "Please set properties of elasticsearch table, " + "they are: hosts, user, password, index");
}
- if (Strings.isNullOrEmpty(properties.get(HOSTS))
- || Strings.isNullOrEmpty(properties.get(HOSTS).trim())) {
+ if (Strings.isNullOrEmpty(properties.get(HOSTS)) || Strings.isNullOrEmpty(properties.get(HOSTS).trim())) {
throw new DdlException("Hosts of ES table is null. "
+ "Please add properties('hosts'='xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx') when create table");
}
hosts = properties.get(HOSTS).trim();
seeds = hosts.split(",");
- if (!Strings.isNullOrEmpty(properties.get(USER))
- && !Strings.isNullOrEmpty(properties.get(USER).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(USER)) && !Strings.isNullOrEmpty(properties.get(USER).trim())) {
userName = properties.get(USER).trim();
}
- if (!Strings.isNullOrEmpty(properties.get(PASSWORD))
- && !Strings.isNullOrEmpty(properties.get(PASSWORD).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(PASSWORD)) && !Strings.isNullOrEmpty(
+ properties.get(PASSWORD).trim())) {
passwd = properties.get(PASSWORD).trim();
}
- if (Strings.isNullOrEmpty(properties.get(INDEX))
- || Strings.isNullOrEmpty(properties.get(INDEX).trim())) {
- throw new DdlException("Index of ES table is null. "
- + "Please add properties('index'='xxxx') when create table");
+ if (Strings.isNullOrEmpty(properties.get(INDEX)) || Strings.isNullOrEmpty(properties.get(INDEX).trim())) {
+ throw new DdlException(
+ "Index of ES table is null. " + "Please add properties('index'='xxxx') when create table");
}
indexName = properties.get(INDEX).trim();
@@ -191,8 +188,8 @@ public class EsTable extends Table {
throw new DdlException("Unsupported/Unknown ES Cluster version [" + properties.get(VERSION) + "] ");
}
} catch (Exception e) {
- throw new DdlException("fail to parse ES major version, version= "
- + properties.get(VERSION).trim() + ", should be like '6.5.3' ");
+ throw new DdlException("fail to parse ES major version, version= " + properties.get(VERSION).trim()
+ + ", should be like '6.5.3' ");
}
}
@@ -222,13 +219,12 @@ public class EsTable extends Table {
}
}
- if (!Strings.isNullOrEmpty(properties.get(TYPE))
- && !Strings.isNullOrEmpty(properties.get(TYPE).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(TYPE)) && !Strings.isNullOrEmpty(properties.get(TYPE).trim())) {
mappingType = properties.get(TYPE).trim();
}
- if (!Strings.isNullOrEmpty(properties.get(TRANSPORT))
- && !Strings.isNullOrEmpty(properties.get(TRANSPORT).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(TRANSPORT)) && !Strings.isNullOrEmpty(
+ properties.get(TRANSPORT).trim())) {
transport = properties.get(TRANSPORT).trim();
if (!(TRANSPORT_HTTP.equals(transport) || TRANSPORT_THRIFT.equals(transport))) {
throw new DdlException("transport of ES table must be http/https(recommend) or thrift(reserved inner usage),"
@@ -250,7 +246,9 @@ public class EsTable extends Table {
tableContext.put("userName", userName);
tableContext.put("passwd", passwd);
tableContext.put("indexName", indexName);
- tableContext.put("mappingType", mappingType);
+ if (mappingType != null) {
+ tableContext.put("mappingType", mappingType);
+ }
tableContext.put("transport", transport);
if (majorVersion != null) {
tableContext.put("majorVersion", majorVersion.toString());
@@ -264,8 +262,8 @@ public class EsTable extends Table {
public TTableDescriptor toThrift() {
TEsTable tEsTable = new TEsTable();
- TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ES_TABLE,
- fullSchema.size(), 0, getName(), "");
+ TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ES_TABLE, fullSchema.size(), 0,
+ getName(), "");
tTableDescriptor.setEsTable(tEsTable);
return tTableDescriptor;
}
@@ -280,7 +278,9 @@ public class EsTable extends Table {
sb.append(userName);
sb.append(passwd);
sb.append(indexName);
- sb.append(mappingType);
+ if (mappingType != null) {
+ sb.append(mappingType);
+ }
sb.append(transport);
} else {
for (Map.Entry<String, String> entry : tableContext.entrySet()) {
@@ -305,6 +305,7 @@ public class EsTable extends Table {
partitionInfo.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
int size = in.readInt();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
index d16fbd81d2..41801687f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
@@ -32,7 +32,7 @@ public class EsMajorVersion {
public static final EsMajorVersion V_6_X = new EsMajorVersion((byte) 6, "6.x");
public static final EsMajorVersion V_7_X = new EsMajorVersion((byte) 7, "7.x");
public static final EsMajorVersion V_8_X = new EsMajorVersion((byte) 8, "8.x");
- public static final EsMajorVersion LATEST = V_7_X;
+ public static final EsMajorVersion LATEST = V_8_X;
public final byte major;
private final String version;
@@ -89,8 +89,9 @@ public class EsMajorVersion {
if (version.startsWith("8.")) {
return new EsMajorVersion((byte) 8, version);
}
- throw new DorisEsException("Unsupported/Unknown ES Cluster version [" + version + "]." +
- "Highest supported version is [" + LATEST.version + "].");
+ throw new DorisEsException(
+ "Unsupported/Unknown ES Cluster version [" + version + "]." + "Highest supported version is ["
+ + LATEST.version + "].");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
index 467047ac8c..5a00719154 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
@@ -127,11 +127,8 @@ public class EsRestClient {
* @return
* @throws Exception
*/
- public String getMapping(String indexName, boolean includeTypeName) throws DorisEsException {
+ public String getMapping(String indexName) throws DorisEsException {
String path = indexName + "/_mapping";
- if (includeTypeName) {
- path += "?include_type_name=true";
- }
String indexMapping = execute(path);
if (indexMapping == null) {
throw new DorisEsException("index[" + indexName + "] not found");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index f736a9ee2f..4057348e66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -37,22 +37,13 @@ public class MappingPhase implements SearchPhase {
// json response for `{index}/_mapping` API
private String jsonMapping;
- private boolean includeTypeName = false;
-
public MappingPhase(EsRestClient client) {
this.client = client;
}
- @Override
- public void preProcess(SearchContext context) {
- if (context.version() != null && context.version().onOrAfter(EsMajorVersion.V_7_X)) {
- includeTypeName = true;
- }
- }
-
@Override
public void execute(SearchContext context) throws DorisEsException {
- jsonMapping = client.getMapping(context.sourceIndex(), includeTypeName);
+ jsonMapping = client.getMapping(context.sourceIndex());
}
@Override
@@ -78,15 +69,10 @@ public class MappingPhase implements SearchPhase {
JSONObject mappings = (JSONObject) docData.get("mappings");
JSONObject rootSchema = (JSONObject) mappings.get(searchContext.type());
JSONObject properties;
- // After (include) 7.x, type was removed from ES mapping, default type is `_doc`
+ // Elasticsearch 7.x, type was removed from ES mapping, default type is `_doc`
// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html
+ // Elasticsearch 8.x, include_type_name parameter is removed
if (rootSchema == null) {
- if (searchContext.type().equals("_doc") == false) {
- throw new DorisEsException("index[" + searchContext.sourceIndex() + "]'s type must be exists, "
- + " and after ES7.x type must be `_doc`, but found ["
- + searchContext.type() + "], for table ["
- + searchContext.esTable().getName() + "]");
- }
properties = (JSONObject) mappings.get("properties");
} else {
properties = (JSONObject) rootSchema.get("properties");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
index 2ed1a4bde8..42f3934a9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
@@ -247,7 +247,9 @@ public class EsScanNode extends ScanNode {
TEsScanRange esScanRange = new TEsScanRange();
esScanRange.setEsHosts(shardAllocations);
esScanRange.setIndex(shardRouting.get(0).getIndexName());
- esScanRange.setType(table.getMappingType());
+ if (table.getType() != null) {
+ esScanRange.setType(table.getMappingType());
+ }
esScanRange.setShardId(shardRouting.get(0).getShardId());
// Scan range
TScanRange scanRange = new TScanRange();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
index d229d7c924..18330eb55a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
@@ -89,7 +89,7 @@ public class MappingPhaseTest extends EsTestCase {
String jsonMapping = loadJsonFromFile("data/es/test_index_mapping.json");
new Expectations(client) {
{
- client.getMapping(anyString, anyBoolean);
+ client.getMapping(anyString);
minTimes = 0;
result = jsonMapping;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 12/13: [hotfix](dev-1.0.1) BE prevent core by nullable not suit in hash join node
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9177ccd30af9321a5ade5845fe7c87c71c9a7ab9
Author: lihaopeng <li...@baidu.com>
AuthorDate: Sun Jun 26 23:55:21 2022 +0800
[hotfix](dev-1.0.1) BE prevent core by nullable not suit in hash join node
---
be/src/vec/exec/join/vhash_join_node.cpp | 145 ++++++++++++---------
be/src/vec/exec/vaggregation_node.cpp | 29 +++--
be/src/vec/exec/vaggregation_node.h | 1 +
be/src/vec/exprs/vslot_ref.h | 1 -
be/src/vec/functions/functions_logical.cpp | 18 ++-
.../org/apache/doris/planner/AggregationNode.java | 1 +
gensrc/thrift/PlanNodes.thrift | 1 +
7 files changed, 118 insertions(+), 78 deletions(-)
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index c8d7f6e77c..596bef712b 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -28,12 +28,11 @@
namespace doris::vectorized {
-std::variant<std::false_type, std::true_type>
-static inline make_bool_variant(bool condition) {
+std::variant<std::false_type, std::true_type> static inline make_bool_variant(bool condition) {
if (condition) {
- return std::true_type{};
+ return std::true_type {};
} else {
- return std::false_type{};
+ return std::false_type {};
}
}
@@ -178,7 +177,7 @@ struct ProcessHashTableProbe {
// output build side result column
template <bool have_other_join_conjunct = false>
void build_side_output_column(MutableColumns& mcol, int column_offset, int column_length,
- const std::vector<bool>& output_slot_flags, int size) {
+ const std::vector<bool>& output_slot_flags, int size) {
constexpr auto is_semi_anti_join = JoinOpType::value == TJoinOp::RIGHT_ANTI_JOIN ||
JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN ||
JoinOpType::value == TJoinOp::LEFT_ANTI_JOIN ||
@@ -192,8 +191,8 @@ struct ProcessHashTableProbe {
for (int i = 0; i < column_length; i++) {
auto& column = *_build_blocks[0].get_by_position(i).column;
if (output_slot_flags[i]) {
- mcol[i + column_offset]->insert_indices_from(column, _build_block_rows.data(),
- _build_block_rows.data() + size);
+ mcol[i + column_offset]->insert_indices_from(
+ column, _build_block_rows.data(), _build_block_rows.data() + size);
} else {
mcol[i + column_offset]->resize(size);
}
@@ -205,14 +204,19 @@ struct ProcessHashTableProbe {
if constexpr (probe_all) {
if (_build_block_offsets[j] == -1) {
DCHECK(mcol[i + column_offset]->is_nullable());
- assert_cast<ColumnNullable *>(
- mcol[i + column_offset].get())->insert_join_null_data();
+ assert_cast<ColumnNullable*>(mcol[i + column_offset].get())
+ ->insert_join_null_data();
} else {
- auto &column = *_build_blocks[_build_block_offsets[j]].get_by_position(i).column;
- mcol[i + column_offset]->insert_from(column, _build_block_rows[j]);
+ auto& column = *_build_blocks[_build_block_offsets[j]]
+ .get_by_position(i)
+ .column;
+ mcol[i + column_offset]->insert_from(column,
+ _build_block_rows[j]);
}
} else {
- auto &column = *_build_blocks[_build_block_offsets[j]].get_by_position(i).column;
+ auto& column = *_build_blocks[_build_block_offsets[j]]
+ .get_by_position(i)
+ .column;
mcol[i + column_offset]->insert_from(column, _build_block_rows[j]);
}
}
@@ -225,7 +229,8 @@ struct ProcessHashTableProbe {
}
// output probe side result column
- void probe_side_output_column(MutableColumns& mcol, const std::vector<bool>& output_slot_flags, int size) {
+ void probe_side_output_column(MutableColumns& mcol, const std::vector<bool>& output_slot_flags,
+ int size) {
for (int i = 0; i < output_slot_flags.size(); ++i) {
if (output_slot_flags[i]) {
auto& column = _probe_block.get_by_position(i).column;
@@ -244,8 +249,8 @@ struct ProcessHashTableProbe {
using KeyGetter = typename HashTableContext::State;
using Mapped = typename HashTableContext::Mapped;
- int right_col_idx = _join_node->_is_right_semi_anti ? 0 :
- _join_node->_left_table_data_types.size();
+ int right_col_idx =
+ _join_node->_is_right_semi_anti ? 0 : _join_node->_left_table_data_types.size();
int right_col_len = _join_node->_right_table_data_types.size();
KeyGetter key_getter(_probe_raw_ptrs, _join_node->_probe_key_sz, nullptr);
@@ -258,15 +263,15 @@ struct ProcessHashTableProbe {
memset(_items_counts.data(), 0, sizeof(uint32_t) * _probe_rows);
constexpr auto need_to_set_visited = JoinOpType::value == TJoinOp::RIGHT_ANTI_JOIN ||
- JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN ||
- JoinOpType::value == TJoinOp::RIGHT_OUTER_JOIN ||
- JoinOpType::value == TJoinOp::FULL_OUTER_JOIN;
+ JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN ||
+ JoinOpType::value == TJoinOp::RIGHT_OUTER_JOIN ||
+ JoinOpType::value == TJoinOp::FULL_OUTER_JOIN;
constexpr auto is_right_semi_anti_join = JoinOpType::value == TJoinOp::RIGHT_ANTI_JOIN ||
- JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN;
+ JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN;
constexpr auto probe_all = JoinOpType::value == TJoinOp::LEFT_OUTER_JOIN ||
- JoinOpType::value == TJoinOp::FULL_OUTER_JOIN;
+ JoinOpType::value == TJoinOp::FULL_OUTER_JOIN;
{
SCOPED_TIMER(_search_hashtable_timer);
@@ -279,9 +284,11 @@ struct ProcessHashTableProbe {
}
int last_offset = current_offset;
auto find_result = (*null_map)[_probe_index]
- ? decltype(key_getter.find_key(hash_table_ctx.hash_table, _probe_index,
- _arena)) {nullptr, false}
- : key_getter.find_key(hash_table_ctx.hash_table, _probe_index, _arena);
+ ? decltype(key_getter.find_key(hash_table_ctx.hash_table,
+ _probe_index,
+ _arena)) {nullptr, false}
+ : key_getter.find_key(hash_table_ctx.hash_table,
+ _probe_index, _arena);
if constexpr (JoinOpType::value == TJoinOp::LEFT_ANTI_JOIN) {
if (!find_result.is_found()) {
@@ -297,8 +304,7 @@ struct ProcessHashTableProbe {
// TODO: Iterators are currently considered to be a heavy operation and have a certain impact on performance.
// We should rethink whether to use this iterator mode in the future. Now just opt the one row case
if (mapped.get_row_count() == 1) {
- if constexpr (need_to_set_visited)
- mapped.visited = true;
+ if constexpr (need_to_set_visited) mapped.visited = true;
if constexpr (!is_right_semi_anti_join) {
_build_block_offsets[current_offset] = mapped.block_offset;
@@ -308,7 +314,8 @@ struct ProcessHashTableProbe {
} else {
// prefetch is more useful while matching to multiple rows
if (_probe_index + 2 < _probe_rows)
- key_getter.prefetch(hash_table_ctx.hash_table, _probe_index + 2, _arena);
+ key_getter.prefetch(hash_table_ctx.hash_table, _probe_index + 2,
+ _arena);
for (auto it = mapped.begin(); it.ok(); ++it) {
if constexpr (!is_right_semi_anti_join) {
@@ -321,8 +328,7 @@ struct ProcessHashTableProbe {
}
++current_offset;
}
- if constexpr (need_to_set_visited)
- it->visited = true;
+ if constexpr (need_to_set_visited) it->visited = true;
}
}
} else {
@@ -345,10 +351,11 @@ struct ProcessHashTableProbe {
{
SCOPED_TIMER(_build_side_output_timer);
build_side_output_column(mcol, right_col_idx, right_col_len,
- _join_node->_right_output_slot_flags, current_offset);
+ _join_node->_right_output_slot_flags, current_offset);
}
- if constexpr (JoinOpType::value != TJoinOp::RIGHT_SEMI_JOIN && JoinOpType::value != TJoinOp::RIGHT_ANTI_JOIN) {
+ if constexpr (JoinOpType::value != TJoinOp::RIGHT_SEMI_JOIN &&
+ JoinOpType::value != TJoinOp::RIGHT_ANTI_JOIN) {
SCOPED_TIMER(_probe_side_output_timer);
probe_side_output_column(mcol, _join_node->_left_output_slot_flags, current_offset);
}
@@ -453,7 +460,7 @@ struct ProcessHashTableProbe {
{
SCOPED_TIMER(_build_side_output_timer);
build_side_output_column<true>(mcol, right_col_idx, right_col_len,
- _join_node->_right_output_slot_flags, current_offset);
+ _join_node->_right_output_slot_flags, current_offset);
}
{
SCOPED_TIMER(_probe_side_output_timer);
@@ -528,9 +535,11 @@ struct ProcessHashTableProbe {
auto new_filter_column = ColumnVector<UInt8>::create();
auto& filter_map = new_filter_column->get_data();
- if (!column->empty()) filter_map.emplace_back(column->get_bool(0) && visited_map[0]);
+ if (!column->empty())
+ filter_map.emplace_back(column->get_bool(0) && visited_map[0]);
for (int i = 1; i < column->size(); ++i) {
- if ((visited_map[i] && column->get_bool(i)) || (same_to_prev[i] && filter_map[i - 1])) {
+ if ((visited_map[i] && column->get_bool(i)) ||
+ (same_to_prev[i] && filter_map[i - 1])) {
filter_map.push_back(true);
filter_map[i - 1] = !same_to_prev[i] && filter_map[i - 1];
} else {
@@ -561,7 +570,8 @@ struct ProcessHashTableProbe {
output_block->clear();
} else {
if constexpr (JoinOpType::value == TJoinOp::LEFT_SEMI_JOIN ||
- JoinOpType::value == TJoinOp::LEFT_ANTI_JOIN) orig_columns = right_col_idx;
+ JoinOpType::value == TJoinOp::LEFT_ANTI_JOIN)
+ orig_columns = right_col_idx;
Block::filter_block(output_block, result_column_id, orig_columns);
}
}
@@ -597,11 +607,9 @@ struct ProcessHashTableProbe {
auto& mapped = iter->get_second();
for (auto it = mapped.begin(); it.ok(); ++it) {
if constexpr (JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN) {
- if (it->visited)
- insert_from_hash_table(it->block_offset, it->row_num);
+ if (it->visited) insert_from_hash_table(it->block_offset, it->row_num);
} else {
- if (!it->visited)
- insert_from_hash_table(it->block_offset, it->row_num);
+ if (!it->visited) insert_from_hash_table(it->block_offset, it->row_num);
}
}
}
@@ -612,7 +620,7 @@ struct ProcessHashTableProbe {
JoinOpType::value == TJoinOp::FULL_OUTER_JOIN) {
for (int i = 0; i < right_col_idx; ++i) {
for (int j = 0; j < block_size; ++j) {
- assert_cast<ColumnNullable *>(mcol[i].get())->insert_join_null_data();
+ assert_cast<ColumnNullable*>(mcol[i].get())->insert_join_null_data();
}
}
}
@@ -674,15 +682,15 @@ HashJoinNode::~HashJoinNode() = default;
void HashJoinNode::init_join_op() {
switch (_join_op) {
-#define M(NAME) \
- case TJoinOp::NAME: \
- _join_op_variants.emplace<std::integral_constant<TJoinOp::type, TJoinOp::NAME>>(); \
- break;
+#define M(NAME) \
+ case TJoinOp::NAME: \
+ _join_op_variants.emplace<std::integral_constant<TJoinOp::type, TJoinOp::NAME>>(); \
+ break;
APPLY_FOR_JOINOP_VARIANTS(M);
#undef M
- default:
- //do nothing
- break;
+ default:
+ //do nothing
+ break;
}
}
@@ -741,8 +749,8 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
}
for (const auto& filter_desc : _runtime_filter_descs) {
- RETURN_IF_ERROR(state->runtime_filter_mgr()->regist_filter(RuntimeFilterRole::PRODUCER,
- filter_desc, state->query_options()));
+ RETURN_IF_ERROR(state->runtime_filter_mgr()->regist_filter(
+ RuntimeFilterRole::PRODUCER, filter_desc, state->query_options()));
}
// init left/right output slots flags, only column of slot_id in _hash_output_slot_ids need
@@ -753,9 +761,10 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
auto init_output_slots_flags = [this](auto& tuple_descs, auto& output_slot_flags) {
for (const auto& tuple_desc : tuple_descs) {
for (const auto& slot_desc : tuple_desc->slots()) {
- output_slot_flags.emplace_back(_hash_output_slot_ids.empty() ||
- std::find(_hash_output_slot_ids.begin(), _hash_output_slot_ids.end(),
- slot_desc->id()) != _hash_output_slot_ids.end());
+ output_slot_flags.emplace_back(
+ _hash_output_slot_ids.empty() ||
+ std::find(_hash_output_slot_ids.begin(), _hash_output_slot_ids.end(),
+ slot_desc->id()) != _hash_output_slot_ids.end());
}
}
};
@@ -916,8 +925,8 @@ Status HashJoinNode::get_next(RuntimeState* state, Block* output_block, bool* eo
LOG(FATAL) << "FATAL: uninited hash table";
}
}
- }, _hash_table_variants,
- _join_op_variants,
+ },
+ _hash_table_variants, _join_op_variants,
make_bool_variant(_have_other_join_conjunct),
make_bool_variant(_probe_ignore_null));
} else if (_probe_eos) {
@@ -935,8 +944,7 @@ Status HashJoinNode::get_next(RuntimeState* state, Block* output_block, bool* eo
LOG(FATAL) << "FATAL: uninited hash table";
}
},
- _hash_table_variants,
- _join_op_variants);
+ _hash_table_variants, _join_op_variants);
} else {
*eos = true;
return Status::OK();
@@ -1022,6 +1030,7 @@ Status HashJoinNode::open(RuntimeState* state) {
RETURN_IF_ERROR(VExpr::open(_build_expr_ctxs, state));
RETURN_IF_ERROR(VExpr::open(_probe_expr_ctxs, state));
+ RETURN_IF_ERROR(VExpr::open(_output_expr_ctxs, state));
if (_vother_join_conjunct_ptr) {
RETURN_IF_ERROR((*_vother_join_conjunct_ptr)->open(state));
}
@@ -1051,7 +1060,9 @@ Status HashJoinNode::_hash_table_build(RuntimeState* state) {
_mem_used += block.allocated_bytes();
RETURN_IF_LIMIT_EXCEEDED(state, "Hash join, while getting next from the child 1.");
- if (block.rows() != 0) { mutable_block.merge(block); }
+ if (block.rows() != 0) {
+ mutable_block.merge(block);
+ }
// make one block for each 4 gigabytes
constexpr static auto BUILD_BLOCK_MAX_SIZE = 4 * 1024UL * 1024UL * 1024UL;
@@ -1100,7 +1111,7 @@ Status HashJoinNode::_extract_build_join_column(Block& block, NullMap& null_map,
// TODO: opt the column is const
block.get_by_position(result_col_id).column =
block.get_by_position(result_col_id).column->convert_to_full_column_if_const();
-
+
if (_is_null_safe_eq_join[i]) {
raw_ptrs[i] = block.get_by_position(result_col_id).column.get();
} else {
@@ -1209,7 +1220,7 @@ Status HashJoinNode::_process_build_block(RuntimeState* state, Block& block, uin
if constexpr (!std::is_same_v<HashTableCtxType, std::monostate>) {
#define CALL_BUILD_FUNCTION(HAS_NULL, BUILD_UNIQUE) \
ProcessHashTableBuild<HashTableCtxType, HAS_NULL, BUILD_UNIQUE> hash_table_build_process( \
- rows, block, raw_ptrs, this, state->batch_size(), offset); \
+ rows, block, raw_ptrs, this, state->batch_size(), offset); \
st = hash_table_build_process(arg, &null_map_val, has_runtime_filter);
if (std::pair {has_null, _build_unique} == std::pair {true, true}) {
CALL_BUILD_FUNCTION(true, true);
@@ -1332,13 +1343,25 @@ Status HashJoinNode::_build_output_block(Block* origin_block, Block* output_bloc
: MutableBlock(VectorizedUtils::create_empty_columnswithtypename(
_output_row_desc));
auto rows = origin_block->rows();
+ // TODO: After FE plan support same nullable of output expr and origin block and mutable column
+ // we should repalce `insert_column_datas` by `insert_range_from`
+
+ auto insert_column_datas = [](auto& to, const auto& from, size_t rows) {
+ if (to->is_nullable() && !from.is_nullable()) {
+ auto& null_column = reinterpret_cast<ColumnNullable&>(*to);
+ null_column.get_nested_column().insert_range_from(from, 0, rows);
+ null_column.get_null_map_column().get_data().resize_fill(rows, 0);
+ } else {
+ to->insert_range_from(from, 0, rows);
+ }
+ };
if (rows != 0) {
auto& mutable_columns = mutable_block.mutable_columns();
if (_output_expr_ctxs.empty()) {
DCHECK(mutable_columns.size() == origin_block->columns());
for (int i = 0; i < mutable_columns.size(); ++i) {
- mutable_columns[i]->insert_range_from(*origin_block->get_by_position(i).column, 0,
- rows);
+ insert_column_datas(mutable_columns[i], *origin_block->get_by_position(i).column,
+ rows);
}
} else {
DCHECK(mutable_columns.size() == _output_expr_ctxs.size());
@@ -1347,7 +1370,7 @@ Status HashJoinNode::_build_output_block(Block* origin_block, Block* output_bloc
RETURN_IF_ERROR(_output_expr_ctxs[i]->execute(origin_block, &result_column_id));
auto column_ptr = origin_block->get_by_position(result_column_id)
.column->convert_to_full_column_if_const();
- mutable_columns[i]->insert_range_from(*column_ptr, 0, rows);
+ insert_column_datas(mutable_columns[i], *column_ptr, rows);
}
}
diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp
index 8230d4d697..66f6037715 100644
--- a/be/src/vec/exec/vaggregation_node.cpp
+++ b/be/src/vec/exec/vaggregation_node.cpp
@@ -119,6 +119,11 @@ Status AggregationNode::init(const TPlanNode& tnode, RuntimeState* state) {
const auto& agg_functions = tnode.agg_node.aggregate_functions;
_is_merge = std::any_of(agg_functions.cbegin(), agg_functions.cend(),
[](const auto& e) { return e.nodes[0].agg_expr.is_merge_agg; });
+
+ // only corner case in query : https://github.com/apache/doris/issues/10302
+ // agg will do merge in update stage. in this case the merge function should use probe expr (slotref) column
+ // id to do merge like update function
+ _is_update_stage = tnode.agg_node.is_update_stage;
return Status::OK();
}
@@ -526,12 +531,14 @@ Status AggregationNode::_merge_without_key(Block* block) {
std::unique_ptr<char[]> deserialize_buffer(new char[_total_size_of_aggregate_states]);
int rows = block->rows();
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
- DCHECK(_aggregate_evaluators[i]->input_exprs_ctxs().size() == 1 &&
- _aggregate_evaluators[i]->input_exprs_ctxs()[0]->root()->is_slot_ref());
- int col_id =
- ((VSlotRef*)_aggregate_evaluators[i]->input_exprs_ctxs()[0]->root())->column_id();
if (_aggregate_evaluators[i]->is_merge()) {
- auto column = block->get_by_position(col_id).column;
+ auto column =
+ block->get_by_position(_is_update_stage ? ((VSlotRef*)_aggregate_evaluators[i]
+ ->input_exprs_ctxs()[0]
+ ->root())
+ ->column_id()
+ : i)
+ .column;
if (column->is_nullable()) {
column = ((ColumnNullable*)column.get())->get_nested_column_ptr();
}
@@ -1050,12 +1057,14 @@ Status AggregationNode::_merge_with_serialized_key(Block* block) {
std::unique_ptr<char[]> deserialize_buffer(new char[_total_size_of_aggregate_states]);
for (int i = 0; i < _aggregate_evaluators.size(); ++i) {
- DCHECK(_aggregate_evaluators[i]->input_exprs_ctxs().size() == 1 &&
- _aggregate_evaluators[i]->input_exprs_ctxs()[0]->root()->is_slot_ref());
- int col_id =
- ((VSlotRef*)_aggregate_evaluators[i]->input_exprs_ctxs()[0]->root())->column_id();
if (_aggregate_evaluators[i]->is_merge()) {
- auto column = block->get_by_position(col_id).column;
+ auto column =
+ block->get_by_position(_is_update_stage ? ((VSlotRef*)_aggregate_evaluators[i]
+ ->input_exprs_ctxs()[0]
+ ->root())
+ ->column_id()
+ : i + key_size)
+ .column;
if (column->is_nullable()) {
column = ((ColumnNullable*)column.get())->get_nested_column_ptr();
}
diff --git a/be/src/vec/exec/vaggregation_node.h b/be/src/vec/exec/vaggregation_node.h
index f020b90a6e..d46f856f8b 100644
--- a/be/src/vec/exec/vaggregation_node.h
+++ b/be/src/vec/exec/vaggregation_node.h
@@ -433,6 +433,7 @@ private:
bool _needs_finalize;
bool _is_merge;
+ bool _is_update_stage;
std::unique_ptr<MemPool> _mem_pool;
size_t _align_aggregate_states = 1;
diff --git a/be/src/vec/exprs/vslot_ref.h b/be/src/vec/exprs/vslot_ref.h
index 1bc78a4c5c..c00a018b3b 100644
--- a/be/src/vec/exprs/vslot_ref.h
+++ b/be/src/vec/exprs/vslot_ref.h
@@ -38,7 +38,6 @@ public:
virtual const std::string& expr_name() const override;
virtual std::string debug_string() const override;
virtual bool is_constant() const override { return false; }
-
const int column_id() const { return _column_id; }
private:
diff --git a/be/src/vec/functions/functions_logical.cpp b/be/src/vec/functions/functions_logical.cpp
index 8ef554fd9b..0c8bded880 100644
--- a/be/src/vec/functions/functions_logical.cpp
+++ b/be/src/vec/functions/functions_logical.cpp
@@ -204,7 +204,7 @@ class AssociativeGenericApplierImpl {
public:
/// Remembers the last N columns from `in`.
AssociativeGenericApplierImpl(const ColumnRawPtrs& in)
- : val_getter{ValueGetterBuilder::build(in[in.size() - N])}, next{in} {}
+ : val_getter {ValueGetterBuilder::build(in[in.size() - N])}, next {in} {}
/// Returns a combination of values in the i-th row of all columns stored in the constructor.
inline ResultValueType apply(const size_t i) const {
@@ -227,7 +227,7 @@ class AssociativeGenericApplierImpl<Op, 1> {
public:
/// Remembers the last N columns from `in`.
AssociativeGenericApplierImpl(const ColumnRawPtrs& in)
- : val_getter{ValueGetterBuilder::build(in[in.size() - 1])} {}
+ : val_getter {ValueGetterBuilder::build(in[in.size() - 1])} {}
inline ResultValueType apply(const size_t i) const { return val_getter(i); }
@@ -449,14 +449,20 @@ Status FunctionAnyArityLogical<Impl, Name>::execute_impl(FunctionContext* contex
size_t result_index,
size_t input_rows_count) {
ColumnRawPtrs args_in;
- for (const auto arg_index : arguments)
- args_in.push_back(block.get_by_position(arg_index).column.get());
+ bool is_nullable = false;
+ for (const auto arg_index : arguments) {
+ auto& data = block.get_by_position(arg_index);
+ args_in.push_back(data.column.get());
+ is_nullable |= data.column->is_nullable();
+ }
auto& result_info = block.get_by_position(result_index);
- if (result_info.type->is_nullable())
+ if (is_nullable) {
+ result_info.type = make_nullable(result_info.type);
execute_for_ternary_logic_impl<Impl>(std::move(args_in), result_info, input_rows_count);
- else
+ } else {
basic_execute_impl<Impl>(std::move(args_in), result_info, input_rows_count);
+ }
return Status::OK();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index 65ee4d6d8a..e7dee2651c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -277,6 +277,7 @@ public class AggregationNode extends PlanNode {
aggInfo.getIntermediateTupleId().asInt(),
aggInfo.getOutputTupleId().asInt(), needsFinalize);
msg.agg_node.setUseStreamingPreaggregation(useStreamingPreagg);
+ msg.agg_node.setIsUpdateStage(!aggInfo.isMerge());
List<Expr> groupingExprs = aggInfo.getGroupingExprs();
if (groupingExprs != null) {
msg.agg_node.setGroupingExprs(Expr.treesToThrift(groupingExprs));
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 34be03f8b6..dfe90968cb 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -464,6 +464,7 @@ struct TAggregationNode {
// rows have been aggregated, and this node is not an intermediate node.
5: required bool need_finalize
6: optional bool use_streaming_preaggregation
+ 7: optional bool is_update_stage
}
struct TRepeatNode {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/13: [log] add more error info for hdfs reader writer (#10475)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 926138c6a031677e44211e0f707ad404a6ff06d8
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Wed Jun 29 12:02:27 2022 +0800
[log] add more error info for hdfs reader writer (#10475)
---
be/src/exec/hdfs_file_reader.cpp | 19 ++++++++-----------
be/src/exec/hdfs_writer.cpp | 29 +++++++++++++++--------------
2 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index b042018e55..c71dc4a39a 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -62,21 +62,18 @@ Status HdfsFileReader::open() {
if (_hdfs_file == nullptr) {
std::stringstream ss;
ss << "open file failed. "
- << "(BE: " << BackendOptions::get_localhost() << ")" << _namenode << _path
- << ", err: " << strerror(errno);
- ;
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode:" << _namenode << ", path:" << _path << ", err: " << hdfsGetLastError();
return Status::InternalError(ss.str());
}
- LOG(INFO) << "open file. " << _namenode << _path;
+ VLOG_NOTICE << "open file, namenode:" << _namenode << ", path:" << _path;
return seek(_current_offset);
}
void HdfsFileReader::close() {
if (!closed()) {
if (_hdfs_file != nullptr && _hdfs_fs != nullptr) {
- std::stringstream ss;
- ss << "close hdfs file: " << _namenode << _path;
- LOG(INFO) << ss.str();
+ VLOG_NOTICE << "close hdfs file: " << _namenode << _path;
//If the hdfs file was valid, the memory associated with it will
// be freed at the end of this call, even if there was an I/O error
hdfsCloseFile(_hdfs_fs, _hdfs_file);
@@ -125,7 +122,7 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
std::stringstream ss;
ss << "hdfsSeek failed. "
<< "(BE: " << BackendOptions::get_localhost() << ")" << _namenode << _path
- << ", err: " << strerror(errno);
+ << ", err: " << hdfsGetLastError();
;
return Status::InternalError(ss.str());
}
@@ -136,7 +133,7 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
std::stringstream ss;
ss << "Read hdfs file failed. "
<< "(BE: " << BackendOptions::get_localhost() << ")" << _namenode << _path
- << ", err: " << strerror(errno);
+ << ", err: " << hdfsGetLastError();
;
return Status::InternalError(ss.str());
}
@@ -156,7 +153,7 @@ int64_t HdfsFileReader::size() {
hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str());
if (file_info == nullptr) {
LOG(WARNING) << "get path info failed: " << _namenode << _path
- << ", err: " << strerror(errno);
+ << ", err: " << hdfsGetLastError();
;
close();
return -1;
@@ -176,7 +173,7 @@ Status HdfsFileReader::seek(int64_t position) {
std::stringstream ss;
ss << "Seek to offset failed. "
<< "(BE: " << BackendOptions::get_localhost() << ")"
- << " offset=" << position << ", err: " << strerror(errno);
+ << " offset=" << position << ", err: " << hdfsGetLastError();
return Status::InternalError(ss.str());
}
return Status::OK();
diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp
index b45fbe4449..38bb98704e 100644
--- a/be/src/exec/hdfs_writer.cpp
+++ b/be/src/exec/hdfs_writer.cpp
@@ -64,13 +64,14 @@ Status HDFSWriter::open() {
std::string hdfs_dir = hdfs_path.parent_path().string();
exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str());
if (exists != 0) {
- LOG(INFO) << "hdfs dir doesn't exist, create it: " << hdfs_dir;
+ VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir;
int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str());
if (ret != 0) {
std::stringstream ss;
- ss << "create dir failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
- << " namenode: " << _namenode << " path: " << hdfs_dir
- << ", err: " << strerror(errno);
+ ss << "create dir failed. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode: " << _namenode << " path: " << hdfs_dir
+ << ", err: " << hdfsGetLastError();
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -79,13 +80,13 @@ Status HDFSWriter::open() {
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
- ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
- << " namenode:" << _namenode << " path:" << _path
- << ", err: " << strerror(errno);
+ ss << "open file failed. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
- LOG(INFO) << "open file. namenode:" << _namenode << " path:" << _path;
+ VLOG_NOTICE << "open file. namenode:" << _namenode << ", path:" << _path;
return Status::OK();
}
@@ -97,9 +98,9 @@ Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len
int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len);
if (result < 0) {
std::stringstream ss;
- ss << "write file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
- << "namenode:" << _namenode << " path:" << _path
- << ", err: " << strerror(errno);
+ ss << "write file failed. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -124,9 +125,9 @@ Status HDFSWriter::close() {
int result = hdfsFlush(_hdfs_fs, _hdfs_file);
if (result == -1) {
std::stringstream ss;
- ss << "failed to flush hdfs file. " << "(BE: " << BackendOptions::get_localhost() << ")"
- << "namenode:" << _namenode << " path:" << _path
- << ", err: " << strerror(errno);
+ ss << "failed to flush hdfs file. "
+ << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError();
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/13: [hotfix](dev-1.0.1) fix send fragment stuck bug
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6f649184be1b95b67d54c685ae12355d24bc6f9b
Author: morningman <mo...@163.com>
AuthorDate: Wed Jun 29 20:01:38 2022 +0800
[hotfix](dev-1.0.1) fix send fragment stuck bug
This bug has been fixed in #10214
---
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 75dc09b9f5..61eb8ecf00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -579,9 +579,11 @@ public class Coordinator {
int profileFragmentId = 0;
long memoryLimit = queryOptions.getMemLimit();
Map<Long, BackendExecStates> beToExecStates = Maps.newHashMap();
- // If #fragments >=3, use twoPhaseExecution with exec_plan_fragments_prepare and exec_plan_fragments_start,
+ // If #fragments >=2, use twoPhaseExecution with exec_plan_fragments_prepare and exec_plan_fragments_start,
// else use exec_plan_fragments directly.
- boolean twoPhaseExecution = fragments.size() >= 3;
+ // we choose #fragments >=2 because in some cases
+ // we need ensure that A fragment is already prepared to receive data before B fragment sends data.
+ boolean twoPhaseExecution = fragments.size() >= 2;
for (PlanFragment fragment : fragments) {
FragmentExecParams params = fragmentExecParamsMap.get(fragment.getFragmentId());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/13: [hotfix] add rm -f to avoid build failed
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1b03e68d1103877240af4188e12edde64fa33a5a
Author: morningman <mo...@163.com>
AuthorDate: Mon Jun 27 09:41:48 2022 +0800
[hotfix] add rm -f to avoid build failed
---
build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.sh b/build.sh
index 54cef6140a..0e537a7808 100755
--- a/build.sh
+++ b/build.sh
@@ -375,7 +375,7 @@ if [ ${BUILD_BE} -eq 1 ]; then
cp -r -p ${DORIS_HOME}/be/output/conf/* ${DORIS_OUTPUT}/be/conf/
cp -r -p ${DORIS_HOME}/be/output/lib/* ${DORIS_OUTPUT}/be/lib/
# make a soft link palo_be point to doris_be, for forward compatibility
- cd ${DORIS_OUTPUT}/be/lib && rm palo_be && ln -s doris_be palo_be && cd -
+ cd ${DORIS_OUTPUT}/be/lib && rm -f palo_be && ln -s doris_be palo_be && cd -
cp -r -p ${DORIS_HOME}/be/output/udf/*.a ${DORIS_OUTPUT}/udf/lib/
cp -r -p ${DORIS_HOME}/be/output/udf/include/* ${DORIS_OUTPUT}/udf/include/
cp -r -p ${DORIS_HOME}/webroot/be/* ${DORIS_OUTPUT}/be/www/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 11/13: [hotfix](dev-1.0.1) fix inline view bug for vec engine
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 108085330a7f0ff8083f501e53e12ae92d74905b
Author: EmmyMiao87 <52...@qq.com>
AuthorDate: Fri Jun 24 12:29:02 2022 +0800
[hotfix](dev-1.0.1) fix inline view bug for vec engine
---
be/src/vec/exec/join/vhash_join_node.cpp | 6 ++--
.../java/org/apache/doris/analysis/Analyzer.java | 13 +++++--
.../apache/doris/analysis/ExprSubstitutionMap.java | 4 +++
.../java/org/apache/doris/analysis/FromClause.java | 23 ++++++++++++-
.../org/apache/doris/analysis/InlineViewRef.java | 1 +
.../org/apache/doris/planner/HashJoinNode.java | 40 ++++++++++++++++++++--
.../apache/doris/planner/SingleNodePlanner.java | 12 +++----
7 files changed, 84 insertions(+), 15 deletions(-)
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index 37d31a0a13..c8d7f6e77c 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -1172,15 +1172,15 @@ Status HashJoinNode::_extract_probe_join_column(Block& block, NullMap& null_map,
Status HashJoinNode::_process_build_block(RuntimeState* state, Block& block, uint8_t offset) {
SCOPED_TIMER(_build_table_timer);
+ if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) {
+ _convert_block_to_null(block);
+ }
size_t rows = block.rows();
if (UNLIKELY(rows == 0)) {
return Status::OK();
}
COUNTER_UPDATE(_build_rows_counter, rows);
- if (_join_op == TJoinOp::LEFT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) {
- _convert_block_to_null(block);
- }
ColumnRawPtrs raw_ptrs(_build_expr_ctxs.size());
NullMap null_map_val(rows);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index d05cb2685d..fa40b57d2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -217,8 +217,9 @@ public class Analyzer {
private final Map<TupleId, List<ExprId>> eqJoinConjuncts = Maps.newHashMap();
// set of conjuncts that have been assigned to some PlanNode
- private Set<ExprId> assignedConjuncts =
- Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());
+ private Set<ExprId> assignedConjuncts = Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());
+
+ private Set<TupleId> inlineViewTupleIds = Sets.newHashSet();
// map from outer-joined tuple id, ie, one that is nullable in this select block,
// to the last Join clause (represented by its rhs table ref) that outer-joined it
@@ -841,6 +842,10 @@ public class Analyzer {
return result;
}
+ public void registerInlineViewTupleId(TupleId tupleId) {
+ globalState.inlineViewTupleIds.add(tupleId);
+ }
+
/**
* Register conjuncts that are outer joined by a full outer join. For a given
* predicate, we record the last full outer join that outer-joined any of its
@@ -2068,6 +2073,10 @@ public class Analyzer {
return globalState.outerJoinedTupleIds.containsKey(tid);
}
+ public boolean isInlineView(TupleId tid) {
+ return globalState.inlineViewTupleIds.contains(tid);
+ }
+
public boolean containSubquery() {
return globalState.containsSubquery;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
index 20e68cf5c3..062eef4df8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
@@ -106,6 +106,10 @@ public final class ExprSubstitutionMap {
}
}
+ public void updateLhsExprs(List<Expr> lhsExprList) {
+ lhs_ = lhsExprList;
+ }
+
/**
* Return a map which is equivalent to applying f followed by g,
* i.e., g(f()).
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
index e77fd6ac21..fa6242f01f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
@@ -147,8 +147,11 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
tblRef.analyze(analyzer);
leftTblRef = tblRef;
}
-
checkExternalTable(analyzer);
+
+ // Fix the problem of column nullable attribute error caused by inline view + outer join
+ changeTblRefToNullable(analyzer);
+
// TODO: remove when query from hive table is supported
checkFromHiveTable(analyzer);
@@ -184,6 +187,24 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
}
}
+ // set null-side inlinve view column
+ // For example: select * from (select a as k1 from t) tmp right join b on tmp.k1=b.k1
+ // The columns from tmp should be nullable.
+ // The table ref tmp will be used by HashJoinNode.computeOutputTuple()
+ private void changeTblRefToNullable(Analyzer analyzer) {
+ for (TableRef tableRef : tableRefs_) {
+ if (!(tableRef instanceof InlineViewRef)) {
+ continue;
+ }
+ InlineViewRef inlineViewRef = (InlineViewRef) tableRef;
+ if (analyzer.isOuterJoined(inlineViewRef.getId())) {
+ for (SlotDescriptor slotDescriptor : inlineViewRef.getDesc().getSlots()) {
+ slotDescriptor.setIsNullable(true);
+ }
+ }
+ }
+ }
+
public FromClause clone() {
ArrayList<TableRef> clone = Lists.newArrayList();
for (TableRef tblRef : tableRefs_) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
index 7bf43b78b6..01791ed17f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
@@ -286,6 +286,7 @@ public class InlineViewRef extends TableRef {
TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor();
result.setIsMaterialized(false);
result.setTable(inlineView);
+ analyzer.registerInlineViewTupleId(result.getId());
return result;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index 0534209275..52c623f3e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -28,10 +28,10 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
+import org.apache.doris.analysis.TupleIsNullPredicate;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.CheckedMath;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.Pair;
@@ -320,7 +320,7 @@ public class HashJoinNode extends PlanNode {
}
}
- private void computeOutputTuple(Analyzer analyzer) throws AnalysisException {
+ private void computeOutputTuple(Analyzer analyzer) throws UserException {
// 1. create new tuple
vOutputTupleDesc = analyzer.getDescTbl().createTupleDescriptor();
boolean copyLeft = false;
@@ -362,6 +362,8 @@ public class HashJoinNode extends PlanNode {
break;
}
ExprSubstitutionMap srcTblRefToOutputTupleSmap = new ExprSubstitutionMap();
+ int leftNullableNumber = 0;
+ int rightNullableNumber = 0;
if (copyLeft) {
for (TupleDescriptor leftTupleDesc : analyzer.getDescTbl().getTupleDesc(getChild(0).getOutputTblRefIds())) {
for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) {
@@ -372,6 +374,7 @@ public class HashJoinNode extends PlanNode {
analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, leftSlotDesc);
if (leftNullable) {
outputSlotDesc.setIsNullable(true);
+ leftNullableNumber++;
}
srcTblRefToOutputTupleSmap.put(new SlotRef(leftSlotDesc), new SlotRef(outputSlotDesc));
}
@@ -388,6 +391,7 @@ public class HashJoinNode extends PlanNode {
analyzer.getDescTbl().copySlotDescriptor(vOutputTupleDesc, rightSlotDesc);
if (rightNullable) {
outputSlotDesc.setIsNullable(true);
+ rightNullableNumber++;
}
srcTblRefToOutputTupleSmap.put(new SlotRef(rightSlotDesc), new SlotRef(outputSlotDesc));
}
@@ -406,7 +410,37 @@ public class HashJoinNode extends PlanNode {
}
}
vOutputTupleDesc.computeStatAndMemLayout();
- // 3. change the outputSmap
+ // 3. add tupleisnull in null-side
+ Preconditions.checkState(srcTblRefToOutputTupleSmap.getLhs().size() == vSrcToOutputSMap.getLhs().size());
+ // Condition1: the left child is null-side
+ // Condition2: the left child is a inline view
+ // Then: add tuple is null in left child columns
+ if (leftNullable && getChild(0).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(0).tblRefIds.get(0))) {
+ List<Expr> tupleIsNullLhs = TupleIsNullPredicate
+ .wrapExprs(vSrcToOutputSMap.getLhs().subList(0, leftNullableNumber), getChild(0).getTupleIds(),
+ analyzer);
+ tupleIsNullLhs
+ .addAll(vSrcToOutputSMap.getLhs().subList(leftNullableNumber, vSrcToOutputSMap.getLhs().size()));
+ vSrcToOutputSMap.updateLhsExprs(tupleIsNullLhs);
+ }
+ // Condition1: the right child is null-side
+ // Condition2: the right child is a inline view
+ // Then: add tuple is null in right child columns
+ if (rightNullable && getChild(1).tblRefIds.size() == 1 && analyzer.isInlineView(getChild(1).tblRefIds.get(0))) {
+ if (rightNullableNumber != 0) {
+ int rightBeginIndex = vSrcToOutputSMap.size() - rightNullableNumber;
+ List<Expr> tupleIsNullLhs = TupleIsNullPredicate
+ .wrapExprs(vSrcToOutputSMap.getLhs().subList(rightBeginIndex, vSrcToOutputSMap.size()),
+ getChild(1).getTupleIds(), analyzer);
+ List<Expr> newLhsList = Lists.newArrayList();
+ if (rightBeginIndex > 0) {
+ newLhsList.addAll(vSrcToOutputSMap.getLhs().subList(0, rightBeginIndex));
+ }
+ newLhsList.addAll(tupleIsNullLhs);
+ vSrcToOutputSMap.updateLhsExprs(newLhsList);
+ }
+ }
+ // 4. change the outputSmap
outputSmap = ExprSubstitutionMap.combineAndReplace(outputSmap, srcTblRefToOutputTupleSmap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 1f4a90a3ee..c804ed624f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -59,6 +59,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.VectorizedUtil;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@@ -66,7 +67,6 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -1347,10 +1347,10 @@ public class SingleNodePlanner {
unionNode.addConstExprList(selectStmt.getBaseTblResultExprs());
unionNode.init(analyzer);
//set outputSmap to substitute literal in outputExpr
- unionNode.setWithoutTupleIsNullOutputSmap(inlineViewRef.getSmap());
- if (analyzer.isOuterJoined(inlineViewRef.getId())) {
- List<Expr> nullableRhs = TupleIsNullPredicate.wrapExprs(
- inlineViewRef.getSmap().getRhs(), unionNode.getTupleIds(), analyzer);
+ if (analyzer.isOuterJoined(inlineViewRef.getId()) && !VectorizedUtil.isVectorized()) {
+ unionNode.setWithoutTupleIsNullOutputSmap(inlineViewRef.getSmap());
+ List<Expr> nullableRhs = TupleIsNullPredicate
+ .wrapExprs(inlineViewRef.getSmap().getRhs(), unionNode.getTupleIds(), analyzer);
unionNode.setOutputSmap(new ExprSubstitutionMap(inlineViewRef.getSmap().getLhs(), nullableRhs));
}
return unionNode;
@@ -1370,7 +1370,7 @@ public class SingleNodePlanner {
ExprSubstitutionMap outputSmap = ExprSubstitutionMap.compose(
inlineViewRef.getSmap(), rootNode.getOutputSmap(), analyzer);
- if (analyzer.isOuterJoined(inlineViewRef.getId())) {
+ if (analyzer.isOuterJoined(inlineViewRef.getId()) && !VectorizedUtil.isVectorized()) {
rootNode.setWithoutTupleIsNullOutputSmap(outputSmap);
// Exprs against non-matched rows of an outer join should always return NULL.
// Make the rhs exprs of the output smap nullable, if necessary. This expr wrapping
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/13: [hotfix] change hive, broker, iceberg table query to non-vec engine
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 36bbe02f96cf2dd6d8370b5ff7cc5cee0a924c5d
Author: morningman <mo...@163.com>
AuthorDate: Fri Jun 24 13:03:04 2022 +0800
[hotfix] change hive, broker, iceberg table query to non-vec engine
---
.../java/org/apache/doris/analysis/FromClause.java | 37 +++++++++++++++++++++-
...ectorizedUtil.java => VecNotImplException.java} | 21 +++---------
.../apache/doris/common/util/VectorizedUtil.java | 19 +++++++++++
.../java/org/apache/doris/qe/StmtExecutor.java | 13 ++++++--
thirdparty/vars.sh | 2 +-
5 files changed, 70 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
index be71841967..e77fd6ac21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
@@ -20,11 +20,14 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.VecNotImplException;
+import org.apache.doris.common.util.VectorizedUtil;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -145,15 +148,47 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
leftTblRef = tblRef;
}
+ checkExternalTable(analyzer);
// TODO: remove when query from hive table is supported
checkFromHiveTable(analyzer);
analyzed_ = true;
}
+ private void checkExternalTable(Analyzer analyzer) throws UserException {
+ for (TableRef tblRef : tableRefs_) {
+ if (!(tblRef instanceof BaseTableRef)) {
+ continue;
+ }
+
+ TableName tableName = tblRef.getName();
+ String dbName = tableName.getDb();
+ if (Strings.isNullOrEmpty(dbName)) {
+ dbName = analyzer.getDefaultDb();
+ } else {
+ dbName = ClusterNamespace.getFullName(analyzer.getClusterName(), tblRef.getName().getDb());
+ }
+ if (Strings.isNullOrEmpty(dbName)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
+ }
+
+ Database db = analyzer.getCatalog().getDbOrAnalysisException(dbName);
+ String tblName = tableName.getTbl();
+ Table table = db.getTableOrAnalysisException(tblName);
+ if (VectorizedUtil.isVectorized()) {
+ if (table.getType() == TableType.BROKER || table.getType() == TableType.HIVE
+ || table.getType() == TableType.ICEBERG) {
+ throw new VecNotImplException("Not support table type " + table.getType() + " in vec engine");
+ }
+ }
+ }
+ }
+
public FromClause clone() {
ArrayList<TableRef> clone = Lists.newArrayList();
- for (TableRef tblRef: tableRefs_) clone.add(tblRef.clone());
+ for (TableRef tblRef : tableRefs_) {
+ clone.add(tblRef.clone());
+ }
return new FromClause(clone);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/VecNotImplException.java
similarity index 53%
copy from fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java
copy to fe/fe-core/src/main/java/org/apache/doris/common/VecNotImplException.java
index 6146bbc5a8..2c5d12e7d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/VecNotImplException.java
@@ -15,23 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.common.util;
+package org.apache.doris.common;
-import org.apache.doris.qe.ConnectContext;
-
-public class VectorizedUtil {
- /**
- * 1. Return false if there is no current connection (Rule1 to be changed)
- * 2. Returns the vectorized switch value of the query 'globalState.enableQueryVec'
- * 3. If it is not currently a query, return the vectorized switch value of the session 'enableVectorizedEngine'
- * @return true: vec. false: non-vec
- */
- public static boolean isVectorized() {
- ConnectContext connectContext = ConnectContext.get();
- if (connectContext == null) {
- return false;
- }
- return connectContext.getSessionVariable().enableVectorizedEngine();
+public class VecNotImplException extends UserException {
+ public VecNotImplException(String msg) {
+ super(msg);
}
}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java
index 6146bbc5a8..7e3a501ebd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java
@@ -17,7 +17,12 @@
package org.apache.doris.common.util;
+import org.apache.doris.analysis.SetVar;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.common.DdlException;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.VariableMgr;
public class VectorizedUtil {
/**
@@ -33,5 +38,19 @@ public class VectorizedUtil {
}
return connectContext.getSessionVariable().enableVectorizedEngine();
}
+
+ public static void switchToQueryNonVec() {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext == null) {
+ return;
+ }
+ SessionVariable sessionVariable = connectContext.getSessionVariable();
+ sessionVariable.setIsSingleSetVar(true);
+ try {
+ VariableMgr.setVar(sessionVariable, new SetVar("enable_vectorized_engine", new StringLiteral("false")));
+ } catch (DdlException e) {
+ // do nothing
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 7531f00e45..db4251206b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -67,6 +67,7 @@ import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.VecNotImplException;
import org.apache.doris.common.Version;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.MetaLockUtils;
@@ -76,6 +77,7 @@ import org.apache.doris.common.util.QueryPlannerProfile;
import org.apache.doris.common.util.RuntimeProfile;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.common.util.VectorizedUtil;
import org.apache.doris.load.EtlJobType;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.MysqlChannel;
@@ -118,7 +120,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-
+import com.google.protobuf.ByteString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TException;
@@ -137,8 +139,6 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
-import com.google.protobuf.ByteString;
-
// Do one COM_QUERY process.
// first: Parse receive byte array to statement struct.
// second: Do handle function for statement.
@@ -594,6 +594,13 @@ public class StmtExecutor implements ProfileWriter {
} else {
resetAnalyzerAndStmt();
}
+ } catch (VecNotImplException e) {
+ if (i == analyzeTimes) {
+ throw e;
+ } else {
+ resetAnalyzerAndStmt();
+ VectorizedUtil.switchToQueryNonVec();
+ }
} catch (UserException e) {
throw e;
} catch (Exception e) {
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index d0121cfc95..8d9ce00628 100755
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -45,7 +45,7 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
# source of all dependencies, default unuse it
-export REPOSITORY_URL=https://doris-thirdparty-hk-1308700295.cos.ap-hongkong.myqcloud.com/thirdparty
+export REPOSITORY_URL=
#####################################################
# Download url, filename and unpaced filename
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/13: [hotfix](dev-1.0.1) modify stop_be and disable meta_tool build by default
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7caa1eb6abc342b1ee6caddffdc7bf8bd72af7d2
Author: morningman <mo...@163.com>
AuthorDate: Wed Jun 29 15:00:30 2022 +0800
[hotfix](dev-1.0.1) modify stop_be and disable meta_tool build by default
---
bin/stop_be.sh | 2 +-
build.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/bin/stop_be.sh b/bin/stop_be.sh
index 2ca1b47124..d791fb9ff2 100755
--- a/bin/stop_be.sh
+++ b/bin/stop_be.sh
@@ -35,7 +35,7 @@ pidfile=$PID_DIR/be.pid
if [ -f $pidfile ]; then
pid=`cat $pidfile`
pidcomm=`ps -p $pid -o comm=`
- if [ "doris_be"x != "$pidcomm"x ]; then
+ if [ "doris_be"x != "$pidcomm"x -a "palo_be"x != "$pidcomm"x ]; then
echo "ERROR: pid process may not be be. "
exit 1
fi
diff --git a/build.sh b/build.sh
index 6441dab50f..7d1071fd19 100755
--- a/build.sh
+++ b/build.sh
@@ -57,7 +57,7 @@ Usage: $0 <options>
Environment variables:
USE_AVX2 If the CPU does not support AVX2 instruction set, please set USE_AVX2=0. Default is ON.
- BUILD_META_TOOL If set BUILD_META_TOOL=OFF, the output meta_tools binaries will not be compiled. Default is ON.
+ BUILD_META_TOOL If set BUILD_META_TOOL=OFF, the output meta_tools binaries will not be compiled. Default is OFF.
STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Default is OFF.
Eg.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/13: [improvement]change shard num of page cache to 1024 to reduce lock contention (#10449)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4ff62b32b43873889ec92a92dbfee05a818b0c71
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Tue Jun 28 13:54:10 2022 +0800
[improvement]change shard num of page cache to 1024 to reduce lock contention (#10449)
Because shard num of master is changed to be as a config, so here
we add a isolation commit to 1.0.1.
---
be/src/olap/lru_cache.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 0c4bf69795..0ff3ce16a6 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -357,7 +357,7 @@ private:
uint64_t _hit_count = 0; // 命中cache的总次数
};
-static const int kNumShardBits = 4;
+static const int kNumShardBits = 10;
static const int kNumShards = 1 << kNumShardBits;
class ShardedLRUCache : public Cache {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/13: [hotfix](dev-1.0.1) remove incubating
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 40a5673d4c62e1e34dce1bfb84d6ed3078efd161
Author: morningman <mo...@163.com>
AuthorDate: Wed Jun 29 12:26:00 2022 +0800
[hotfix](dev-1.0.1) remove incubating
---
CONTRIBUTING.md | 20 ++++++++++----------
CONTRIBUTING_CN.md | 20 ++++++++++----------
DISCLAIMER | 12 ------------
NOTICE.txt | 2 +-
README.md | 20 ++++++++++----------
build.sh | 2 +-
fe/pom.xml | 10 +++++-----
fs_brokers/apache_hdfs_broker/pom.xml | 10 +++++-----
8 files changed, 42 insertions(+), 54 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 669c2d2f91..f8dcfe0c24 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -21,7 +21,7 @@ under the License.
Thank you very much for your interest in the Doris project. We welcome your suggestions, comments (including criticisms), comments and contributions to the Doris project.
-Your suggestions, comments and comments on Doris can be made directly through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/selection).
+Your suggestions, comments and comments on Doris can be made directly through GitHub's [Issues] (https://github.com/apache/doris/issues/new/selection).
There are many ways to participate in and contribute to Doris projects: code implementation, test writing, process tool improvement, document improvement, and so on. Any contribution will be welcomed and you will be added to the list of contributors. Further, with sufficient contributions, you will have the opportunity to become a Commiter of Apache with Apache mailbox and be included in the list of [Apache Commiters] (http://people.apache.org/committer-index.html).
@@ -31,7 +31,7 @@ Any questions, you can contact us to get timely answers, including dev mail list
For the first time in Doris community, you can:
-* Follow [Doris Github](https://github.com/apache/incubator-doris)
+* Follow [Doris Github](https://github.com/apache/doris)
* Subscribe to our [mailing list] (./subscribe-mail-list.md);
* Join Doris [Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-11jb8gesh-7IukzSrdea6mqoG0HB4gZg)
@@ -39,15 +39,15 @@ Learn the development trends of Doris project in time and give your opinions on
## Doris's code and documentation
-As you can see from [GitHub] (https://github.com/apache/incubator-doris), Apache Doris (incubating) code base mainly consists of three parts: Frontend (FE), Backend (BE) and Broker (to support file reading on external storage systems such as HDFS). Documents are mainly the wiki on Doris website and GitHub, as well as the online help manual when running Doris. Details of these components can be found in the following table:
+As you can see from [GitHub] (https://github.com/apache/doris), Apache Doris code base mainly consists of three parts: Frontend (FE), Backend (BE) and Broker (to support file reading on external storage systems such as HDFS). Documents are mainly the wiki on Doris website and GitHub, as well as the online help manual when running Doris. Details of these components can be found in the following table:
| Component Name | Component Description | Related Language|
|--------|----------------------------|----------|
-| [Frontend daemon (FE)] (https://github.com/apache/incubator-doris) | consists of a query coordinator and a metadata manager | Java|
-| [Backend daemon (BE)] (https://github.com/apache/incubator-doris) | Responsible for storing data and executing query fragments | C++|
-| [Broker] (https://github.com/apache/incubator-doris) | Read HDFS data to Doris | Java|
-| [Website](https://github.com/apache/incubator-doris-website) | Doris Website | Markdown |
-+ [Github Wiki] (https://github.com/apache/incubator-doris/wiki); Doris Github Wiki; Markdown_;
+| [Frontend daemon (FE)] (https://github.com/apache/doris) | consists of a query coordinator and a metadata manager | Java|
+| [Backend daemon (BE)] (https://github.com/apache/doris) | Responsible for storing data and executing query fragments | C++|
+| [Broker] (https://github.com/apache/doris) | Read HDFS data to Doris | Java|
+| [Website](https://github.com/apache/doris-website) | Doris Website | Markdown |
++ [Github Wiki] (https://github.com/apache/doris/wiki); Doris Github Wiki; Markdown_;
| Doris Runtime Help Document | Online Help Manual at Doris Runtime | Markdown|
## Improving documentation
@@ -58,11 +58,11 @@ Browse the document, you can deepen your understanding of Doris, can also help y
If you are interested in improving the quality of documents, whether it is revising the address of a page, correcting a link, and writing a better introductory document, we are very welcome!
-Most of our documents are written in markdown format, and you can modify and submit document changes directly through `docs/` in [GitHub] (https://github.com/apache/incubator-doris). If you submit code changes, you can refer to [Pull Request] (./pull-request.md).
+Most of our documents are written in markdown format, and you can modify and submit document changes directly through `docs/` in [GitHub] (https://github.com/apache/doris). If you submit code changes, you can refer to [Pull Request] (./pull-request.md).
## If a Bug or problem is found
-If a Bug or problem is found, you can directly raise a new Issue through GitHub's [Issues] (https://github.com/apache/incubator-doris/issues/new/select), and we will have someone deal with it regularly.
+If a Bug or problem is found, you can directly raise a new Issue through GitHub's [Issues] (https://github.com/apache/doris/issues/new/select), and we will have someone deal with it regularly.
You can also fix it yourself by reading the analysis code (of course, it's better to talk to us before that, maybe someone has fixed the same problem) and submit a [Pull Request] (./pull-request.md).
diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md
index 7de688f06e..111f10823f 100644
--- a/CONTRIBUTING_CN.md
+++ b/CONTRIBUTING_CN.md
@@ -21,7 +21,7 @@ under the License.
非常感谢您对 Doris 项目感兴趣，我们非常欢迎您对 Doris 项目的各种建议、意见（包括批评）、评论和贡献。
-您对 Doris 的各种建议、意见、评论可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提出。
+您对 Doris 的各种建议、意见、评论可以直接通过 GitHub 的 [Issues](https://github.com/apache/doris/issues/new/choose) 提出。
参与 Doris 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表，进一步，有了足够的贡献后，您还可以有机会成为 Apache 的 Commiter，拥有 Apache 邮箱，并被收录到 [Apache Commiter 列表中](http://people.apache.org/committer-index.html)。
@@ -31,7 +31,7 @@ under the License.
初次来到 Doris 社区，您可以：
-* 关注 Doris [Github 代码库](https://github.com/apache/incubator-doris)
+* 关注 Doris [Github 代码库](https://github.com/apache/doris)
* 订阅我们的 [邮件列表](./subscribe-mail-list.md)；
* 加入 Doris 的 [Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-11jb8gesh-7IukzSrdea6mqoG0HB4gZg)
@@ -39,15 +39,15 @@ under the License.
## Doris 的代码和文档
-正如您在 [GitHub] (https://github.com/apache/incubator-doris) 上看到的，Apache Doris (incubating) 的代码库主要包括三部分：Frontend (FE), Backend (BE) 和 Broker (为了支持 HDFS 等外部存储系统上的文件读取)。文档主要是 Doris 网站和 GitHub 上的 wiki，还有运行 Doris 的时候的在线帮助手册。这些组件的详细情况参见下表：
+正如您在 [GitHub] (https://github.com/apache/doris) 上看到的，Apache Doris 的代码库主要包括三部分：Frontend (FE), Backend (BE) 和 Broker (为了支持 HDFS 等外部存储系统上的文件读取)。文档主要是 Doris 网站和 GitHub 上的 wiki，还有运行 Doris 的时候的在线帮助手册。这些组件的详细情况参见下表：
| 组件名称 | 组件描述 | 相关语言 |
|--------|----------------------------|----------|
-| [Frontend daemon (FE)](https://github.com/apache/incubator-doris)| 由“查询协调器”和“元数据管理器”组成 | Java|
-| [Backend daemon (BE)](https://github.com/apache/incubator-doris) | 负责存储数据和执行查询片段 | C++|
-| [Broker](https://github.com/apache/incubator-doris) | 读取 HDFS 数据到 Doris | Java |
-| [Website](https://github.com/apache/incubator-doris-website) | Doris 网站 | Markdown |
-| [GitHub Wiki](https://github.com/apache/incubator-doris/wiki) | Doris GitHub Wiki | Markdown |
+| [Frontend daemon (FE)](https://github.com/apache/doris)| 由“查询协调器”和“元数据管理器”组成 | Java|
+| [Backend daemon (BE)](https://github.com/apache/doris) | 负责存储数据和执行查询片段 | C++|
+| [Broker](https://github.com/apache/doris) | 读取 HDFS 数据到 Doris | Java |
+| [Website](https://github.com/apache/doris-website) | Doris 网站 | Markdown |
+| [GitHub Wiki](https://github.com/apache/doris/wiki) | Doris GitHub Wiki | Markdown |
| Doris 运行时 help 文档 | 运行 Doris 的时候的在线帮助手册 | Markdown |
## 改进文档
@@ -58,11 +58,11 @@ under the License.
如果您对改进文档的质量感兴趣，不论是修订一个页面的地址、更正一个链接、以及写一篇更优秀的入门文档，我们都非常欢迎！
-我们的文档大多数是使用 markdown 格式编写的，您可以直接通过在 [GitHub] (https://github.com/apache/incubator-doris) 中的 `docs/` 中修改并提交文档变更。如果提交代码变更，可以参阅 [Pull Request](./pull-request.md)。
+我们的文档大多数是使用 markdown 格式编写的，您可以直接通过在 [GitHub] (https://github.com/apache/doris) 中的 `docs/` 中修改并提交文档变更。如果提交代码变更，可以参阅 [Pull Request](./pull-request.md)。
## 如果发现了一个 Bug 或问题
-如果发现了一个 Bug 或问题，您可以直接通过 GitHub 的 [Issues](https://github.com/apache/incubator-doris/issues/new/choose) 提一个新的 Issue，我们会有人定期处理。
+如果发现了一个 Bug 或问题，您可以直接通过 GitHub 的 [Issues](https://github.com/apache/doris/issues/new/choose) 提一个新的 Issue，我们会有人定期处理。
您也可以通过阅读分析代码自己修复（当然在这之前最好能和我们交流下，或许已经有人在修复同样的问题了），然后提交一个 [Pull Request](./pull-request.md)。
diff --git a/DISCLAIMER b/DISCLAIMER
deleted file mode 100644
index bcae8b923f..0000000000
--- a/DISCLAIMER
+++ /dev/null
@@ -1,12 +0,0 @@
-Apache Doris (incubating) is an effort undergoing incubation at The
-Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted
-projects until a further review indicates that the
-infrastructure, communications, and decision making process have
-stabilized in a manner consistent with other successful ASF
-projects.
-
-While incubation status is not necessarily a reflection
-of the completeness or stability of the code, it does indicate
-that the project has yet to be fully endorsed by the ASF.
diff --git a/NOTICE.txt b/NOTICE.txt
index 7f0561b0f5..a14d712816 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,4 +1,4 @@
-Apache Doris (incubating)
+Apache Doris
Copyright 2018-2022 The Apache Software Foundation
This product includes software developed at
diff --git a/README.md b/README.md
index 53d195916c..2b1a046e2c 100644
--- a/README.md
+++ b/README.md
@@ -17,10 +17,10 @@ specific language governing permissions and limitations
under the License.
-->
-# Apache Doris (incubating)
+# Apache Doris
[](https://www.apache.org/licenses/LICENSE-2.0.html)
-[](https://github.com/apache/incubator-doris)
-[](https://github.com/apache/incubator-doris/releases)
+[](https://github.com/apache/doris)
+[](https://github.com/apache/doris/releases)
[](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-11jb8gesh-7IukzSrdea6mqoG0HB4gZg)
[](https://gitter.im/apache-doris/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -33,13 +33,13 @@ Its original name was Palo, developed in Baidu. After donated to Apache Software
- Doris provides high availability, reliability, fault tolerance, and scalability.
-The main advantages of Doris are the simplicity (of developing, deploying and using) and meeting many data serving requirements in a single system. For details, refer to [Overview](https://github.com/apache/incubator-doris/wiki/Doris-Overview).
+The main advantages of Doris are the simplicity (of developing, deploying and using) and meeting many data serving requirements in a single system. For details, refer to [Overview](https://github.com/apache/doris/wiki/Doris-Overview).
**Official website: https://doris.apache.org/**
-[](https://www.apiseven.com/en/contributor-graph?chart=contributorMonthlyActivity&repo=apache/incubator-doris)
+[](https://www.apiseven.com/en/contributor-graph?chart=contributorMonthlyActivity&repo=apache/doris)
-[](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/incubator-doris)
+[](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/doris)
## License
@@ -55,15 +55,15 @@ Doris mainly integrates the technology of [Google Mesa](https://research.google/
## Compile and install
-See [Compilation](https://doris.incubator.apache.org/installing/compilation-with-ldb-toolchain.html)
+See [Compilation](https://doris.apache.org/installing/compilation-with-ldb-toolchain.html)
## Getting start
-See [Basic Usage](https://doris.incubator.apache.org/getting-started/basic-usage.html)
+See [Basic Usage](https://doris.apache.org/getting-started/basic-usage.html)
## Report issues or submit pull request
-If you find any bugs, feel free to file a [GitHub issue](https://github.com/apache/incubator-doris/issues) or fix it by submitting a [pull request](https://github.com/apache/incubator-doris/pulls).
+If you find any bugs, feel free to file a [GitHub issue](https://github.com/apache/doris/issues) or fix it by submitting a [pull request](https://github.com/apache/doris/pulls).
## Contact Us
@@ -75,6 +75,6 @@ Contact us through the following mailing list.
## Links
-* Doris official site - <http://doris.incubator.apache.org>
+* Doris official site - <http://doris.apache.org>
* Developer Mailing list - <de...@doris.apache.org>. Mail to <de...@doris.apache.org>, follow the reply to subscribe the mail list.
* Slack channel - [Join the Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-11jb8gesh-7IukzSrdea6mqoG0HB4gZg)
diff --git a/build.sh b/build.sh
index 0e537a7808..6441dab50f 100755
--- a/build.sh
+++ b/build.sh
@@ -17,7 +17,7 @@
# under the License.
##############################################################
-# This script is used to compile Apache Doris(incubating).
+# This script is used to compile Apache Doris
# Usage:
# sh build.sh --help
# Eg:
diff --git a/fe/pom.xml b/fe/pom.xml
index be3cec77ac..ec390c9c26 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -39,14 +39,14 @@ under the License.
</license>
</licenses>
<scm>
- <connection>scm:git:https://git@github.com/apache/incubator-doris.git</connection>
- <developerConnection>scm:git:https://git@github.com/apache/incubator-doris.git</developerConnection>
- <url>scm:git:https://git@github.com/apache/incubator-doris.git</url>
+ <connection>scm:git:https://git@github.com/apache/doris.git</connection>
+ <developerConnection>scm:git:https://git@github.com/apache/doris.git</developerConnection>
+ <url>scm:git:https://git@github.com/apache/doris.git</url>
<tag>HEAD</tag>
</scm>
<issueManagement>
<system>GitHub</system>
- <url>https://github.com/apache/incubator-doris/issues</url>
+ <url>https://github.com/apache/doris/issues</url>
</issueManagement>
<mailingLists>
<mailingList>
@@ -196,7 +196,7 @@ under the License.
<aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
<mariadb-java-client.version>3.0.4</mariadb-java-client.version>
- <revision>0.15-SNAPSHOT</revision>
+ <revision>1.0-SNAPSHOT</revision>
<project.scm.id>github</project.scm.id>
</properties>
<profiles>
diff --git a/fs_brokers/apache_hdfs_broker/pom.xml b/fs_brokers/apache_hdfs_broker/pom.xml
index c15f5f63ba..77006b8f9e 100644
--- a/fs_brokers/apache_hdfs_broker/pom.xml
+++ b/fs_brokers/apache_hdfs_broker/pom.xml
@@ -27,7 +27,7 @@ under the License.
</parent>
<groupId>org.apache.doris</groupId>
<artifactId>apache-hdfs-broker</artifactId>
- <version>0.15-SNAPSHOT</version>
+ <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Doris HDFS Broker</name>
<url>https://doris.apache.org/</url>
@@ -39,14 +39,14 @@ under the License.
</license>
</licenses>
<scm>
- <connection>scm:git:https://git@github.com/apache/incubator-doris.git</connection>
- <developerConnection>scm:git:https://git@github.com/apache/incubator-doris.git</developerConnection>
- <url>scm:git:https://git@github.com/apache/incubator-doris.git</url>
+ <connection>scm:git:https://git@github.com/apache/doris.git</connection>
+ <developerConnection>scm:git:https://git@github.com/apache/doris.git</developerConnection>
+ <url>scm:git:https://git@github.com/apache/doris.git</url>
<tag>HEAD</tag>
</scm>
<issueManagement>
<system>GitHub</system>
- <url>https://github.com/apache/incubator-doris/issues</url>
+ <url>https://github.com/apache/doris/issues</url>
</issueManagement>
<mailingLists>
<mailingList>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/13: [fix](backup) fix mkdir failed (#10422) (#10423)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ade515225c42b62f81a837507a3f4448d205fcea
Author: HB <13...@qq.com>
AuthorDate: Sun Jun 26 09:55:48 2022 +0800
[fix](backup) fix mkdir failed (#10422) (#10423)
---
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 5f07552c79..b2dc122b3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -659,7 +659,7 @@ public class BackupJob extends AbstractJob {
Files.walk(localJobDirPath,
FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
}
- if (!jobDir.mkdir()) {
+ if (!jobDir.mkdirs()) {
status = new Status(ErrCode.COMMON_ERROR, "Failed to create tmp dir: " + localJobDirPath);
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/13: [fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d51a8d2131131cb4ff93157fa3b58efd87f17cc9
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Sun Jun 26 09:52:16 2022 +0800
[fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
---
.../apache/doris/alter/SchemaChangeHandler.java | 16 ++++-
.../org/apache/doris/analysis/CreateTableStmt.java | 27 ++++----
.../java/org/apache/doris/catalog/Catalog.java | 15 ++++
.../org/apache/doris/catalog/DistributionInfo.java | 33 ---------
.../apache/doris/alter/SchemaChangeJobV2Test.java | 79 ++++++++++++++++++++--
.../org/apache/doris/catalog/CatalogTestUtil.java | 14 ----
.../org/apache/doris/catalog/CreateTableTest.java | 30 ++++++++
7 files changed, 147 insertions(+), 67 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index d12382f926..69e6fbf768 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.ReplicaAllocation;
@@ -768,7 +769,15 @@ public class SchemaChangeHandler extends AlterHandler {
newColumn.setIsKey(true);
} else if (newColumn.getAggregationType() == AggregateType.SUM
&& newColumn.getDefaultValue() != null && !newColumn.getDefaultValue().equals("0")) {
- throw new DdlException("The default value of '" + newColName + "' with SUM aggregation function must be zero");
+ throw new DdlException("The default value of '"
+ + newColName + "' with SUM aggregation function must be zero");
+ } else if (olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo) {
+ if (newColumn.getAggregationType() == AggregateType.REPLACE
+ || newColumn.getAggregationType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new DdlException("Can not add value column with aggregation type "
+ + newColumn.getAggregationType() + " for olap table with random distribution : "
+ + newColName);
+ }
}
} else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
if (newColumn.getAggregationType() != null) {
@@ -1481,6 +1490,11 @@ public class SchemaChangeHandler extends AlterHandler {
Catalog.getCurrentCatalog().modifyTableColocate(db, olapTable, colocateGroup, false, null);
return;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE)) {
+ String distributionType = properties.get(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE);
+ if (!distributionType.equalsIgnoreCase("random")) {
+ throw new DdlException("Only support modifying distribution type of table from"
+ + " hash to random");
+ }
Catalog.getCurrentCatalog().convertDistributionType(db, olapTable);
return;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SEND_CLEAR_ALTER_TASK)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 47a34b9554..bafafe71cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
@@ -91,9 +92,6 @@ public class CreateTableStmt extends DdlStmt {
engineNames.add("iceberg");
}
- // for backup. set to -1 for normal use
- private int tableSignature;
-
public CreateTableStmt() {
// for persist
tableName = new TableName();
@@ -164,7 +162,6 @@ public class CreateTableStmt extends DdlStmt {
this.ifNotExists = ifNotExists;
this.comment = Strings.nullToEmpty(comment);
- this.tableSignature = -1;
this.rollupAlterClauseList = rollupAlterClauseList == null ? new ArrayList<>() : rollupAlterClauseList;
}
@@ -240,14 +237,6 @@ public class CreateTableStmt extends DdlStmt {
return tableName.getDb();
}
- public void setTableSignature(int tableSignature) {
- this.tableSignature = tableSignature;
- }
-
- public int getTableSignature() {
- return tableSignature;
- }
-
public void setTableName(String newTableName) {
tableName = new TableName(tableName.getDb(), newTableName);
}
@@ -427,6 +416,20 @@ public class CreateTableStmt extends DdlStmt {
throw new AnalysisException("Create olap table should contain distribution desc");
}
distributionDesc.analyze(columnSet, columnDefs);
+ if (distributionDesc.type == DistributionInfo.DistributionInfoType.RANDOM) {
+ if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
+ throw new AnalysisException("Create unique keys table should not contain random distribution desc");
+ } else if (keysDesc.getKeysType() == KeysType.AGG_KEYS) {
+ for (ColumnDef columnDef : columnDefs) {
+ if (columnDef.getAggregateType() == AggregateType.REPLACE
+ || columnDef.getAggregateType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new AnalysisException("Create aggregate keys table with value columns of which"
+ + " aggregate type is " + columnDef.getAggregateType() + " should not contain random"
+ + " distribution desc");
+ }
+ }
+ }
+ }
} else if (engineName.equalsIgnoreCase("elasticsearch")) {
EsUtil.analyzePartitionAndDistributionDesc(partitionDesc, distributionDesc);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 69832b139a..3ae89e10e2 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -7074,6 +7074,21 @@ public class Catalog {
public void convertDistributionType(Database db, OlapTable tbl) throws DdlException {
tbl.writeLockOrDdlException();
try {
+ if (tbl.isColocateTable()) {
+ throw new DdlException("Cannot change distribution type of colocate table.");
+ }
+ if (tbl.getKeysType() == KeysType.UNIQUE_KEYS) {
+ throw new DdlException("Cannot change distribution type of unique keys table.");
+ }
+ if (tbl.getKeysType() == KeysType.AGG_KEYS) {
+ for (Column column : tbl.getBaseSchema()) {
+ if (column.getAggregationType() == AggregateType.REPLACE
+ || column.getAggregationType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new DdlException("Cannot change distribution type of aggregate keys table which has value"
+ + " columns with " + column.getAggregationType() + " type.");
+ }
+ }
+ }
if (!tbl.convertHashDistributionToRandomDistribution()) {
throw new DdlException("Table " + tbl.getName() + " is not hash distributed");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
index ef1712baa5..d228ceb3bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
@@ -18,11 +18,9 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.DistributionDesc;
-import org.apache.doris.analysis.Expr;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
-import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.NotImplementedException;
@@ -30,8 +28,6 @@ import org.apache.commons.lang.NotImplementedException;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
public abstract class DistributionInfo implements Writable {
@@ -89,33 +85,4 @@ public abstract class DistributionInfo implements Writable {
public boolean equals(DistributionInfo info) {
return false;
}
-
- public static List<Expr> toDistExpr(OlapTable tbl, DistributionInfo distInfo, Map<String, Expr> exprByCol) {
- List<Expr> distExprs = Lists.newArrayList();
- if (distInfo instanceof RandomDistributionInfo) {
- for (Column col : tbl.getBaseSchema()) {
- if (col.isKey()) {
- Expr distExpr = exprByCol.get(col.getName());
- // used to compute hash
- if (col.getDataType() == PrimitiveType.CHAR) {
- distExpr.setType(Type.CHAR);
- }
- distExprs.add(distExpr);
- } else {
- break;
- }
- }
- } else if (distInfo instanceof HashDistributionInfo) {
- HashDistributionInfo hashDistInfo = (HashDistributionInfo) distInfo;
- for (Column col : hashDistInfo.getDistributionColumns()) {
- Expr distExpr = exprByCol.get(col.getName());
- // used to compute hash
- if (col.getDataType() == PrimitiveType.CHAR) {
- distExpr.setType(Type.CHAR);
- }
- distExprs.add(distExpr);
- }
- }
- return distExprs;
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
index 6d78eb84a0..72f3046c6e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
@@ -33,11 +33,13 @@ import org.apache.doris.backup.CatalogMocker;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.CatalogTestUtil;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.FakeCatalog;
import org.apache.doris.catalog.FakeEditLog;
+import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
@@ -49,6 +51,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@@ -64,9 +67,11 @@ import org.apache.doris.thrift.TTaskType;
import org.apache.doris.transaction.FakeTransactionIDGenerator;
import org.apache.doris.transaction.GlobalTransactionMgr;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec;
+import mockit.Expectations;
+import mockit.Injectable;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -357,7 +362,7 @@ public class SchemaChangeJobV2Test {
Assert.assertEquals(3, olapTable.getTableProperty().getDynamicPartitionProperty().getBuckets());
}
- public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, String propertyValue, String missPropertyKey)
+ public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, String propertyValue)
throws UserException {
fakeCatalog = new FakeCatalog();
FakeCatalog.setCatalog(masterCatalog);
@@ -378,11 +383,11 @@ public class SchemaChangeJobV2Test {
@Test
public void testModifyDynamicPartitionWithoutTableProperty() throws UserException {
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.ENABLE, "false", DynamicPartitionProperty.TIME_UNIT);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.TIME_UNIT, "day", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.END, "3", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.PREFIX, "p", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.BUCKETS, "30", DynamicPartitionProperty.ENABLE);
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.ENABLE, "false");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.TIME_UNIT, "day");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.END, "3");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.PREFIX, "p");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.BUCKETS, "30");
}
@Test
@@ -436,4 +441,64 @@ public class SchemaChangeJobV2Test {
Partition partition1 = olapTable.getPartition(CatalogTestUtil.testPartitionId1);
Assert.assertTrue(partition1.getDistributionInfo().getType() == DistributionInfo.DistributionInfoType.RANDOM);
}
+
+ @Test
+ public void testAbnormalModifyTableDistributionType1(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = true;
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change distribution type of colocate table.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
+
+ @Test
+ public void testAbnormalModifyTableDistributionType2(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = false;
+ table.getKeysType();
+ result = KeysType.UNIQUE_KEYS;
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change distribution type of unique keys table.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
+
+ @Test
+ public void testAbnormalModifyTableDistributionType3(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = false;
+ table.getKeysType();
+ result = KeysType.AGG_KEYS;
+ table.getBaseSchema();
+ result = Lists.newArrayList(
+ new Column("k1", Type.INT, true, null, "0", ""),
+ new Column("v1", Type.INT, false, AggregateType.REPLACE, "0", ""));
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change "
+ + "distribution type of aggregate keys table which has value columns with REPLACE type.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index e950d4f512..831f67acae 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -28,7 +28,6 @@ import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.persist.EditLog;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
@@ -38,7 +37,6 @@ import com.google.common.collect.Maps;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -326,16 +324,4 @@ public class CatalogTestUtil {
backend.setAlive(true);
return backend;
}
-
- public static Backend createBackend(long id, String host, int heartPort, int bePort, int httpPort,
- long totalCapacityB, long avaiLabelCapacityB) {
- Backend backend = createBackend(id, host, heartPort, bePort, httpPort);
- Map<String, TDisk> backendDisks = new HashMap<String, TDisk>();
- String rootPath = "root_path";
- TDisk disk = new TDisk(rootPath, totalCapacityB, avaiLabelCapacityB, true);
- backendDisks.put(rootPath, disk);
- backend.updateDisks(backendDisks);
- backend.setAlive(true);
- return backend;
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 082a05cf40..f335e48c09 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -485,6 +485,36 @@ public class CreateTableTest {
" \"dynamic_partition.start_day_of_month\" = \"3\"\n" +
");"));
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+ "Create unique keys table should not contain random distribution desc",
+ () -> createTable("CREATE TABLE test.tbl21\n"
+ + "(\n"
+ + " `k1` bigint(20) NULL COMMENT \"\",\n"
+ + " `k2` largeint(40) NULL COMMENT \"\",\n"
+ + " `v1` varchar(204) NULL COMMENT \"\",\n"
+ + " `v2` smallint(6) NULL DEFAULT \"10\" COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "UNIQUE KEY(`k1`, `k2`)\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 32\n"
+ + "PROPERTIES (\n"
+ + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ + ");"));
+
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+ "Create aggregate keys table with value columns of which aggregate type"
+ + " is REPLACE should not contain random distribution desc",
+ () -> createTable("CREATE TABLE test.tbl22\n"
+ + "(\n"
+ + " `k1` bigint(20) NULL COMMENT \"\",\n"
+ + " `k2` largeint(40) NULL COMMENT \"\",\n"
+ + " `v1` bigint(20) REPLACE NULL COMMENT \"\",\n"
+ + " `v2` smallint(6) REPLACE_IF_NOT_NULL NULL DEFAULT \"10\" COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`k1`, `k2`)\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 32\n"
+ + "PROPERTIES (\n"
+ + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ + ");"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 13/13: [bugfix]fix bitmap function bug (#10477)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3948ef23b5d7336f3998d38e911127dce80ad19b
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Wed Jun 29 09:04:26 2022 +0800
[bugfix]fix bitmap function bug (#10477)
---
.../aggregate_function_bitmap.h | 28 ++++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
index f4a237dbb9..e2fd2b8989 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h
@@ -36,9 +36,23 @@ struct AggregateFunctionBitmapUnionOp {
res.add(data);
}
- static void add(BitmapValue& res, const BitmapValue& data, bool& is_first) { res |= data; }
+ static void add(BitmapValue& res, const BitmapValue& data, bool& is_first) {
+ if (UNLIKELY(is_first)) {
+ res = data;
+ is_first = false;
+ } else {
+ res |= data;
+ }
+ }
- static void merge(BitmapValue& res, const BitmapValue& data, bool& is_first) { res |= data; }
+ static void merge(BitmapValue& res, const BitmapValue& data, bool& is_first) {
+ if (UNLIKELY(is_first)) {
+ res = data;
+ is_first = false;
+ } else {
+ res |= data;
+ }
+ }
};
struct AggregateFunctionBitmapIntersectOp {
@@ -79,6 +93,8 @@ struct AggregateFunctionBitmapData {
void read(BufferReadable& buf) { DataTypeBitMap::deserialize_as_stream(value, buf); }
+ void reset() { is_first = true; }
+
BitmapValue& get() { return value; }
};
@@ -125,6 +141,10 @@ public:
column.get_data().push_back(
const_cast<AggregateFunctionBitmapData<Op>&>(this->data(place)).get());
}
+
+ void reset(AggregateDataPtr __restrict place) const override {
+ this->data(place).reset();
+ }
};
template <bool nullable, typename ColVecType>
@@ -179,6 +199,10 @@ public:
auto& column = static_cast<ColVecResult&>(to);
column.get_data().push_back(value_data.cardinality());
}
+
+ void reset(AggregateDataPtr __restrict place) const override {
+ this->data(place).reset();
+ }
};
AggregateFunctionPtr create_aggregate_function_bitmap_union(const std::string& name,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org