You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/27 01:33:11 UTC
[doris] branch dev-1.0.1 updated (cedf2f240a -> 07d5672116)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from cedf2f240a [bug] fix window function nullable type bug
new 1df9ae0aaa [fix](doe) fix doe on es v8 (#10391)
new f0927d9755 [fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
new 07d5672116 [fix](backup) fix mkdir failed (#10422) (#10423)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/es/es_scan_reader.cpp | 32 ++++++---
docs/en/extending-doris/doris-on-es.md | 14 +---
docs/zh-CN/extending-doris/doris-on-es.md | 14 +---
.../apache/doris/alter/SchemaChangeHandler.java | 16 ++++-
.../org/apache/doris/analysis/CreateTableStmt.java | 27 ++++----
.../java/org/apache/doris/backup/BackupJob.java | 2 +-
.../java/org/apache/doris/catalog/Catalog.java | 27 ++++++--
.../org/apache/doris/catalog/DistributionInfo.java | 33 ---------
.../java/org/apache/doris/catalog/EsTable.java | 53 ++++++++-------
.../external/elasticsearch/EsMajorVersion.java | 7 +-
.../doris/external/elasticsearch/EsRestClient.java | 5 +-
.../doris/external/elasticsearch/MappingPhase.java | 20 +-----
.../java/org/apache/doris/planner/EsScanNode.java | 4 +-
.../apache/doris/alter/SchemaChangeJobV2Test.java | 79 ++++++++++++++++++++--
.../org/apache/doris/catalog/CatalogTestUtil.java | 14 ----
.../org/apache/doris/catalog/CreateTableTest.java | 30 ++++++++
.../external/elasticsearch/MappingPhaseTest.java | 2 +-
17 files changed, 219 insertions(+), 160 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [fix](doe) fix doe on es v8 (#10391)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1df9ae0aaa901739b823c670f1ea968518a3a364
Author: Stalary <st...@163.com>
AuthorDate: Sun Jun 26 09:51:29 2022 +0800
[fix](doe) fix doe on es v8 (#10391)
doris on es8 can not work, because type change. The use of type is no longer recommended in es7,
and support for type has been removed from es8.
1. /_mapping not support include_type_name
2. /_search not support use type
---
be/src/exec/es/es_scan_reader.cpp | 32 +++++++++----
docs/en/extending-doris/doris-on-es.md | 14 +-----
docs/zh-CN/extending-doris/doris-on-es.md | 14 +-----
.../java/org/apache/doris/catalog/Catalog.java | 12 ++---
.../java/org/apache/doris/catalog/EsTable.java | 53 +++++++++++-----------
.../external/elasticsearch/EsMajorVersion.java | 7 +--
.../doris/external/elasticsearch/EsRestClient.java | 5 +-
.../doris/external/elasticsearch/MappingPhase.java | 20 ++------
.../java/org/apache/doris/planner/EsScanNode.java | 4 +-
.../external/elasticsearch/MappingPhaseTest.java | 2 +-
10 files changed, 71 insertions(+), 92 deletions(-)
diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp
index 87131a7647..5d67d995cf 100644
--- a/be/src/exec/es/es_scan_reader.cpp
+++ b/be/src/exec/es/es_scan_reader.cpp
@@ -47,7 +47,9 @@ ESScanReader::ESScanReader(const std::string& target,
_doc_value_mode(doc_value_mode) {
_target = target;
_index = props.at(KEY_INDEX);
- _type = props.at(KEY_TYPE);
+ if (props.find(KEY_TYPE) != props.end()) {
+ _type = props.at(KEY_TYPE);
+ }
if (props.find(KEY_USER_NAME) != props.end()) {
_user_name = props.at(KEY_USER_NAME);
}
@@ -73,20 +75,32 @@ ESScanReader::ESScanReader(const std::string& target,
_exactly_once = true;
std::stringstream scratch;
// just send a normal search against the elasticsearch with additional terminate_after param to achieve terminate early effect when limit take effect
- scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
- << "/_search?"
- << "terminate_after=" << props.at(KEY_TERMINATE_AFTER) << REQUEST_PREFERENCE_PREFIX
- << _shards << "&" << filter_path;
+ if (_type.empty()) {
+ scratch << _target << REQUEST_SEPARATOR << _index << "/_search?"
+ << "terminate_after=" << props.at(KEY_TERMINATE_AFTER)
+ << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path;
+ } else {
+ scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
+ << "/_search?"
+ << "terminate_after=" << props.at(KEY_TERMINATE_AFTER)
+ << REQUEST_PREFERENCE_PREFIX << _shards << "&" << filter_path;
+ }
_search_url = scratch.str();
} else {
_exactly_once = false;
std::stringstream scratch;
// scroll request for scanning
// add terminate_after for the first scroll to avoid decompress all postings list
- scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
- << "/_search?"
- << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards << "&"
- << filter_path << "&terminate_after=" << batch_size_str;
+ if (_type.empty()) {
+ scratch << _target << REQUEST_SEPARATOR << _index << "/_search?"
+ << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards
+ << "&" << filter_path << "&terminate_after=" << batch_size_str;
+ } else {
+ scratch << _target << REQUEST_SEPARATOR << _index << REQUEST_SEPARATOR << _type
+ << "/_search?"
+ << "scroll=" << _scroll_keep_alive << REQUEST_PREFERENCE_PREFIX << _shards
+ << "&" << filter_path << "&terminate_after=" << batch_size_str;
+ }
_init_scroll_url = scratch.str();
_next_scroll_url = _target + REQUEST_SEARCH_SCROLL_PATH + "?" + filter_path;
}
diff --git a/docs/en/extending-doris/doris-on-es.md b/docs/en/extending-doris/doris-on-es.md
index a653986c35..8a3a104b4a 100644
--- a/docs/en/extending-doris/doris-on-es.md
+++ b/docs/en/extending-doris/doris-on-es.md
@@ -119,7 +119,6 @@ PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
"type" = "doc",
-
"user" = "root",
"password" = "root"
);
@@ -131,7 +130,7 @@ Parameter | Description
---|---
**hosts** | ES Cluster Connection Address, maybe one or more node, load-balance is also accepted
**index** | the related ES index name, alias is supported, and if you use doc_value, you need to use the real name
-**type** | the type for this index, If not specified, `_doc` will be used
+**type** | the type for this index, ES 7.x and later versions do not pass this parameter
**user** | username for ES
**password** | password for the user
@@ -188,10 +187,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_docvalue_scan" = "true"
);
```
@@ -229,10 +226,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_keyword_sniff" = "true"
);
```
@@ -341,10 +336,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"nodes_discovery" = "true"
);
```
@@ -370,10 +363,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"http_ssl_enabled" = "true"
);
```
@@ -561,8 +552,7 @@ PROPERTIES (
"hosts" = "http://127.0.0.1:8200",
"user" = "root",
"password" = "root",
-"index" = "doe",
-"type" = "doc"
+"index" = "doe"
}
```
`Notice`:
diff --git a/docs/zh-CN/extending-doris/doris-on-es.md b/docs/zh-CN/extending-doris/doris-on-es.md
index 7840b6be27..1ca84bf4c5 100644
--- a/docs/zh-CN/extending-doris/doris-on-es.md
+++ b/docs/zh-CN/extending-doris/doris-on-es.md
@@ -117,7 +117,6 @@ PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
"type" = "doc",
-
"user" = "root",
"password" = "root"
);
@@ -129,7 +128,7 @@ PROPERTIES (
---|---
**hosts** | ES集群地址，可以是一个或多个，也可以是ES前端的负载均衡地址
**index** | 对应的ES的index名字，支持alias，如果使用doc_value，需要使用真实的名称
-**type** | index的type，不指定的情况会使用_doc
+**type** | index的type，ES 7.x及以后的版本不传此参数
**user** | ES集群用户名
**password** | 对应用户的密码信息
@@ -185,10 +184,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_docvalue_scan" = "true"
);
```
@@ -226,10 +223,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"enable_keyword_sniff" = "true"
);
```
@@ -338,10 +333,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"nodes_discovery" = "true"
);
```
@@ -367,10 +360,8 @@ CREATE EXTERNAL TABLE `test` (
PROPERTIES (
"hosts" = "http://192.168.0.1:8200,http://192.168.0.2:8200",
"index" = "test",
-"type" = "doc",
"user" = "root",
"password" = "root",
-
"http_ssl_enabled" = "true"
);
```
@@ -559,8 +550,7 @@ PROPERTIES (
"hosts" = "http://127.0.0.1:8200",
"user" = "root",
"password" = "root",
-"index" = "doe",
-"type" = "doc"
+"index" = "doe"
}
```
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 22e532f78f..69832b139a 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -4362,12 +4362,8 @@ public class Catalog {
if (partitionInfo.getType() == PartitionType.RANGE) {
sb.append("\n");
sb.append("PARTITION BY RANGE(");
- idx = 0;
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo;
for (Column column : rangePartitionInfo.getPartitionColumns()) {
- if (idx != 0) {
- sb.append(", ");
- }
sb.append("`").append(column.getName()).append("`");
}
sb.append(")\n()");
@@ -4379,12 +4375,14 @@ public class Catalog {
sb.append("\"user\" = \"").append(esTable.getUserName()).append("\",\n");
sb.append("\"password\" = \"").append(hidePassword ? "" : esTable.getPasswd()).append("\",\n");
sb.append("\"index\" = \"").append(esTable.getIndexName()).append("\",\n");
- sb.append("\"type\" = \"").append(esTable.getMappingType()).append("\",\n");
+ if (esTable.getMappingType() != null) {
+ sb.append("\"type\" = \"").append(esTable.getMappingType()).append("\",\n");
+ }
sb.append("\"transport\" = \"").append(esTable.getTransport()).append("\",\n");
sb.append("\"enable_docvalue_scan\" = \"").append(esTable.isDocValueScanEnable()).append("\",\n");
sb.append("\"max_docvalue_fields\" = \"").append(esTable.maxDocValueFields()).append("\",\n");
- sb.append("\"enable_keyword_sniff\" = \"").append(esTable.isKeywordSniffEnable()).append("\"\n");
- sb.append("\"nodes_discovery\" = \"").append(esTable.isNodesDiscovery()).append("\"\n");
+ sb.append("\"enable_keyword_sniff\" = \"").append(esTable.isKeywordSniffEnable()).append("\",\n");
+ sb.append("\"nodes_discovery\" = \"").append(esTable.isNodesDiscovery()).append("\",\n");
sb.append("\"http_ssl_enabled\" = \"").append(esTable.isHttpSslEnabled()).append("\"\n");
sb.append(")");
} else if (table.getType() == TableType.HIVE) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index df19a86d96..cc5d902080 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -73,8 +73,8 @@ public class EsTable extends Table {
// index name can be specific index、wildcard matched or alias.
private String indexName;
- // which type used for `indexName`, default to `_doc`
- private String mappingType = "_doc";
+ // which type used for `indexName`
+ private String mappingType = null;
private String transport = "http";
// only save the partition definition, save the partition key,
// partition list is got from es cluster dynamically and is saved in esTableState
@@ -116,8 +116,8 @@ public class EsTable extends Table {
super(TableType.ELASTICSEARCH);
}
- public EsTable(long id, String name, List<Column> schema,
- Map<String, String> properties, PartitionInfo partitionInfo) throws DdlException {
+ public EsTable(long id, String name, List<Column> schema, Map<String, String> properties,
+ PartitionInfo partitionInfo) throws DdlException {
super(id, name, TableType.ELASTICSEARCH, schema);
this.partitionInfo = partitionInfo;
validate(properties);
@@ -154,32 +154,29 @@ public class EsTable extends Table {
private void validate(Map<String, String> properties) throws DdlException {
if (properties == null) {
- throw new DdlException("Please set properties of elasticsearch table, "
- + "they are: hosts, user, password, index");
+ throw new DdlException(
+ "Please set properties of elasticsearch table, " + "they are: hosts, user, password, index");
}
- if (Strings.isNullOrEmpty(properties.get(HOSTS))
- || Strings.isNullOrEmpty(properties.get(HOSTS).trim())) {
+ if (Strings.isNullOrEmpty(properties.get(HOSTS)) || Strings.isNullOrEmpty(properties.get(HOSTS).trim())) {
throw new DdlException("Hosts of ES table is null. "
+ "Please add properties('hosts'='xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx') when create table");
}
hosts = properties.get(HOSTS).trim();
seeds = hosts.split(",");
- if (!Strings.isNullOrEmpty(properties.get(USER))
- && !Strings.isNullOrEmpty(properties.get(USER).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(USER)) && !Strings.isNullOrEmpty(properties.get(USER).trim())) {
userName = properties.get(USER).trim();
}
- if (!Strings.isNullOrEmpty(properties.get(PASSWORD))
- && !Strings.isNullOrEmpty(properties.get(PASSWORD).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(PASSWORD)) && !Strings.isNullOrEmpty(
+ properties.get(PASSWORD).trim())) {
passwd = properties.get(PASSWORD).trim();
}
- if (Strings.isNullOrEmpty(properties.get(INDEX))
- || Strings.isNullOrEmpty(properties.get(INDEX).trim())) {
- throw new DdlException("Index of ES table is null. "
- + "Please add properties('index'='xxxx') when create table");
+ if (Strings.isNullOrEmpty(properties.get(INDEX)) || Strings.isNullOrEmpty(properties.get(INDEX).trim())) {
+ throw new DdlException(
+ "Index of ES table is null. " + "Please add properties('index'='xxxx') when create table");
}
indexName = properties.get(INDEX).trim();
@@ -191,8 +188,8 @@ public class EsTable extends Table {
throw new DdlException("Unsupported/Unknown ES Cluster version [" + properties.get(VERSION) + "] ");
}
} catch (Exception e) {
- throw new DdlException("fail to parse ES major version, version= "
- + properties.get(VERSION).trim() + ", should be like '6.5.3' ");
+ throw new DdlException("fail to parse ES major version, version= " + properties.get(VERSION).trim()
+ + ", should be like '6.5.3' ");
}
}
@@ -222,13 +219,12 @@ public class EsTable extends Table {
}
}
- if (!Strings.isNullOrEmpty(properties.get(TYPE))
- && !Strings.isNullOrEmpty(properties.get(TYPE).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(TYPE)) && !Strings.isNullOrEmpty(properties.get(TYPE).trim())) {
mappingType = properties.get(TYPE).trim();
}
- if (!Strings.isNullOrEmpty(properties.get(TRANSPORT))
- && !Strings.isNullOrEmpty(properties.get(TRANSPORT).trim())) {
+ if (!Strings.isNullOrEmpty(properties.get(TRANSPORT)) && !Strings.isNullOrEmpty(
+ properties.get(TRANSPORT).trim())) {
transport = properties.get(TRANSPORT).trim();
if (!(TRANSPORT_HTTP.equals(transport) || TRANSPORT_THRIFT.equals(transport))) {
throw new DdlException("transport of ES table must be http/https(recommend) or thrift(reserved inner usage),"
@@ -250,7 +246,9 @@ public class EsTable extends Table {
tableContext.put("userName", userName);
tableContext.put("passwd", passwd);
tableContext.put("indexName", indexName);
- tableContext.put("mappingType", mappingType);
+ if (mappingType != null) {
+ tableContext.put("mappingType", mappingType);
+ }
tableContext.put("transport", transport);
if (majorVersion != null) {
tableContext.put("majorVersion", majorVersion.toString());
@@ -264,8 +262,8 @@ public class EsTable extends Table {
public TTableDescriptor toThrift() {
TEsTable tEsTable = new TEsTable();
- TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ES_TABLE,
- fullSchema.size(), 0, getName(), "");
+ TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ES_TABLE, fullSchema.size(), 0,
+ getName(), "");
tTableDescriptor.setEsTable(tEsTable);
return tTableDescriptor;
}
@@ -280,7 +278,9 @@ public class EsTable extends Table {
sb.append(userName);
sb.append(passwd);
sb.append(indexName);
- sb.append(mappingType);
+ if (mappingType != null) {
+ sb.append(mappingType);
+ }
sb.append(transport);
} else {
for (Map.Entry<String, String> entry : tableContext.entrySet()) {
@@ -305,6 +305,7 @@ public class EsTable extends Table {
partitionInfo.write(out);
}
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
int size = in.readInt();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
index d16fbd81d2..41801687f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java
@@ -32,7 +32,7 @@ public class EsMajorVersion {
public static final EsMajorVersion V_6_X = new EsMajorVersion((byte) 6, "6.x");
public static final EsMajorVersion V_7_X = new EsMajorVersion((byte) 7, "7.x");
public static final EsMajorVersion V_8_X = new EsMajorVersion((byte) 8, "8.x");
- public static final EsMajorVersion LATEST = V_7_X;
+ public static final EsMajorVersion LATEST = V_8_X;
public final byte major;
private final String version;
@@ -89,8 +89,9 @@ public class EsMajorVersion {
if (version.startsWith("8.")) {
return new EsMajorVersion((byte) 8, version);
}
- throw new DorisEsException("Unsupported/Unknown ES Cluster version [" + version + "]." +
- "Highest supported version is [" + LATEST.version + "].");
+ throw new DorisEsException(
+ "Unsupported/Unknown ES Cluster version [" + version + "]." + "Highest supported version is ["
+ + LATEST.version + "].");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
index 467047ac8c..5a00719154 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
@@ -127,11 +127,8 @@ public class EsRestClient {
* @return
* @throws Exception
*/
- public String getMapping(String indexName, boolean includeTypeName) throws DorisEsException {
+ public String getMapping(String indexName) throws DorisEsException {
String path = indexName + "/_mapping";
- if (includeTypeName) {
- path += "?include_type_name=true";
- }
String indexMapping = execute(path);
if (indexMapping == null) {
throw new DorisEsException("index[" + indexName + "] not found");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index f736a9ee2f..4057348e66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -37,22 +37,13 @@ public class MappingPhase implements SearchPhase {
// json response for `{index}/_mapping` API
private String jsonMapping;
- private boolean includeTypeName = false;
-
public MappingPhase(EsRestClient client) {
this.client = client;
}
- @Override
- public void preProcess(SearchContext context) {
- if (context.version() != null && context.version().onOrAfter(EsMajorVersion.V_7_X)) {
- includeTypeName = true;
- }
- }
-
@Override
public void execute(SearchContext context) throws DorisEsException {
- jsonMapping = client.getMapping(context.sourceIndex(), includeTypeName);
+ jsonMapping = client.getMapping(context.sourceIndex());
}
@Override
@@ -78,15 +69,10 @@ public class MappingPhase implements SearchPhase {
JSONObject mappings = (JSONObject) docData.get("mappings");
JSONObject rootSchema = (JSONObject) mappings.get(searchContext.type());
JSONObject properties;
- // After (include) 7.x, type was removed from ES mapping, default type is `_doc`
+ // Elasticsearch 7.x, type was removed from ES mapping, default type is `_doc`
// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html
+ // Elasticsearch 8.x, include_type_name parameter is removed
if (rootSchema == null) {
- if (searchContext.type().equals("_doc") == false) {
- throw new DorisEsException("index[" + searchContext.sourceIndex() + "]'s type must be exists, "
- + " and after ES7.x type must be `_doc`, but found ["
- + searchContext.type() + "], for table ["
- + searchContext.esTable().getName() + "]");
- }
properties = (JSONObject) mappings.get("properties");
} else {
properties = (JSONObject) rootSchema.get("properties");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
index 2ed1a4bde8..42f3934a9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
@@ -247,7 +247,9 @@ public class EsScanNode extends ScanNode {
TEsScanRange esScanRange = new TEsScanRange();
esScanRange.setEsHosts(shardAllocations);
esScanRange.setIndex(shardRouting.get(0).getIndexName());
- esScanRange.setType(table.getMappingType());
+ if (table.getType() != null) {
+ esScanRange.setType(table.getMappingType());
+ }
esScanRange.setShardId(shardRouting.get(0).getShardId());
// Scan range
TScanRange scanRange = new TScanRange();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
index d229d7c924..18330eb55a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/MappingPhaseTest.java
@@ -89,7 +89,7 @@ public class MappingPhaseTest extends EsTestCase {
String jsonMapping = loadJsonFromFile("data/es/test_index_mapping.json");
new Expectations(client) {
{
- client.getMapping(anyString, anyBoolean);
+ client.getMapping(anyString);
minTimes = 0;
result = jsonMapping;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [fix](backup) fix mkdir failed (#10422) (#10423)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 07d5672116893505bcf747d897b82f5c6f17d559
Author: HB <13...@qq.com>
AuthorDate: Sun Jun 26 09:55:48 2022 +0800
[fix](backup) fix mkdir failed (#10422) (#10423)
---
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 5f07552c79..b2dc122b3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -659,7 +659,7 @@ public class BackupJob extends AbstractJob {
Files.walk(localJobDirPath,
FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
}
- if (!jobDir.mkdir()) {
+ if (!jobDir.mkdirs()) {
status = new Status(ErrCode.COMMON_ERROR, "Failed to create tmp dir: " + localJobDirPath);
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f0927d9755bc01ea23033ed2d1ba596dadb2d4d2
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Sun Jun 26 09:52:16 2022 +0800
[fix](random-distribution) Make aggregate keys table with replace type columns and unique keys table can only have hash distribution to make data computing correctly (#10414)
---
.../apache/doris/alter/SchemaChangeHandler.java | 16 ++++-
.../org/apache/doris/analysis/CreateTableStmt.java | 27 ++++----
.../java/org/apache/doris/catalog/Catalog.java | 15 ++++
.../org/apache/doris/catalog/DistributionInfo.java | 33 ---------
.../apache/doris/alter/SchemaChangeJobV2Test.java | 79 ++++++++++++++++++++--
.../org/apache/doris/catalog/CatalogTestUtil.java | 14 ----
.../org/apache/doris/catalog/CreateTableTest.java | 30 ++++++++
7 files changed, 147 insertions(+), 67 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index d12382f926..69e6fbf768 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.ReplicaAllocation;
@@ -768,7 +769,15 @@ public class SchemaChangeHandler extends AlterHandler {
newColumn.setIsKey(true);
} else if (newColumn.getAggregationType() == AggregateType.SUM
&& newColumn.getDefaultValue() != null && !newColumn.getDefaultValue().equals("0")) {
- throw new DdlException("The default value of '" + newColName + "' with SUM aggregation function must be zero");
+ throw new DdlException("The default value of '"
+ + newColName + "' with SUM aggregation function must be zero");
+ } else if (olapTable.getDefaultDistributionInfo() instanceof RandomDistributionInfo) {
+ if (newColumn.getAggregationType() == AggregateType.REPLACE
+ || newColumn.getAggregationType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new DdlException("Can not add value column with aggregation type "
+ + newColumn.getAggregationType() + " for olap table with random distribution : "
+ + newColName);
+ }
}
} else if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) {
if (newColumn.getAggregationType() != null) {
@@ -1481,6 +1490,11 @@ public class SchemaChangeHandler extends AlterHandler {
Catalog.getCurrentCatalog().modifyTableColocate(db, olapTable, colocateGroup, false, null);
return;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE)) {
+ String distributionType = properties.get(PropertyAnalyzer.PROPERTIES_DISTRIBUTION_TYPE);
+ if (!distributionType.equalsIgnoreCase("random")) {
+ throw new DdlException("Only support modifying distribution type of table from"
+ + " hash to random");
+ }
Catalog.getCurrentCatalog().convertDistributionType(db, olapTable);
return;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SEND_CLEAR_ALTER_TASK)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 47a34b9554..bafafe71cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
@@ -91,9 +92,6 @@ public class CreateTableStmt extends DdlStmt {
engineNames.add("iceberg");
}
- // for backup. set to -1 for normal use
- private int tableSignature;
-
public CreateTableStmt() {
// for persist
tableName = new TableName();
@@ -164,7 +162,6 @@ public class CreateTableStmt extends DdlStmt {
this.ifNotExists = ifNotExists;
this.comment = Strings.nullToEmpty(comment);
- this.tableSignature = -1;
this.rollupAlterClauseList = rollupAlterClauseList == null ? new ArrayList<>() : rollupAlterClauseList;
}
@@ -240,14 +237,6 @@ public class CreateTableStmt extends DdlStmt {
return tableName.getDb();
}
- public void setTableSignature(int tableSignature) {
- this.tableSignature = tableSignature;
- }
-
- public int getTableSignature() {
- return tableSignature;
- }
-
public void setTableName(String newTableName) {
tableName = new TableName(tableName.getDb(), newTableName);
}
@@ -427,6 +416,20 @@ public class CreateTableStmt extends DdlStmt {
throw new AnalysisException("Create olap table should contain distribution desc");
}
distributionDesc.analyze(columnSet, columnDefs);
+ if (distributionDesc.type == DistributionInfo.DistributionInfoType.RANDOM) {
+ if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
+ throw new AnalysisException("Create unique keys table should not contain random distribution desc");
+ } else if (keysDesc.getKeysType() == KeysType.AGG_KEYS) {
+ for (ColumnDef columnDef : columnDefs) {
+ if (columnDef.getAggregateType() == AggregateType.REPLACE
+ || columnDef.getAggregateType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new AnalysisException("Create aggregate keys table with value columns of which"
+ + " aggregate type is " + columnDef.getAggregateType() + " should not contain random"
+ + " distribution desc");
+ }
+ }
+ }
+ }
} else if (engineName.equalsIgnoreCase("elasticsearch")) {
EsUtil.analyzePartitionAndDistributionDesc(partitionDesc, distributionDesc);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 69832b139a..3ae89e10e2 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -7074,6 +7074,21 @@ public class Catalog {
public void convertDistributionType(Database db, OlapTable tbl) throws DdlException {
tbl.writeLockOrDdlException();
try {
+ if (tbl.isColocateTable()) {
+ throw new DdlException("Cannot change distribution type of colocate table.");
+ }
+ if (tbl.getKeysType() == KeysType.UNIQUE_KEYS) {
+ throw new DdlException("Cannot change distribution type of unique keys table.");
+ }
+ if (tbl.getKeysType() == KeysType.AGG_KEYS) {
+ for (Column column : tbl.getBaseSchema()) {
+ if (column.getAggregationType() == AggregateType.REPLACE
+ || column.getAggregationType() == AggregateType.REPLACE_IF_NOT_NULL) {
+ throw new DdlException("Cannot change distribution type of aggregate keys table which has value"
+ + " columns with " + column.getAggregationType() + " type.");
+ }
+ }
+ }
if (!tbl.convertHashDistributionToRandomDistribution()) {
throw new DdlException("Table " + tbl.getName() + " is not hash distributed");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
index ef1712baa5..d228ceb3bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
@@ -18,11 +18,9 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.DistributionDesc;
-import org.apache.doris.analysis.Expr;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
-import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.NotImplementedException;
@@ -30,8 +28,6 @@ import org.apache.commons.lang.NotImplementedException;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
public abstract class DistributionInfo implements Writable {
@@ -89,33 +85,4 @@ public abstract class DistributionInfo implements Writable {
public boolean equals(DistributionInfo info) {
return false;
}
-
- public static List<Expr> toDistExpr(OlapTable tbl, DistributionInfo distInfo, Map<String, Expr> exprByCol) {
- List<Expr> distExprs = Lists.newArrayList();
- if (distInfo instanceof RandomDistributionInfo) {
- for (Column col : tbl.getBaseSchema()) {
- if (col.isKey()) {
- Expr distExpr = exprByCol.get(col.getName());
- // used to compute hash
- if (col.getDataType() == PrimitiveType.CHAR) {
- distExpr.setType(Type.CHAR);
- }
- distExprs.add(distExpr);
- } else {
- break;
- }
- }
- } else if (distInfo instanceof HashDistributionInfo) {
- HashDistributionInfo hashDistInfo = (HashDistributionInfo) distInfo;
- for (Column col : hashDistInfo.getDistributionColumns()) {
- Expr distExpr = exprByCol.get(col.getName());
- // used to compute hash
- if (col.getDataType() == PrimitiveType.CHAR) {
- distExpr.setType(Type.CHAR);
- }
- distExprs.add(distExpr);
- }
- }
- return distExprs;
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
index 6d78eb84a0..72f3046c6e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
@@ -33,11 +33,13 @@ import org.apache.doris.backup.CatalogMocker;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.CatalogTestUtil;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.FakeCatalog;
import org.apache.doris.catalog.FakeEditLog;
+import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
@@ -49,6 +51,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@@ -64,9 +67,11 @@ import org.apache.doris.thrift.TTaskType;
import org.apache.doris.transaction.FakeTransactionIDGenerator;
import org.apache.doris.transaction.GlobalTransactionMgr;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec;
+import mockit.Expectations;
+import mockit.Injectable;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -357,7 +362,7 @@ public class SchemaChangeJobV2Test {
Assert.assertEquals(3, olapTable.getTableProperty().getDynamicPartitionProperty().getBuckets());
}
- public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, String propertyValue, String missPropertyKey)
+ public void modifyDynamicPartitionWithoutTableProperty(String propertyKey, String propertyValue)
throws UserException {
fakeCatalog = new FakeCatalog();
FakeCatalog.setCatalog(masterCatalog);
@@ -378,11 +383,11 @@ public class SchemaChangeJobV2Test {
@Test
public void testModifyDynamicPartitionWithoutTableProperty() throws UserException {
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.ENABLE, "false", DynamicPartitionProperty.TIME_UNIT);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.TIME_UNIT, "day", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.END, "3", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.PREFIX, "p", DynamicPartitionProperty.ENABLE);
- modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.BUCKETS, "30", DynamicPartitionProperty.ENABLE);
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.ENABLE, "false");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.TIME_UNIT, "day");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.END, "3");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.PREFIX, "p");
+ modifyDynamicPartitionWithoutTableProperty(DynamicPartitionProperty.BUCKETS, "30");
}
@Test
@@ -436,4 +441,64 @@ public class SchemaChangeJobV2Test {
Partition partition1 = olapTable.getPartition(CatalogTestUtil.testPartitionId1);
Assert.assertTrue(partition1.getDistributionInfo().getType() == DistributionInfo.DistributionInfoType.RANDOM);
}
+
+ @Test
+ public void testAbnormalModifyTableDistributionType1(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = true;
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change distribution type of colocate table.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
+
+ @Test
+ public void testAbnormalModifyTableDistributionType2(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = false;
+ table.getKeysType();
+ result = KeysType.UNIQUE_KEYS;
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change distribution type of unique keys table.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
+
+ @Test
+ public void testAbnormalModifyTableDistributionType3(@Injectable OlapTable table) throws UserException {
+ fakeCatalog = new FakeCatalog();
+ fakeEditLog = new FakeEditLog();
+ FakeCatalog.setCatalog(masterCatalog);
+ Database db = masterCatalog.getDb(CatalogTestUtil.testDbId1).get();
+ new Expectations() {
+ {
+ table.isColocateTable();
+ result = false;
+ table.getKeysType();
+ result = KeysType.AGG_KEYS;
+ table.getBaseSchema();
+ result = Lists.newArrayList(
+ new Column("k1", Type.INT, true, null, "0", ""),
+ new Column("v1", Type.INT, false, AggregateType.REPLACE, "0", ""));
+ }
+ };
+ expectedEx.expect(DdlException.class);
+ expectedEx.expectMessage("errCode = 2, detailMessage = Cannot change "
+ + "distribution type of aggregate keys table which has value columns with REPLACE type.");
+ Catalog.getCurrentCatalog().convertDistributionType(db, table);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index e950d4f512..831f67acae 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -28,7 +28,6 @@ import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.persist.EditLog;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
-import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
@@ -38,7 +37,6 @@ import com.google.common.collect.Maps;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -326,16 +324,4 @@ public class CatalogTestUtil {
backend.setAlive(true);
return backend;
}
-
- public static Backend createBackend(long id, String host, int heartPort, int bePort, int httpPort,
- long totalCapacityB, long avaiLabelCapacityB) {
- Backend backend = createBackend(id, host, heartPort, bePort, httpPort);
- Map<String, TDisk> backendDisks = new HashMap<String, TDisk>();
- String rootPath = "root_path";
- TDisk disk = new TDisk(rootPath, totalCapacityB, avaiLabelCapacityB, true);
- backendDisks.put(rootPath, disk);
- backend.updateDisks(backendDisks);
- backend.setAlive(true);
- return backend;
- }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 082a05cf40..f335e48c09 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -485,6 +485,36 @@ public class CreateTableTest {
" \"dynamic_partition.start_day_of_month\" = \"3\"\n" +
");"));
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+ "Create unique keys table should not contain random distribution desc",
+ () -> createTable("CREATE TABLE test.tbl21\n"
+ + "(\n"
+ + " `k1` bigint(20) NULL COMMENT \"\",\n"
+ + " `k2` largeint(40) NULL COMMENT \"\",\n"
+ + " `v1` varchar(204) NULL COMMENT \"\",\n"
+ + " `v2` smallint(6) NULL DEFAULT \"10\" COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "UNIQUE KEY(`k1`, `k2`)\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 32\n"
+ + "PROPERTIES (\n"
+ + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ + ");"));
+
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+ "Create aggregate keys table with value columns of which aggregate type"
+ + " is REPLACE should not contain random distribution desc",
+ () -> createTable("CREATE TABLE test.tbl22\n"
+ + "(\n"
+ + " `k1` bigint(20) NULL COMMENT \"\",\n"
+ + " `k2` largeint(40) NULL COMMENT \"\",\n"
+ + " `v1` bigint(20) REPLACE NULL COMMENT \"\",\n"
+ + " `v2` smallint(6) REPLACE_IF_NOT_NULL NULL DEFAULT \"10\" COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "AGGREGATE KEY(`k1`, `k2`)\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 32\n"
+ + "PROPERTIES (\n"
+ + "\"replication_allocation\" = \"tag.location.default: 1\"\n"
+ + ");"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org