You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/21 15:25:42 UTC
[doris] 01/02: [opt](inverted index) support the same column create different type index (#21972)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bdbe4afbdc764429f521aad26539bc7bef8311b8
Author: YueW <45...@users.noreply.github.com>
AuthorDate: Fri Jul 21 23:02:39 2023 +0800
[opt](inverted index) support the same column create different type index (#21972)
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 1 +
.../apache/doris/alter/SchemaChangeHandler.java | 1 +
.../org/apache/doris/analysis/CreateTableStmt.java | 9 +-
.../doris/catalog/MaterializedIndexMeta.java | 4 +
.../inverted_index_p0/test_add_drop_index.groovy | 2 +-
.../inverted_index_p0/test_create_index_1.groovy | 182 +++++++++++++++++++++
.../inverted_index_p0/test_create_index_2.groovy | 176 ++++++++++++++++++++
7 files changed, 371 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 1c6bfbadb0..59a44a0d2d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -794,6 +794,7 @@ bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining)
// such as: where A = '' and B = ','
// the predicate of A and B need downgrade without index query.
// above case can downgrade without index query
+ LOG(INFO) << "will downgrade without index to evaluate predicate, because of res: " << res;
return true;
}
return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index ff5d54f8a5..e949c5c2c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2676,6 +2676,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
+ currentIndexMeta.setIndexes(indexes);
}
olapTable.setIndexes(indexes);
olapTable.rebuildFullSchema();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 556bc03a03..ef8bec00f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,6 +17,7 @@
package org.apache.doris.analysis;
+import org.apache.doris.analysis.IndexDef.IndexType;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DistributionInfo;
@@ -31,6 +32,7 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.AutoBucketUtils;
import org.apache.doris.common.util.ParseUtil;
@@ -588,7 +590,7 @@ public class CreateTableStmt extends DdlStmt {
if (CollectionUtils.isNotEmpty(indexDefs)) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- Set<List<String>> distinctCol = new HashSet<>();
+ Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>();
for (IndexDef indexDef : indexDefs) {
indexDef.analyze();
@@ -613,13 +615,14 @@ public class CreateTableStmt extends DdlStmt {
indexDef.getColumns(), indexDef.getIndexType(),
indexDef.getProperties(), indexDef.getComment()));
distinct.add(indexDef.getIndexName());
- distinctCol.add(indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()));
+ distinctCol.add(Pair.of(indexDef.getIndexType(),
+ indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));
}
if (distinct.size() != indexes.size()) {
throw new AnalysisException("index name must be unique.");
}
if (distinctCol.size() != indexes.size()) {
- throw new AnalysisException("same index columns have multiple index name is not allowed.");
+ throw new AnalysisException("same index columns have multiple same type index is not allowed.");
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
index 12cab9f7ec..06f8aa6318 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
@@ -137,6 +137,10 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
return indexes != null ? indexes : Lists.newArrayList();
}
+ public void setIndexes(List<Index> newIndexes) {
+ this.indexes = newIndexes;
+ }
+
public List<Column> getSchema() {
return getSchema(true);
}
diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
index 243f05ec2d..9bdff40418 100644
--- a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
+++ b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
@@ -85,7 +85,7 @@ suite("test_add_drop_index", "inverted_index"){
logger.info("create same duplicate with different name index, result: " + ex)
}
assertEquals(create_dup_index_result, "fail")
- // case1.3 create duplicate different index for one colume with same name
+ // case1.3 create duplicate different index for one colume with different name
sql "create index age_idx_diff on ${indexTbName1}(`age`) using bitmap"
wait_for_latest_op_on_table_finish(indexTbName1, timeout)
show_result = sql "show index from ${indexTbName1}"
diff --git a/regression-test/suites/inverted_index_p0/test_create_index_1.groovy b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy
new file mode 100644
index 0000000000..49bd5033dd
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_create_index_1", "inverted_index"){
+ // prepare test table
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail: " + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout")
+ }
+
+ def indexTbName1 = "test_create_index_1"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+ // case 1: create table with index
+ // case 1.1: create duplicate same index for one colume with same name
+ def create_dup_index_result = "fail"
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ name varchar(50),
+ age int NOT NULL,
+ grade int NOT NULL,
+ registDate datetime NULL,
+ studentInfo char(100),
+ tearchComment string,
+ INDEX age_idx(age) USING INVERTED COMMENT 'age index',
+ INDEX age_idx(age) USING INVERTED COMMENT 'age index'
+ )
+ DUPLICATE KEY(`name`)
+ DISTRIBUTED BY HASH(`name`) BUCKETS 10
+ properties("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with same name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.2: create duplicate same index for one colume with different name
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ name varchar(50),
+ age int NOT NULL,
+ grade int NOT NULL,
+ registDate datetime NULL,
+ studentInfo char(100),
+ tearchComment string,
+ INDEX age_idx_1(age) USING INVERTED COMMENT 'age index',
+ INDEX age_idx_2(age) USING INVERTED COMMENT 'age index'
+ )
+ DUPLICATE KEY(`name`)
+ DISTRIBUTED BY HASH(`name`) BUCKETS 10
+ properties("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with different name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.3: create duplicate different index for one colume with same name
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ name varchar(50),
+ age int NOT NULL,
+ grade int NOT NULL,
+ registDate datetime NULL,
+ studentInfo char(100),
+ tearchComment string,
+ INDEX age_idx(age) USING BITMAP COMMENT 'age index',
+ INDEX age_idx(age) USING INVERTED COMMENT 'age index'
+ )
+ DUPLICATE KEY(`name`)
+ DISTRIBUTED BY HASH(`name`) BUCKETS 10
+ properties("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate different index for one colume with same name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.4: create duplicate different index for one colume with different name
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ name varchar(50),
+ age int NOT NULL,
+ grade int NOT NULL,
+ registDate datetime NULL,
+ studentInfo char(100),
+ tearchComment string,
+ INDEX age_idx_1(age) USING BITMAP COMMENT 'age index',
+ INDEX age_idx_2(age) USING INVERTED COMMENT 'age index'
+ )
+ DUPLICATE KEY(`name`)
+ DISTRIBUTED BY HASH(`name`) BUCKETS 10
+ properties("replication_num" = "1");
+ """
+
+ def show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "age_idx_1")
+ assertEquals(show_result[1][2], "age_idx_2")
+
+ // drop index
+ sql "drop index age_idx_1 on ${indexTbName1}"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+ sql "drop index age_idx_2 on ${indexTbName1}"
+ show_result = sql "show index from ${indexTbName1}"
+ assertEquals(show_result.size(), 0)
+
+ // case 2: alter add index
+ sql "create index age_idx on ${indexTbName1}(age) using inverted"
+ show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result[0][2], "age_idx")
+ // case 2.1: create duplicate same index for one colume with same name
+ try {
+ sql "create index age_idx on ${indexTbName1}(`age`) using inverted"
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with same name, result: " + ex)
+ }
+
+ // case 2.2: create duplicate same index for one colume with different name
+ try {
+ sql "create index age_idx_2 on ${indexTbName1}(`age`) using inverted"
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with different name, result: " + ex)
+ }
+
+ // case 2.3: create duplicate different index for one colume with same name
+ try {
+ sql "create index age_idx on ${indexTbName1}(`age`) using bitmap"
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate different index for one colume with same name, result: " + ex)
+ }
+
+ // 2.4: create duplicate different index for one colume with different name
+ sql "create index age_idx_2 on ${indexTbName1}(`age`) using bitmap"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+ show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "age_idx")
+ assertEquals(show_result[1][2], "age_idx_2")
+}
diff --git a/regression-test/suites/inverted_index_p0/test_create_index_2.groovy b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy
new file mode 100644
index 0000000000..a9aa616971
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_create_index_2", "inverted_index"){
+ // prepare test table
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail: " + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout")
+ }
+
+ def indexTbName1 = "test_create_index_2"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+ // case 1: create table with index
+ // case 1.1: create duplicate same index for one colume with same name
+ def create_dup_index_result = "fail"
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ id INT DEFAULT '10',
+ name VARCHAR(32) DEFAULT '',
+ INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index',
+ INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index'
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with same name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.2: create duplicate same index for one colume with different name
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ id INT DEFAULT '10',
+ name VARCHAR(32) DEFAULT '',
+ INDEX name_idx_1(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index',
+ INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index'
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with different name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.3: create duplicate different index for one colume with same name
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ id INT DEFAULT '10',
+ name VARCHAR(32) DEFAULT '',
+ INDEX name_idx(name) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index',
+ INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index'
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate different index for one colume with same name, result: " + ex)
+ }
+ assertEquals(create_dup_index_result, "fail")
+
+ // case 1.4: create duplicate different index for one colume with different name
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ id INT DEFAULT '10',
+ name VARCHAR(32) DEFAULT '',
+ INDEX name_idx_1(name) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index',
+ INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index'
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+
+ def show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "name_idx_1")
+ assertEquals(show_result[1][2], "name_idx_2")
+
+ // drop index
+ sql "drop index name_idx_1 on ${indexTbName1}"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+ sql "drop index name_idx_2 on ${indexTbName1}"
+ show_result = sql "show index from ${indexTbName1}"
+ assertEquals(show_result.size(), 0)
+
+ // case 2: alter add index
+ sql """
+ create index name_idx on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index';
+ """
+ show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result[0][2], "name_idx")
+ // case 2.1: create duplicate same index for one colume with same name
+ try {
+ sql """
+ create index name_idx on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index';
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with same name, result: " + ex)
+ }
+
+ // case 2.2: create duplicate same index for one colume with different name
+ try {
+ sql """
+ create index name_idx_2 on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index';
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate same index for one colume with different name, result: " + ex)
+ }
+
+ // case 2.3: create duplicate different index for one colume with same name
+ try {
+ sql """
+ create index name_idx on ${indexTbName1}(name) using ngram_bf properties("gram_size"="3", "bf_size"="256") comment 'name index';
+ """
+ create_dup_index_result = "success"
+ } catch(Exception ex) {
+ logger.info("create duplicate different index for one colume with same name, result: " + ex)
+ }
+
+ // 2.4: create duplicate different index for one colume with different name
+ sql """
+ create index name_idx_2 on ${indexTbName1}(name) using ngram_bf properties("gram_size"="3", "bf_size"="256") comment 'name index';
+ """
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+ show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "name_idx")
+ assertEquals(show_result[1][2], "name_idx_2")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org