You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2025/04/09 06:37:08 UTC
(doris) branch branch-3.0 updated: [fix] (inverted index) Disallow variant columns from using inverted index format v1 (#49823) (#49890)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new d2cae4d21bf [fix] (inverted index) Disallow variant columns from using inverted index format v1 (#49823) (#49890)
d2cae4d21bf is described below
commit d2cae4d21bfe2c7d3bf1730f9c0a2c3f8f8e0a40
Author: Sun Chenyang <su...@selectdb.com>
AuthorDate: Wed Apr 9 14:37:00 2025 +0800
[fix] (inverted index) Disallow variant columns from using inverted index format v1 (#49823) (#49890)
pick from master #49823
---
.../main/java/org/apache/doris/common/Config.java | 4 +
.../apache/doris/alter/SchemaChangeHandler.java | 6 +-
.../org/apache/doris/analysis/CreateTableStmt.java | 12 +-
.../java/org/apache/doris/analysis/IndexDef.java | 14 +-
.../trees/plans/commands/info/CreateTableInfo.java | 9 +-
.../trees/plans/commands/info/IndexDefinition.java | 15 +-
.../org/apache/doris/analysis/IndexDefTest.java | 5 +-
.../trees/plans/commands/IndexDefinitionTest.java | 12 +-
...e_compaction_with_variant_inverted_index.groovy | 40 ++---
.../inverted_index_p0/load/test_insert.groovy | 18 ++-
.../inverted_index_p0/load/test_stream_load.groovy | 18 ++-
...sted_index_file_http_action_with_variant.groovy | 33 ++--
.../test_variant_index_format_v1.groovy | 27 ++--
.../suites/load_p2/test_single_replica_load.groovy | 27 ++--
.../test_array_contains_with_inverted_index.groovy | 172 +++++++++++----------
.../with_index/test_array_index_write.groovy | 133 ++++++++--------
.../test_array_inverted_index_profile.groovy | 164 ++++++++++----------
.../suites/variant_p0/with_index/var_index.groovy | 109 +++++++------
18 files changed, 445 insertions(+), 373 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 475a2060b42..3d8b2d3f3ee 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3320,4 +3320,8 @@ public class Config extends ConfigBase {
public static long meta_service_rpc_reconnect_interval_ms = 5000;
public static long meta_service_rpc_retry_cnt = 10;
+
+ @ConfField(mutable = true, masterOnly = true, description = {"是否允许 variant 类型的列使用倒排索引格式 v1",
+ "Whether to allow the use of inverted index v1 for variant"})
+ public static boolean enable_inverted_index_v1_for_variant = false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 162fb239b54..2b1d2b5aefe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -98,7 +98,6 @@ import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.ClearAlterTask;
import org.apache.doris.task.UpdateTabletMetaInfoTask;
-import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTaskType;
@@ -2733,15 +2732,12 @@ public class SchemaChangeHandler extends AlterHandler {
alterIndex.setIndexId(Env.getCurrentEnv().getNextId());
}
- boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat()
- == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable()
- .getDisableInvertedIndexV1ForVaraint();
for (String col : indexDef.getColumns()) {
Column column = olapTable.getColumn(col);
if (column != null) {
indexDef.checkColumn(column, olapTable.getKeysType(),
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
- disableInvertedIndexV1ForVariant);
+ olapTable.getInvertedIndexFileStorageFormat());
} else {
throw new DdlException("index column does not exist in table. invalid column: " + col);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 431f0dff8ff..70b08c6b379 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -569,9 +569,9 @@ public class CreateTableStmt extends DdlStmt implements NotFallbackInParser {
if (CollectionUtils.isNotEmpty(indexDefs)) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>();
- boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
- new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1
- && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint();
+ TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = PropertyAnalyzer
+ .analyzeInvertedIndexFileStorageFormat(new HashMap<>(properties));
+
for (IndexDef indexDef : indexDefs) {
indexDef.analyze();
if (!engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
@@ -581,8 +581,10 @@ public class CreateTableStmt extends DdlStmt implements NotFallbackInParser {
boolean found = false;
for (Column column : columns) {
if (column.getName().equalsIgnoreCase(indexColName)) {
- indexDef.checkColumn(column, getKeysDesc().getKeysType(),
- enableUniqueKeyMergeOnWrite, disableInvertedIndexV1ForVariant);
+ indexDef.checkColumn(column,
+ getKeysDesc().getKeysType(),
+ enableUniqueKeyMergeOnWrite,
+ invertedIndexFileStorageFormat);
found = true;
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index 0f07a953b9e..530c0c3f9ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -21,6 +21,8 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -213,7 +215,7 @@ public class IndexDef {
}
public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite,
- boolean disableInvertedIndexV1ForVariant) throws AnalysisException {
+ TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) throws AnalysisException {
if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER
|| indexType == IndexType.NGRAM_BF) {
String indexColName = column.getName();
@@ -228,7 +230,15 @@ public class IndexDef {
// In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS.
// when the subcolumn type changes, it may result in missing files, causing link file failure.
- if (colType.isVariantType() && disableInvertedIndexV1ForVariant) {
+ // There are two cases in which the inverted index format v1 is not supported:
+ // 1. in cloud mode
+ // 2. enable_inverted_index_v1_for_variant = false
+ boolean notSupportInvertedIndexForVariant =
+ (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1
+ || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT)
+ && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant);
+
+ if (colType.isVariantType() && notSupportInvertedIndexForVariant) {
throw new AnalysisException(colType + " is not supported in inverted index format V1,"
+ "Please set properties(\"inverted_index_storage_format\"= \"v2\"),"
+ "or upgrade to a newer version");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index c60b129c965..46319a583e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -597,11 +597,10 @@ public class CreateTableInfo {
if (!indexes.isEmpty()) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
Set<Pair<IndexDef.IndexType, List<String>>> distinctCol = new HashSet<>();
- boolean disableInvertedIndexV1ForVariant = false;
+ TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat;
try {
- disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
- new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1
- && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint();
+ invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
+ new HashMap<>(properties));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
@@ -617,7 +616,7 @@ public class CreateTableInfo {
for (ColumnDefinition column : columns) {
if (column.getName().equalsIgnoreCase(indexColName)) {
indexDef.checkColumn(column, keysType, isEnableMergeOnWrite,
- disableInvertedIndexV1ForVariant);
+ invertedIndexFileStorageFormat);
found = true;
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
index f3ad6e81ec1..77a90755250 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
@@ -23,9 +23,11 @@ import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
+import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
@@ -93,7 +95,8 @@ public class IndexDefinition {
* checkColumn
*/
public void checkColumn(ColumnDefinition column, KeysType keysType,
- boolean enableUniqueKeyMergeOnWrite, boolean disableInvertedIndexV1ForVariant) throws AnalysisException {
+ boolean enableUniqueKeyMergeOnWrite,
+ TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) throws AnalysisException {
if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED
|| indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) {
String indexColName = column.getName();
@@ -109,7 +112,15 @@ public class IndexDefinition {
// In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS.
// when the subcolumn type changes, it may result in missing files, causing link file failure.
- if (colType.isVariantType() && disableInvertedIndexV1ForVariant) {
+ // There are two cases in which the inverted index format v1 is not supported:
+ // 1. in cloud mode
+ // 2. enable_inverted_index_v1_for_variant = false
+ boolean notSupportInvertedIndexForVariant =
+ (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1
+ || invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT)
+ && (Config.isCloudMode() || !Config.enable_inverted_index_v1_for_variant);
+
+ if (colType.isVariantType() && notSupportInvertedIndexForVariant) {
throw new AnalysisException(colType + " is not supported in inverted index format V1,"
+ "Please set properties(\"inverted_index_storage_format\"= \"v2\"),"
+ "or upgrade to a newer version");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
index 56c78e2430a..3d528ce1b68 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.collect.Lists;
import org.junit.Assert;
@@ -66,9 +67,7 @@ public class IndexDefTest {
try {
def = new IndexDef("variant_index", false, Lists.newArrayList("col1"),
IndexDef.IndexType.INVERTED, null, "comment");
- boolean isIndexFormatV1 = true;
- def.checkColumn(new Column("col1", PrimitiveType.VARIANT), KeysType.UNIQUE_KEYS, true, isIndexFormatV1);
- Assert.fail("No exception throws.");
+ def.checkColumn(new Column("col1", PrimitiveType.VARIANT), KeysType.UNIQUE_KEYS, true, TInvertedIndexFileStorageFormat.V1);
} catch (AnalysisException e) {
Assert.assertTrue(e instanceof AnalysisException);
Assert.assertTrue(e.getMessage().contains("not supported in inverted index format V1"));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java
index 2afff52c40e..803ae6d508b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java
@@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VariantType;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
@@ -39,9 +40,8 @@ public class IndexDefinitionTest {
IndexDefinition def = new IndexDefinition("variant_index", Lists.newArrayList("col1"), "INVERTED",
null, "comment");
try {
- boolean isIndexFormatV1 = true;
def.checkColumn(new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true,
- null, "comment"), KeysType.UNIQUE_KEYS, true, isIndexFormatV1);
+ null, "comment"), KeysType.UNIQUE_KEYS, true, TInvertedIndexFileStorageFormat.V1);
Assertions.fail("No exception throws.");
} catch (AnalysisException e) {
org.junit.jupiter.api.Assertions.assertInstanceOf(
@@ -60,7 +60,7 @@ public class IndexDefinitionTest {
properties, "comment");
def.checkColumn(
new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null, "comment"),
- KeysType.DUP_KEYS, false, false);
+ KeysType.DUP_KEYS, false, null);
}
@Test
@@ -75,7 +75,7 @@ public class IndexDefinitionTest {
def.checkColumn(
new ColumnDefinition("col1", IntegerType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
- KeysType.DUP_KEYS, false, false));
+ KeysType.DUP_KEYS, false, null));
}
@Test
@@ -89,7 +89,7 @@ public class IndexDefinitionTest {
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
- KeysType.DUP_KEYS, false, false));
+ KeysType.DUP_KEYS, false, null));
}
@Test
@@ -103,6 +103,6 @@ public class IndexDefinitionTest {
Assertions.assertThrows(AnalysisException.class, () ->
def.checkColumn(new ColumnDefinition("col1", StringType.INSTANCE, false, AggregateType.NONE, true, null,
"comment"),
- KeysType.DUP_KEYS, false, false));
+ KeysType.DUP_KEYS, false, null));
}
}
diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
index dc178556ebb..3850f4b1b13 100644
--- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
+++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
@@ -132,26 +132,26 @@ suite("test_single_compaction_with_variant_inverted", "p2, nonConcurrent") {
sql """ DROP TABLE IF EXISTS ${tableName}; """
- sql """ set disable_inverted_index_v1_for_variant = false """
- sql """
- CREATE TABLE ${tableName} (
- `id` int(11) NULL,
- `name` varchar(255) NULL,
- `score` int(11) NULL,
- `properties` variant,
- INDEX idx_props (`properties`) USING INVERTED PROPERTIES("parser" = "none") COMMENT ''
- ) ENGINE=OLAP
- DUPLICATE KEY(`id`)
- COMMENT 'OLAP'
- DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES (
- "replication_num" = "2",
- "enable_single_replica_compaction" = "true",
- "inverted_index_storage_format" = "V1",
- "compaction_policy" = "time_series"
- );
- """
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL,
+ `name` varchar(255) NULL,
+ `score` int(11) NULL,
+ `properties` variant,
+ INDEX idx_props (`properties`) USING INVERTED PROPERTIES("parser" = "none") COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "2",
+ "enable_single_replica_compaction" = "true",
+ "inverted_index_storage_format" = "V1",
+ "compaction_policy" = "time_series"
+ );
+ """
+ }
def tablets = sql_return_maparray """ show tablets from ${tableName}; """
diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
index 3f80fccc56d..a5ca4edf099 100644
--- a/regression-test/suites/inverted_index_p0/load/test_insert.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
@@ -72,14 +72,20 @@ suite("test_insert_with_index", "p0, nonConcurrent") {
}
set_be_config("inverted_index_ram_dir_enable", "true")
- sql """ set disable_inverted_index_v1_for_variant = false """
- test.call("V1")
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ if (isCloudMode()) {
+ return;
+ }
+ test.call("V1")
+ }
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "false")
- sql """ set disable_inverted_index_v1_for_variant = false """
- test.call("V1")
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: false]) {
+ if (isCloudMode()) {
+ return;
+ }
+ test.call("V1")
+ }
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "true")
}
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
index 28875acc438..66c4468e3f2 100644
--- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
@@ -105,14 +105,20 @@ suite("test_stream_load_with_inverted_index_p0", "p0, nonConcurrent") {
}
set_be_config("inverted_index_ram_dir_enable", "true")
- sql """ set disable_inverted_index_v1_for_variant = false """
- test.call("V1")
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ if (isCloudMode()) {
+ return;
+ }
+ test.call("V1")
+ }
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "false")
- sql """ set disable_inverted_index_v1_for_variant = false """
- test.call("V1")
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: false]) {
+ if (isCloudMode()) {
+ return;
+ }
+ test.call("V1")
+ }
test.call("V2")
set_be_config("inverted_index_ram_dir_enable", "true")
}
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
index a7718e3927a..e242f858872 100644
--- a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
+++ b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
@@ -64,18 +64,18 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0"
def tableName = "test_show_nested_index_file_http_action_with_variant_" + format
sql "DROP TABLE IF EXISTS ${tableName}"
- sql """ set disable_inverted_index_v1_for_variant = false """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "${format}");
- """
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ k bigint,
+ v variant,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "${format}");
+ """
+ }
load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""")
@@ -125,8 +125,13 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0"
qt_sql """select count() from ${tableName} where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core'"""
}
- run_test("V1")
- run_test("V2")
+ if (isCloudMode()) {
+ run_test("V2")
+ } else {
+ run_test("V1")
+ run_test("V2")
+ }
+
set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "2147483648")
}
diff --git a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
index 2bf1d20b106..3e42674fc9a 100644
--- a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
+++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
@@ -60,18 +60,21 @@ suite("test_variant_index_format_v1", "p2, nonConcurrent") {
def table_name = "github_events"
sql """DROP TABLE IF EXISTS ${table_name}"""
- sql """ set disable_inverted_index_v1_for_variant = false """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_name} (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
- """
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ if (isCloudMode()) {
+ return;
+ }
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ k bigint,
+ v variant,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
+ }
set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296")
load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""")
diff --git a/regression-test/suites/load_p2/test_single_replica_load.groovy b/regression-test/suites/load_p2/test_single_replica_load.groovy
index 67f0b4639f8..68d1e687744 100644
--- a/regression-test/suites/load_p2/test_single_replica_load.groovy
+++ b/regression-test/suites/load_p2/test_single_replica_load.groovy
@@ -51,18 +51,21 @@ suite("test_single_replica_load", "p2, nonConcurrent") {
def tableName = "test_single_replica_load"
sql "DROP TABLE IF EXISTS ${tableName}"
- sql """ set disable_inverted_index_v1_for_variant = false """
- sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
- k bigint,
- v variant,
- INDEX idx(v) USING INVERTED PROPERTIES("parser"="standard") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "2", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
- """
- sql """ set disable_inverted_index_v1_for_variant = true """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ if (isCloudMode()) {
+ return;
+ }
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ k bigint,
+ v variant,
+ INDEX idx(v) USING INVERTED PROPERTIES("parser"="standard") COMMENT ''
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "2", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
+ }
load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
diff --git a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
index 06813514808..55b83e69811 100644
--- a/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
+++ b/regression-test/suites/variant_p0/with_index/test_array_contains_with_inverted_index.groovy
@@ -18,94 +18,98 @@
suite("test_array_contains_with_inverted_index"){
// prepare test table
def indexTblName = "tai"
- sql "set disable_inverted_index_v1_for_variant = false"
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
- // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true
- sql """ set enable_common_expr_pushdown = true; """
- sql """ set enable_common_expr_pushdown_for_inverted_index = true; """
- sql """ set enable_profile = true;"""
+ // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true
+ sql """ set enable_common_expr_pushdown = true; """
+ sql """ set enable_common_expr_pushdown_for_inverted_index = true; """
+ sql """ set enable_profile = true;"""
- sql "DROP TABLE IF EXISTS ${indexTblName}"
- // create 1 replica table
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- sql """
- CREATE TABLE IF NOT EXISTS `${indexTblName}` (
- `apply_date` date NULL COMMENT '',
- `id` varchar(60) NOT NULL COMMENT '',
- `inventors` variant NULL COMMENT '',
- INDEX index_inverted_inventors(inventors) USING INVERTED COMMENT ''
- ) ENGINE=OLAP
- DUPLICATE KEY(`apply_date`, `id`)
- COMMENT 'OLAP'
- DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "is_being_synced" = "false",
- "storage_format" = "V2",
- "light_schema_change" = "true",
- "disable_auto_compaction" = "false",
- "enable_single_replica_compaction" = "false",
- "inverted_index_storage_format" = "$storageFormat"
- );
- """
+ sql "DROP TABLE IF EXISTS ${indexTblName}"
+ // create 1 replica table
+ def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
+ if (storageFormat == "V1" && isCloudMode()) {
+ return;
+ }
+ sql """
+ CREATE TABLE IF NOT EXISTS `${indexTblName}` (
+ `apply_date` date NULL COMMENT '',
+ `id` varchar(60) NOT NULL COMMENT '',
+ `inventors` variant NULL COMMENT '',
+ INDEX index_inverted_inventors(inventors) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`apply_date`, `id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "inverted_index_storage_format" = "$storageFormat"
+ );
+ """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '{"inventors":["a", "b", "c"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '{"inventors":["d", "e", "f", "g", "h", "i", "j", "k", "l"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '48a33ec3453a28bce84b8f96fe161956', '{"inventors":["m"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '021603e7dcfe65d44af0efd0e5aee154', '{"inventors":["n"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '9fcb57ae675f0af4d613d9e6c0e8a2a2', '{"inventors":["o"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a5', '{"inventors":[]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '{"inventors":[null,null,null]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a7', '{"inventors":[null,null,null]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a8', '{"inventors":[]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a648a447b8f71522f11632eba4b4adde', '{"inventors":["p", "q", "r", "s", "t"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a9fb5c985c90bf05f3bee5ca3ae95260', '{"inventors":["u", "v"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '0974e7a82e30d1af83205e474fadd0a2', '{"inventors":["w"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '26823b3995ee38bd145ddd910b2f6300', '{"inventors":["x"]}'); """
- sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'ee27ee1da291e46403c408e220bed6e1', '{"inventors":["y"]}'); """
- sql """ set enable_common_expr_pushdown = true """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '{"inventors":["a", "b", "c"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '{"inventors":["d", "e", "f", "g", "h", "i", "j", "k", "l"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '48a33ec3453a28bce84b8f96fe161956', '{"inventors":["m"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '021603e7dcfe65d44af0efd0e5aee154', '{"inventors":["n"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '9fcb57ae675f0af4d613d9e6c0e8a2a2', '{"inventors":["o"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a5', '{"inventors":[]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '{"inventors":[null,null,null]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a7', '{"inventors":[null,null,null]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a8', '{"inventors":[]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a648a447b8f71522f11632eba4b4adde', '{"inventors":["p", "q", "r", "s", "t"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a9fb5c985c90bf05f3bee5ca3ae95260', '{"inventors":["u", "v"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '0974e7a82e30d1af83205e474fadd0a2', '{"inventors":["w"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '26823b3995ee38bd145ddd910b2f6300', '{"inventors":["x"]}'); """
+ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'ee27ee1da291e46403c408e220bed6e1', '{"inventors":["y"]}'); """
+ sql """ set enable_common_expr_pushdown = true """
- qt_sql """ select count() from ${indexTblName}"""
- def param_contains = ["'s'", "''", null]
- for (int i = 0 ; i < param_contains.size(); ++i) {
- def p = param_contains[i]
- log.info("param: ${p}")
- order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select * from tai where (array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
- }
+ qt_sql """ select count() from ${indexTblName}"""
+ def param_contains = ["'s'", "''", null]
+ for (int i = 0 ; i < param_contains.size(); ++i) {
+ def p = param_contains[i]
+ log.info("param: ${p}")
+ order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select * from tai where array_contains(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select * from tai where !array_contains(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select * from tai where (array_contains(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
+ }
- // test arrays_overlap with inverted index
- // now if we use inverted index we will not eval exprs
- def param = [["'s'", "'t'"], [], null, ["'s'", "''", "'t'"], ["'s'", null, "'t'"], [null, "''"], ["'s'", null, "'t'", "''"]] // null for arrays_overlap will return null which in predicate will lead to return empty set
- for (int i = 0 ; i < param.size(); ++i) {
- def p = param[i]
- log.info("param: ${p}")
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where (arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
- order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where (arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
+ // test arrays_overlap with inverted index
+ // now if we use inverted index we will not eval exprs
+ def param = [["'s'", "'t'"], [], null, ["'s'", "''", "'t'"], ["'s'", null, "'t'"], [null, "''"], ["'s'", null, "'t'", "''"]] // null for arrays_overlap will return null which in predicate will lead to return empty set
+ for (int i = 0 ; i < param.size(); ++i) {
+ def p = param[i]
+ log.info("param: ${p}")
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where !arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) or apply_date = '2017-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = true)*/ * from tai where (arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
+ order_qt_sql """ select /*+SET_VAR(enable_common_expr_pushdown = false)*/ * from tai where (arrays_overlap(cast(inventors['inventors'] as array<text>), ${p}) and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
+ }
}
}
diff --git a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
index ba37537a8cc..306a9860616 100644
--- a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
+++ b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy
@@ -16,78 +16,81 @@
// under the License.
suite("test_array_index_write", "nonConcurrent"){
- sql "set disable_inverted_index_v1_for_variant = false"
- def create_variant_index_table = { testTablex, parser ->
- def stmt = "CREATE TABLE IF NOT EXISTS " + testTablex + "(\n" +
- " k1 INT NULL,\n" +
- " c_arr VARIANT NULL COMMENT '',\n"
-
- String strTmp = parser == "" ? "INDEX index_inverted_c_arr(c_arr) USING INVERTED COMMENT 'c_arr index',\n" :
- "INDEX index_inverted_c_arr(c_arr) USING INVERTED PROPERTIES( \"parser\"=\" " + parser + "\") COMMENT 'c_arr index',\n"
-
- stmt += strTmp
- stmt = stmt.substring(0, stmt.length()-2)
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- stmt += ") \nENGINE=OLAP\n" +
- "DUPLICATE KEY(`k1`)\n" +
- "COMMENT 'OLAP'\n" +
- "DISTRIBUTED BY HASH(`k1`) BUCKETS 10\n" +
- "PROPERTIES(\"replication_num\" = \"1\", \"inverted_index_storage_format\" = \"$storageFormat\");"
- return stmt
- }
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ def create_variant_index_table = { testTablex, parser ->
+ def stmt = "CREATE TABLE IF NOT EXISTS " + testTablex + "(\n" +
+ " k1 INT NULL,\n" +
+ " c_arr VARIANT NULL COMMENT '',\n"
+
+ String strTmp = parser == "" ? "INDEX index_inverted_c_arr(c_arr) USING INVERTED COMMENT 'c_arr index',\n" :
+ "INDEX index_inverted_c_arr(c_arr) USING INVERTED PROPERTIES( \"parser\"=\" " + parser + "\") COMMENT 'c_arr index',\n"
+
+ stmt += strTmp
+ stmt = stmt.substring(0, stmt.length()-2)
+ def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
+ if (storageFormat == "V1" && isCloudMode()) {
+ return;
+ }
+ stmt += ") \nENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`)\n" +
+ "COMMENT 'OLAP'\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 10\n" +
+ "PROPERTIES(\"replication_num\" = \"1\", \"inverted_index_storage_format\" = \"$storageFormat\");"
+ return stmt
+ }
- def indexTbName = "test_variant_index_parser_empty"
- sql create_variant_index_table.call(indexTbName, "")
+ def indexTbName = "test_variant_index_parser_empty"
+ sql create_variant_index_table.call(indexTbName, "")
- def checkpoints_name = "array_inverted_index.write_index"
+ def checkpoints_name = "array_inverted_index.write_index"
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 3])
- sql "insert into ${indexTbName} values(1, '{\"c_arr\": [\"amory\", \"is\", \"committer\"]}')"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 3])
+ sql "insert into ${indexTbName} values(1, '{\"c_arr\": [\"amory\", \"is\", \"committer\"]}')"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 2])
- sql "insert into ${indexTbName} values(2, '{\"c_arr\": [\"amory\", \"better\"]}')"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 2])
+ sql "insert into ${indexTbName} values(2, '{\"c_arr\": [\"amory\", \"better\"]}')"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 1])
- sql "insert into ${indexTbName} values(3, '{\"c_arr\": [\"amory\", null]}')"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 1])
+ sql "insert into ${indexTbName} values(3, '{\"c_arr\": [\"amory\", null]}')"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
- sql "insert into ${indexTbName} values(4, '{\"c_arr\": [null, null]}')"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
+ sql "insert into ${indexTbName} values(4, '{\"c_arr\": [null, null]}')"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
- sql "insert into ${indexTbName} values(5, '{\"c_arr\": []}')"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
-
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
- sql "insert into ${indexTbName} values(6, null)"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
+ sql "insert into ${indexTbName} values(5, '{\"c_arr\": []}')"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- // test multiple types
- sql """insert into ${indexTbName} values(6, '{"c_arr" : ["text"]}')"""
- sql """insert into ${indexTbName} values(6, '{"c_arr" : [1.1]}')"""
- sql """insert into ${indexTbName} values(6, '{"c_arr" : [1.0]}')"""
- sql """insert into ${indexTbName} values(6, '{"c_arr" : [90]}')"""
- sql """insert into ${indexTbName} values(6, '{"c_arr" : [90999999999999]}')"""
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [single_array_field_count: 0])
+ sql "insert into ${indexTbName} values(6, null)"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
+ // test multiple types
+ sql """insert into ${indexTbName} values(6, '{"c_arr" : ["text"]}')"""
+ sql """insert into ${indexTbName} values(6, '{"c_arr" : [1.1]}')"""
+ sql """insert into ${indexTbName} values(6, '{"c_arr" : [1.0]}')"""
+ sql """insert into ${indexTbName} values(6, '{"c_arr" : [90]}')"""
+ sql """insert into ${indexTbName} values(6, '{"c_arr" : [90999999999999]}')"""
+ }
}
diff --git a/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy b/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy
index 8174a71060a..eb883a5d77f 100644
--- a/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy
+++ b/regression-test/suites/variant_p0/with_index/test_array_inverted_index_profile.groovy
@@ -65,93 +65,97 @@ suite("test_variant_arrayInvertedIdx_profile", "nonConcurrent"){
sql """ set enable_common_expr_pushdown_for_inverted_index = true; """
sql """ set enable_profile = true;"""
sql """ set profile_level = 2;"""
- sql "set disable_inverted_index_v1_for_variant = false"
-
- sql "DROP TABLE IF EXISTS ${indexTblName}"
- def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
- // create 1 replica table
- sql """
- CREATE TABLE IF NOT EXISTS `${indexTblName}` (
- `apply_date` date NULL COMMENT '',
- `id` varchar(60) NOT NULL COMMENT '',
- `inventors` variant NULL COMMENT '',
- INDEX index_inverted_inventors(inventors) USING INVERTED COMMENT ''
- ) ENGINE=OLAP
- DUPLICATE KEY(`apply_date`, `id`)
- COMMENT 'OLAP'
- DISTRIBUTED BY HASH(`id`) BUCKETS 1
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "is_being_synced" = "false",
- "storage_format" = "V2",
- "light_schema_change" = "true",
- "disable_auto_compaction" = "false",
- "enable_single_replica_compaction" = "false",
- "inverted_index_storage_format" = "$storageFormat"
- );
- """
-
- sql """ INSERT INTO `var_arr_idx` (`apply_date`, `id`, `inventors`) VALUES
- ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '{"inventors":["a", "b", "c"]}'),
- ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3', '{"inventors":[]}'),
- ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '{"inventors": ["d", "e", "f", "g", "h", "i", "j", "k", "l"]}'),
- ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL),
- ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332e', '{"inventors": ["m", "n", "o", "p", "q", "r", "s", "t", "u"]}'),
- ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '{"inventors": [null,null,null]}'),
- ('2019-01-01', 'd93d942d985a8fb7547c72dada8d332f', '{"inventors": ["v", "w", "x", "y", "z"]}'); """
-
-
- qt_sql1 """ select count() from ${indexTblName}"""
- def checkpoints_name = "array_func.array_contains"
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
- order_qt_sql2 "select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id;"
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
- }
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
- int randomInt = new Random().nextInt(10)
+ sql "DROP TABLE IF EXISTS ${indexTblName}"
+ def storageFormat = new Random().nextBoolean() ? "V1" : "V2"
+ if (storageFormat == "V1" && isCloudMode()) {
+ return;
+ }
+ // create 1 replica table
+ sql """
+ CREATE TABLE IF NOT EXISTS `${indexTblName}` (
+ `apply_date` date NULL COMMENT '',
+ `id` varchar(60) NOT NULL COMMENT '',
+ `inventors` variant NULL COMMENT '',
+ INDEX index_inverted_inventors(inventors) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`apply_date`, `id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "inverted_index_storage_format" = "$storageFormat"
+ );
+ """
+
+ sql """ INSERT INTO `var_arr_idx` (`apply_date`, `id`, `inventors`) VALUES
+ ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '{"inventors":["a", "b", "c"]}'),
+ ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3', '{"inventors":[]}'),
+ ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '{"inventors": ["d", "e", "f", "g", "h", "i", "j", "k", "l"]}'),
+ ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL),
+ ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332e', '{"inventors": ["m", "n", "o", "p", "q", "r", "s", "t", "u"]}'),
+ ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '{"inventors": [null,null,null]}'),
+ ('2019-01-01', 'd93d942d985a8fb7547c72dada8d332f', '{"inventors": ["v", "w", "x", "y", "z"]}'); """
+
+
+ qt_sql1 """ select count() from ${indexTblName}"""
+ def checkpoints_name = "array_func.array_contains"
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
+ order_qt_sql2 "select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id;"
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
- if (randomInt % 2) {
- profile("test_profile_time_${randomInt}") {
- run {
- sql "/* test_profile_time_${randomInt} */ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id"
- }
+ int randomInt = new Random().nextInt(10)
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsInvertedIndexFiltered: 6"))
- }
- }
- } else {
- profile("test_profile_time_${randomInt}") {
- run {
- sql "/* test_profile_time_${randomInt} */ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id"
- }
+ if (randomInt % 2) {
+ profile("test_profile_time_${randomInt}") {
+ run {
+ sql "/* test_profile_time_${randomInt} */ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id"
+ }
- check { profileString, exception ->
- log.info(profileString)
- assertTrue(profileString.contains("RowsInvertedIndexFiltered: 5"))
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsInvertedIndexFiltered: 6"))
+ }
+ }
+ } else {
+ profile("test_profile_time_${randomInt}") {
+ run {
+ sql "/* test_profile_time_${randomInt} */ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id"
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsInvertedIndexFiltered: 5"))
+ }
}
}
- }
-
+
- // checkRowsInvertedIndexFilter.call("select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id;", 6)
+ // checkRowsInvertedIndexFilter.call("select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 'w') order by id;", 6)
- try {
- GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
- order_qt_sql3 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id; """
- } finally {
- GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [result_bitmap: 1])
+ order_qt_sql3 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id; """
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
+ }
+ // and apply_date will be vectorized filter left is 6 rows for inverted index
+ order_qt_sql4 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2019-01-01' order by id; """
+
+ order_qt_sql5 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') or apply_date = '2017-01-01' order by id; """
+ order_qt_sql6 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') order by id; """
+ order_qt_sql7 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id; """
+ order_qt_sql8 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2019-01-01' order by id; """
+ order_qt_sql9 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') or apply_date = '2017-01-01' order by id; """
+ order_qt_sql10 """ select apply_date,id, inventors['inventors'] from var_arr_idx where (array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
}
- // and apply_date will be vectorized filter left is 6 rows for inverted index
- order_qt_sql4 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2019-01-01' order by id; """
-
- order_qt_sql5 """ select apply_date,id, inventors['inventors'] from var_arr_idx where array_contains(cast(inventors['inventors'] as array<text>), 's') or apply_date = '2017-01-01' order by id; """
- order_qt_sql6 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') order by id; """
- order_qt_sql7 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01' order by id; """
- order_qt_sql8 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2019-01-01' order by id; """
- order_qt_sql9 """ select apply_date,id, inventors['inventors'] from var_arr_idx where !array_contains(cast(inventors['inventors'] as array<text>), 's') or apply_date = '2017-01-01' order by id; """
- order_qt_sql10 """ select apply_date,id, inventors['inventors'] from var_arr_idx where (array_contains(cast(inventors['inventors'] as array<text>), 's') and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """
}
diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy
index 10cb0218ad2..b846043b30c 100644
--- a/regression-test/suites/variant_p0/with_index/var_index.groovy
+++ b/regression-test/suites/variant_p0/with_index/var_index.groovy
@@ -47,7 +47,6 @@ suite("regression_test_variant_var_index", "p0, nonConcurrent"){
sql """insert into var_index values(11, '{"nested": [{"a" : 1}]}'),(11, '{"nested": [{"b" : "1024"}]}')"""
qt_sql "select * from var_index order by k limit 15"
- sql """ set disable_inverted_index_v1_for_variant = true """
sql "DROP TABLE IF EXISTS var_index"
try {
sql """
@@ -82,53 +81,71 @@ suite("regression_test_variant_var_index", "p0, nonConcurrent"){
assertTrue(e.getMessage().contains("not supported in inverted index format V1"))
}
- sql """ set disable_inverted_index_v1_for_variant = false """
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant,
- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
- """
+ setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) {
+ if (isCloudMode()) {
+ sql "DROP TABLE IF EXISTS var_index"
+ try {
+ sql """
+ CREATE TABLE IF NOT EXISTS var_index (
+ k bigint,
+ v variant,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("not supported in inverted index format V1"))
+ }
- sql "DROP TABLE IF EXISTS var_index"
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
- """
- sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
- sql """ set disable_inverted_index_v1_for_variant = true """
+ sql """
+ CREATE TABLE IF NOT EXISTS var_index (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
- try {
- sql """ build index idx_var on var_index"""
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("The idx_var index can not be built on the v column, because it is a variant type column"))
- }
+ try {
+ sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("not supported in inverted index format V1"))
+ }
+ } else {
+ sql """
+ CREATE TABLE IF NOT EXISTS var_index (
+ k bigint,
+ v variant,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT ''
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
- sql "DROP TABLE IF EXISTS var_index"
- sql """
- CREATE TABLE IF NOT EXISTS var_index (
- k bigint,
- v variant
- )
- DUPLICATE KEY(`k`)
- DISTRIBUTED BY HASH(k) BUCKETS 1
- properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V2");
- """
- sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
- try {
- sql """ build index idx_var on var_index"""
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("The idx_var index can not be built on the v column, because it is a variant type column"))
+ sql "DROP TABLE IF EXISTS var_index"
+ sql """
+ CREATE TABLE IF NOT EXISTS var_index (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+ """
+ sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
+ try {
+ sql """ build index idx_var on var_index"""
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains("The idx_var index can not be built on the v column, because it is a variant type column"))
+ }
+ }
+
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org