You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2026/03/05 04:08:01 UTC
(doris) branch master updated: [feat](Variant) add variant_enable_nested_group property and fix variant nullable issue (#61047)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 405d8df83ab [feat](Variant) add variant_enable_nested_group property and fix variant nullable issue (#61047)
405d8df83ab is described below
commit 405d8df83aba541e32ee4aff688481d10e8b276c
Author: lihangyu <li...@selectdb.com>
AuthorDate: Thu Mar 5 12:07:54 2026 +0800
[feat](Variant) add variant_enable_nested_group property and fix variant nullable issue (#61047)
---
.../variant/variant_column_writer_impl.cpp | 27 +++++++++++-------
be/src/olap/tablet_meta.cpp | 3 ++
be/src/olap/tablet_schema.cpp | 4 +++
be/src/olap/tablet_schema.h | 6 ++++
.../vec/functions/array/function_array_utils.cpp | 5 +++-
.../java/org/apache/doris/catalog/ScalarType.java | 7 +++++
.../java/org/apache/doris/catalog/VariantType.java | 18 ++++++++++--
.../main/java/org/apache/doris/catalog/Column.java | 9 ++++++
.../apache/doris/common/util/PropertyAnalyzer.java | 33 ++++++++++++++++++++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 16 +++++++++--
.../org/apache/doris/nereids/types/DataType.java | 3 +-
.../apache/doris/nereids/types/VariantType.java | 11 ++++++--
.../java/org/apache/doris/qe/SessionVariable.java | 13 +++++++++
.../ExternalFileTableValuedFunction.java | 3 +-
gensrc/proto/olap_file.proto | 1 +
gensrc/thrift/Descriptors.thrift | 1 +
16 files changed, 140 insertions(+), 20 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
index f5de9c1e651..e29bdb0141f 100644
--- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp
@@ -1308,11 +1308,13 @@ Status VariantColumnWriterImpl::finalize() {
column->parent_unique_id() == current_variant_uid;
});
if (!has_extracted_columns) {
- RETURN_IF_ERROR(build_nested_group_routing_plan(*ptr, &_nested_group_routing_plan));
+ if (_tablet_column->variant_enable_nested_group()) {
+ RETURN_IF_ERROR(build_nested_group_routing_plan(*ptr, &_nested_group_routing_plan));
- // Root NG dedup is handled in _process_root_column() — see the
- // has_root_ng check there. We intentionally do NOT modify the in-memory
- // root data here because _nested_group_provider->prepare() needs it.
+ // Root NG dedup is handled in _process_root_column() — see the
+ // has_root_ng check there. We intentionally do NOT modify the in-memory
+ // root data here because _nested_group_provider->prepare() needs it.
+ }
}
RETURN_IF_ERROR(ptr->pick_subcolumns_to_sparse_column(
@@ -1341,9 +1343,12 @@ Status VariantColumnWriterImpl::finalize() {
}
// NestedGroup write behavior is determined by the injected provider implementation.
- RETURN_IF_ERROR(_nested_group_provider->prepare(
- *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
- olap_data_convertor.get(), num_rows, &column_id, &_statistics));
+ // Only invoke the provider when nested group writing is enabled.
+ if (_tablet_column->variant_enable_nested_group()) {
+ RETURN_IF_ERROR(_nested_group_provider->prepare(
+ *ptr, /*include_jsonb_subcolumns=*/true, _tablet_column, _opts,
+ olap_data_convertor.get(), num_rows, &column_id, &_statistics));
+ }
if (_binary_writer) {
_binary_writer->merge_stats_to(&_statistics);
}
@@ -1558,9 +1563,11 @@ Status VariantSubcolumnWriter::finalize() {
_opts.meta->set_num_rows(ptr->rows());
++column_id;
- RETURN_IF_ERROR(_nested_group_provider->prepare(*ptr, /*include_jsonb_subcolumns=*/false,
- &flush_column, _opts, olap_data_convertor.get(),
- ptr->rows(), &column_id, &_statistics));
+ if (parent_column.variant_enable_nested_group()) {
+ RETURN_IF_ERROR(_nested_group_provider->prepare(
+ *ptr, /*include_jsonb_subcolumns=*/false, &flush_column, _opts,
+ olap_data_convertor.get(), ptr->rows(), &column_id, &_statistics));
+ }
_statistics.to_pb(_opts.meta->mutable_variant_statistics());
_is_finalized = true;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 8a4c9da9fd2..be701d54f7c 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -562,6 +562,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
if (tcolumn.__isset.variant_doc_hash_shard_count) {
column->set_variant_doc_hash_shard_count(tcolumn.variant_doc_hash_shard_count);
}
+ if (tcolumn.__isset.variant_enable_nested_group) {
+ column->set_variant_enable_nested_group(tcolumn.variant_enable_nested_group);
+ }
}
void TabletMeta::remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version) {
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 051b8b075cf..a7295b6a941 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -678,6 +678,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
if (column.has_variant_doc_hash_shard_count()) {
_variant.doc_hash_shard_count = column.variant_doc_hash_shard_count();
}
+ if (column.has_variant_enable_nested_group()) {
+ _variant.enable_nested_group = column.variant_enable_nested_group();
+ }
if (column.has_pattern_type()) {
_pattern_type = column.pattern_type();
}
@@ -764,6 +767,7 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
column->set_variant_enable_doc_mode(_variant.enable_doc_mode);
column->set_variant_doc_materialization_min_rows(_variant.doc_materialization_min_rows);
column->set_variant_doc_hash_shard_count(_variant.doc_hash_shard_count);
+ column->set_variant_enable_nested_group(_variant.enable_nested_group);
}
void TabletColumn::add_sub_column(TabletColumn& sub_column) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 5fdd35f8a6d..96baf5c45f3 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -80,6 +80,8 @@ public:
bool enable_doc_mode = false;
int64_t doc_materialization_min_rows = 0;
int32_t doc_hash_shard_count = 64;
+
+ bool enable_nested_group = false;
};
TabletColumn();
@@ -279,6 +281,10 @@ public:
_variant.enable_typed_paths_to_sparse = variant_enable_typed_paths_to_sparse;
}
+ bool variant_enable_nested_group() const { return _variant.enable_nested_group; }
+
+ void set_variant_enable_nested_group(bool val) { _variant.enable_nested_group = val; }
+
bool is_decimal() const { return _is_decimal; }
private:
diff --git a/be/src/vec/functions/array/function_array_utils.cpp b/be/src/vec/functions/array/function_array_utils.cpp
index 7620d7179c8..fc96cb7288c 100644
--- a/be/src/vec/functions/array/function_array_utils.cpp
+++ b/be/src/vec/functions/array/function_array_utils.cpp
@@ -27,6 +27,7 @@
#include "vec/columns/column_variant.h"
#include "vec/columns/column_vector.h"
#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
namespace doris::vectorized {
@@ -58,7 +59,9 @@ bool extract_column_array_info(const IColumn& src, ColumnArrayExecutionData& dat
data.nested_type->get_primitive_type() != PrimitiveType::TYPE_VARIANT) {
// set variant root column/type to from column/type
auto variant = ColumnVariant::create(true /*always nullable*/);
- variant->create_root(data.nested_type, make_nullable(data.nested_col)->assume_mutable());
+ auto nullable_nested_type = make_nullable(data.nested_type);
+ auto nullable_col = make_nullable(data.nested_col);
+ variant->create_root(nullable_nested_type, std::move(*nullable_col).mutate());
data.nested_col = variant->get_ptr();
}
return true;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index 99f7ac60a21..dc73f384e14 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -1066,4 +1066,11 @@ public class ScalarType extends Type {
// Backward-compatible default bucket count used by BE doc snapshot writer/reader.
return 128;
}
+
+ public boolean getVariantEnableNestedGroup() {
+ if (this instanceof VariantType) {
+ return ((VariantType) this).getEnableNestedGroup();
+ }
+ return false; // Default to false for backward compatibility.
+ }
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
index da062d21489..69917dbf4b7 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/VariantType.java
@@ -61,6 +61,9 @@ public class VariantType extends ScalarType {
@SerializedName(value = "variantDocShardCount")
private final int variantDocShardCount;
+ @SerializedName(value = "enableNestedGroup")
+ private final boolean enableNestedGroup;
+
private Map<String, String> properties = Maps.newHashMap();
public VariantType() {
@@ -73,6 +76,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields) {
@@ -89,6 +93,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(Map<String, String> properties) {
@@ -102,6 +107,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields, Map<String, String> properties) {
@@ -119,6 +125,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
public VariantType(ArrayList<VariantField> fields, int variantMaxSubcolumnsCount,
@@ -127,7 +134,8 @@ public class VariantType extends ScalarType {
int variantSparseHashShardCount,
boolean enableVariantDocMode,
long variantDocMaterializationMinRows,
- int variantDocShardCount) {
+ int variantDocShardCount,
+ boolean enableNestedGroup) {
super(PrimitiveType.VARIANT);
Preconditions.checkNotNull(fields);
this.predefinedFields = fields;
@@ -141,6 +149,7 @@ public class VariantType extends ScalarType {
this.enableVariantDocMode = enableVariantDocMode;
this.variantDocMaterializationMinRows = variantDocMaterializationMinRows;
this.variantDocShardCount = variantDocShardCount;
+ this.enableNestedGroup = enableNestedGroup;
}
@Override
@@ -218,7 +227,8 @@ public class VariantType extends ScalarType {
&& variantMaxSubcolumnsCount == otherVariantType.variantMaxSubcolumnsCount
&& enableTypedPathsToSparse == otherVariantType.enableTypedPathsToSparse
&& enableVariantDocMode == otherVariantType.enableVariantDocMode
- && variantDocMaterializationMinRows == otherVariantType.variantDocMaterializationMinRows;
+ && variantDocMaterializationMinRows == otherVariantType.variantDocMaterializationMinRows
+ && enableNestedGroup == otherVariantType.enableNestedGroup;
}
@Override
@@ -257,4 +267,8 @@ public class VariantType extends ScalarType {
public int getVariantDocShardCount() {
return variantDocShardCount;
}
+
+ public boolean getEnableNestedGroup() {
+ return enableNestedGroup;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 457259856cf..bf525428061 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -659,6 +659,7 @@ public class Column implements GsonPostProcessable {
tColumn.setVariantEnableDocMode(this.getVariantEnableDocMode());
tColumn.setVariantDocMaterializationMinRows(this.getvariantDocMaterializationMinRows());
tColumn.setVariantDocHashShardCount(this.getVariantDocShardCount());
+ tColumn.setVariantEnableNestedGroup(this.getVariantEnableNestedGroup());
// ATTN:
// Currently, this `toThrift()` method is only used from CreateReplicaTask.
// And CreateReplicaTask does not need `defineExpr` field.
@@ -886,6 +887,7 @@ public class Column implements GsonPostProcessable {
builder.setVariantEnableDocMode(this.getVariantEnableDocMode());
builder.setVariantDocMaterializationMinRows(this.getvariantDocMaterializationMinRows());
builder.setVariantDocHashShardCount(this.getVariantDocShardCount());
+ builder.setVariantEnableNestedGroup(this.getVariantEnableNestedGroup());
// variant may contain predefined structured fields
addChildren(builder);
}
@@ -977,6 +979,9 @@ public class Column implements GsonPostProcessable {
if (this.getVariantDocShardCount() != other.getVariantDocShardCount()) {
throw new DdlException("Can not change variant doc snapshot shard count");
}
+ if (this.getVariantEnableNestedGroup() != other.getVariantEnableNestedGroup()) {
+ throw new DdlException("Can not change variant enable nested group");
+ }
if (CollectionUtils.isNotEmpty(this.getChildren()) || CollectionUtils.isNotEmpty(other.getChildren())) {
throw new DdlException("Can not change variant schema templates");
}
@@ -1336,6 +1341,10 @@ public class Column implements GsonPostProcessable {
return type.isVariantType() ? ((ScalarType) type).getVariantDocShardCount() : 128;
}
+ public boolean getVariantEnableNestedGroup() {
+ return type.isVariantType() ? ((ScalarType) type).getVariantEnableNestedGroup() : false;
+ }
+
public void setFieldPatternType(TPatternType type) {
fieldPatternType = type;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 0d31eeb769b..4a9818aed7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -259,6 +259,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT = "variant_max_subcolumns_count";
public static final String PROPERTIES_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE = "variant_enable_typed_paths_to_sparse";
+ public static final String PROPERTIES_VARIANT_ENABLE_NESTED_GROUP = "variant_enable_nested_group";
public static final String PROPERTIES_TDE_ALGORITHM = "tde_algorithm";
public static final String AES256 = "AES256";
public static final String SM4 = "SM4";
@@ -2063,6 +2064,21 @@ public class PropertyAnalyzer {
return enableTypedPathsToSparse;
}
+ public static boolean analyzeEnableNestedGroup(Map<String, String> properties,
+ boolean defaultValue) throws AnalysisException {
+ boolean enableNestedGroup = defaultValue;
+ if (properties != null && properties.containsKey(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP)) {
+ String enableNestedGroupStr = properties.get(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
+ try {
+ enableNestedGroup = Boolean.parseBoolean(enableNestedGroupStr);
+ } catch (Exception e) {
+ throw new AnalysisException("variant_enable_nested_group must be `true` or `false`");
+ }
+ properties.remove(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
+ }
+ return enableNestedGroup;
+ }
+
public static int analyzeVariantMaxSparseColumnStatisticsSize(Map<String, String> properties, int defuatValue)
throws AnalysisException {
int maxSparseColumnStatisticsSize = defuatValue;
@@ -2176,6 +2192,23 @@ public class PropertyAnalyzer {
+ "cannot be set together");
}
}
+ // variant_enable_nested_group=true is mutually exclusive with
+ // variant_enable_doc_mode=true and variant_max_subcolumns_count>0
+ if (properties != null && properties.containsKey(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP)
+ && "true".equalsIgnoreCase(properties.get(PROPERTIES_VARIANT_ENABLE_NESTED_GROUP))) {
+ if (properties.containsKey(PROPERTIES_VARIANT_ENABLE_DOC_MODE)
+ && "true".equalsIgnoreCase(properties.get(PROPERTIES_VARIANT_ENABLE_DOC_MODE))) {
+ throw new AnalysisException("variant_enable_nested_group and variant_enable_doc_mode "
+ + "cannot both be true");
+ }
+ if (properties.containsKey(PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT)) {
+ int count = Integer.parseInt(properties.get(PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT));
+ if (count > 0) {
+ throw new AnalysisException("variant_enable_nested_group cannot be true when "
+ + "variant_max_subcolumns_count > 0");
+ }
+ }
+ }
}
public static TEncryptionAlgorithm analyzeTDEAlgorithm(Map<String, String> properties) throws AnalysisException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index af20d91c38c..2675541b4f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -5155,6 +5155,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
int variantDocHashShardCount = ConnectContext.get() == null ? 128 :
ConnectContext.get().getSessionVariable().getDefaultVariantDocHashShardCount();
+ boolean enableNestedGroup = ConnectContext.get() == null ? true :
+ ConnectContext.get().getSessionVariable().getDefaultVariantEnableNestedGroup();
+
try {
// validate properties: variant_enable_doc_mode cannot be set together with other properties
PropertyAnalyzer.validateVariantProperties(properties);
@@ -5173,10 +5176,17 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
.analyzeVariantDocMaterializationMinRows(properties, variantDocMaterializationMinRows);
variantDocHashShardCount = PropertyAnalyzer
.analyzeVariantDocHashShardCount(properties, variantDocHashShardCount);
+ enableNestedGroup = PropertyAnalyzer
+ .analyzeEnableNestedGroup(properties, enableNestedGroup);
} catch (org.apache.doris.common.AnalysisException e) {
throw new NotSupportedException(e.getMessage());
}
+ if (enableNestedGroup) {
+ throw new NotSupportedException(
+ "variant_enable_nested_group is not supported now");
+ }
+
// When doc mode is enabled, disable subcolumn extraction and sparse column features
if (enableVariantDocMode) {
// Disable sparse column features
@@ -5214,12 +5224,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
+ " and " + PropertyAnalyzer.PROPERTIES_VARIANT_SPARSE_HASH_SHARD_COUNT
+ " and " + PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_DOC_MODE
+ " and " + PropertyAnalyzer.PROPERTIES_VARIANT_DOC_MATERIALIZATION_MIN_ROWS
- + " and " + PropertyAnalyzer.PROPERTIES_VARIANT_DOC_HASH_SHARD_COUNT);
+ + " and " + PropertyAnalyzer.PROPERTIES_VARIANT_DOC_HASH_SHARD_COUNT
+ + " and " + PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_NESTED_GROUP);
}
return new VariantType(fields, variantMaxSubcolumnsCount, enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize, variantSparseHashShardCount,
- enableVariantDocMode, variantDocMaterializationMinRows, variantDocHashShardCount);
+ enableVariantDocMode, variantDocMaterializationMinRows, variantDocHashShardCount,
+ enableNestedGroup);
}
private static boolean isSupportedVariantDocModeType(DataType type) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 20095922991..61041674db2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -493,7 +493,8 @@ public abstract class DataType {
((org.apache.doris.catalog.VariantType) type).getVariantSparseHashShardCount(),
((org.apache.doris.catalog.VariantType) type).getEnableVariantDocMode(),
((org.apache.doris.catalog.VariantType) type).getvariantDocMaterializationMinRows(),
- ((org.apache.doris.catalog.VariantType) type).getVariantDocShardCount());
+ ((org.apache.doris.catalog.VariantType) type).getVariantDocShardCount(),
+ ((org.apache.doris.catalog.VariantType) type).getEnableNestedGroup());
}
return VariantType.INSTANCE;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
index 4ac0e597e82..16f7d72076f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java
@@ -55,6 +55,7 @@ public class VariantType extends PrimitiveType {
private final boolean enableVariantDocMode;
private final long variantDocMaterializationMinRows;
private final int variantDocShardCount;
+ private final boolean enableNestedGroup;
/**
* Creates a Variant type without predefined fields and only configures the max subcolumn limit.
@@ -70,6 +71,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
/**
@@ -84,6 +86,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = false;
this.variantDocMaterializationMinRows = 0L;
this.variantDocShardCount = 64;
+ this.enableNestedGroup = false;
}
/**
@@ -100,7 +103,8 @@ public class VariantType extends PrimitiveType {
public VariantType(List<VariantField> fields, int variantMaxSubcolumnsCount,
boolean enableTypedPathsToSparse, int variantMaxSparseColumnStatisticsSize,
int variantSparseHashShardCount, boolean enableVariantDocMode,
- long variantDocMaterializationMinRows, int variantDocShardCount) {
+ long variantDocMaterializationMinRows, int variantDocShardCount,
+ boolean enableNestedGroup) {
this.predefinedFields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null"));
this.variantMaxSubcolumnsCount = variantMaxSubcolumnsCount;
this.enableTypedPathsToSparse = enableTypedPathsToSparse;
@@ -109,6 +113,7 @@ public class VariantType extends PrimitiveType {
this.enableVariantDocMode = enableVariantDocMode;
this.variantDocMaterializationMinRows = variantDocMaterializationMinRows;
this.variantDocShardCount = variantDocShardCount;
+ this.enableNestedGroup = enableNestedGroup;
}
@Override
@@ -117,7 +122,7 @@ public class VariantType extends PrimitiveType {
.collect(Collectors.toList()), variantMaxSubcolumnsCount, enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize, variantSparseHashShardCount,
enableVariantDocMode, variantDocMaterializationMinRows,
- variantDocShardCount);
+ variantDocShardCount, enableNestedGroup);
}
@Override
@@ -126,7 +131,7 @@ public class VariantType extends PrimitiveType {
.map(VariantField::toCatalogDataType)
.collect(Collectors.toCollection(ArrayList::new)), variantMaxSubcolumnsCount, enableTypedPathsToSparse,
variantMaxSparseColumnStatisticsSize, variantSparseHashShardCount, enableVariantDocMode,
- variantDocMaterializationMinRows, variantDocShardCount);
+ variantDocMaterializationMinRows, variantDocShardCount, enableNestedGroup);
return type;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 362b69533d2..f92d4566f34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -952,6 +952,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String DEFAULT_VARIANT_DOC_HASH_SHARD_COUNT = "default_variant_doc_hash_shard_count";
+ public static final String DEFAULT_VARIANT_ENABLE_NESTED_GROUP = "default_variant_enable_nested_group";
+
public static final String MULTI_DISTINCT_STRATEGY = "multi_distinct_strategy";
public static final String AGG_PHASE = "agg_phase";
@@ -3443,6 +3445,13 @@ public class SessionVariable implements Serializable, Writable {
)
public int defaultVariantDocHashShardCount = 64;
+ @VariableMgr.VarAttr(
+ name = DEFAULT_VARIANT_ENABLE_NESTED_GROUP,
+ needForward = true,
+ fuzzy = true
+ )
+ public boolean defaultVariantEnableNestedGroup = false;
+
@VariableMgr.VarAttr(
name = "use_v3_storage_format",
fuzzy = true,
@@ -6153,6 +6162,10 @@ public class SessionVariable implements Serializable, Writable {
return defaultVariantDocHashShardCount;
}
+ public boolean getDefaultVariantEnableNestedGroup() {
+ return defaultVariantEnableNestedGroup;
+ }
+
public void readAffectQueryResultVariables(BiConsumer<String, Object> variablesReader) {
for (Field affectQueryResultField : affectQueryResultFields) {
String name = affectQueryResultField.getName();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 5470c83222f..df1a6bd6a10 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -402,7 +402,8 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
/*variantSparseHashShardCount*/ 0,
/*variantEnableDocMode*/ false,
/*variantDocMaterializationMinRows*/ 0,
- /*variantDocShardCount*/ 0);
+ /*variantDocShardCount*/ 0,
+ /*enableNestedGroup*/ false);
parsedNodes = 1;
} else {
type = ScalarType.createType(PrimitiveType.fromThrift(tPrimitiveType),
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index ff54f54aed7..aadacd37879 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -390,6 +390,7 @@ message ColumnPB {
optional int64 variant_doc_materialization_min_rows = 32;
// Number of buckets used to store doc map in variant doc mode.
optional int32 variant_doc_hash_shard_count = 33 [default = 64];
+ optional bool variant_enable_nested_group = 34 [default = false];
}
// Dictionary of Schema info, to reduce TabletSchemaCloudPB fdb kv size
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 6f71a29f72a..eb12834fa18 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -99,6 +99,7 @@ struct TColumn {
26: optional bool variant_enable_doc_mode
27: optional i64 variant_doc_materialization_min_rows
28: optional i32 variant_doc_hash_shard_count
+ 29: optional bool variant_enable_nested_group
}
struct TSlotDescriptor {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org