You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2026/03/12 12:01:29 UTC
(doris) branch master updated: [feature](storage) support change `variant_doc_materialization_min_rows ` and `vertical_compaction_num_columns_per_group ` (#61100)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4eba7f9b0e8 [feature](storage) support change `variant_doc_materialization_min_rows ` and `vertical_compaction_num_columns_per_group ` (#61100)
4eba7f9b0e8 is described below
commit 4eba7f9b0e859680e98b7bf60d9bf2a64b9abcdb
Author: Chenyang Sun <su...@selectdb.com>
AuthorDate: Thu Mar 12 20:01:21 2026 +0800
[feature](storage) support change `variant_doc_materialization_min_rows ` and `vertical_compaction_num_columns_per_group ` (#61100)
### 1. `variant_doc_materialization_min_rows`
- Remove the restriction that prevented modifying this variant column
property via `ALTER TABLE MODIFY COLUMN`.
- This property controls the minimum row threshold for materializing
variant sub-columns. The change is propagated via Light Schema Change.
### 2. `vertical_compaction_num_columns_per_group`
- Allow modifying this table property via `ALTER TABLE SET(...)`. This
controls how many columns are merged in each group during vertical
compaction.
- For wide variant tables, reducing this value significantly reduces
peak
memory usage during compaction.
---
be/src/agent/task_worker_pool.cpp | 5 +
be/src/cloud/cloud_tablet.cpp | 8 ++
be/src/cloud/pb_convert.cpp | 8 ++
be/src/storage/merger.cpp | 34 ++++-
be/src/storage/merger.h | 3 +-
be/src/storage/rowset/segcompaction.cpp | 12 +-
be/src/storage/tablet/tablet_meta.cpp | 18 ++-
be/src/storage/tablet/tablet_meta.h | 11 +-
cloud/src/meta-service/meta_service.cpp | 3 +
.../org/apache/doris/alter/CloudRollupJobV2.java | 3 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../apache/doris/alter/SchemaChangeHandler.java | 26 +++-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Column.java | 3 -
.../main/java/org/apache/doris/catalog/Env.java | 9 ++
.../java/org/apache/doris/catalog/OlapTable.java | 15 ++
.../org/apache/doris/catalog/TableProperty.java | 14 ++
.../cloud/alter/CloudSchemaChangeHandler.java | 19 +++
.../apache/doris/cloud/backup/CloudRestoreJob.java | 3 +-
.../cloud/datasource/CloudInternalCatalog.java | 7 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 27 ++++
.../apache/doris/datasource/InternalCatalog.java | 14 +-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../commands/info/ModifyTablePropertiesOp.java | 17 +++
.../org/apache/doris/task/CreateReplicaTask.java | 7 +-
.../doris/task/UpdateTabletMetaInfoTask.java | 8 +-
.../java/org/apache/doris/task/AgentTaskTest.java | 2 +-
gensrc/proto/cloud.proto | 1 +
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 2 +
.../data/query_p0/system/test_table_properties.out | 8 +-
...variant_modify_doc_materialization_min_rows.out | 22 +++
...ertical_compaction_num_columns_per_group.groovy | 156 +++++++++++++++++++++
...iant_modify_doc_materialization_min_rows.groovy | 98 +++++++++++++
36 files changed, 546 insertions(+), 34 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 52bd24f7d62..415c8b6742a 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -964,6 +964,11 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
tablet_meta_info.time_series_compaction_level_threshold);
need_to_save = true;
}
+ if (tablet_meta_info.__isset.vertical_compaction_num_columns_per_group) {
+ tablet->tablet_meta()->set_vertical_compaction_num_columns_per_group(
+ tablet_meta_info.vertical_compaction_num_columns_per_group);
+ need_to_save = true;
+ }
if (tablet_meta_info.__isset.replica_id) {
tablet->tablet_meta()->set_replica_id(tablet_meta_info.replica_id);
}
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index ee835281acc..eb07def6006 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -1395,6 +1395,14 @@ Status CloudTablet::sync_meta() {
_tablet_meta->mutable_tablet_schema()->set_disable_auto_compaction(
new_disable_auto_compaction);
}
+ // Sync vertical_compaction_num_columns_per_group
+ auto new_vertical_compaction_num_columns_per_group =
+ tablet_meta->vertical_compaction_num_columns_per_group();
+ if (_tablet_meta->vertical_compaction_num_columns_per_group() !=
+ new_vertical_compaction_num_columns_per_group) {
+ _tablet_meta->set_vertical_compaction_num_columns_per_group(
+ new_vertical_compaction_num_columns_per_group);
+ }
return Status::OK();
}
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 03afbdc207c..157a303c2c6 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -648,6 +648,8 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in)
out->set_time_series_compaction_empty_rowsets_threshold(
in.time_series_compaction_empty_rowsets_threshold());
out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold());
+ out->set_vertical_compaction_num_columns_per_group(
+ in.vertical_compaction_num_columns_per_group());
out->set_index_id(in.index_id());
out->set_is_in_memory(in.is_in_memory());
out->set_is_persistent(in.is_persistent());
@@ -725,6 +727,8 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
out->set_time_series_compaction_empty_rowsets_threshold(
in.time_series_compaction_empty_rowsets_threshold());
out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold());
+ out->set_vertical_compaction_num_columns_per_group(
+ in.vertical_compaction_num_columns_per_group());
out->set_index_id(in.index_id());
out->set_is_in_memory(in.is_in_memory());
out->set_is_persistent(in.is_persistent());
@@ -809,6 +813,8 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in)
out->set_time_series_compaction_empty_rowsets_threshold(
in.time_series_compaction_empty_rowsets_threshold());
out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold());
+ out->set_vertical_compaction_num_columns_per_group(
+ in.vertical_compaction_num_columns_per_group());
out->set_index_id(in.index_id());
out->set_is_in_memory(in.is_in_memory());
out->set_is_persistent(in.is_persistent());
@@ -886,6 +892,8 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
out->set_time_series_compaction_empty_rowsets_threshold(
in.time_series_compaction_empty_rowsets_threshold());
out->set_time_series_compaction_level_threshold(in.time_series_compaction_level_threshold());
+ out->set_vertical_compaction_num_columns_per_group(
+ in.vertical_compaction_num_columns_per_group());
out->set_index_id(in.index_id());
out->set_is_in_memory(in.is_in_memory());
out->set_is_persistent(in.is_persistent());
diff --git a/be/src/storage/merger.cpp b/be/src/storage/merger.cpp
index 3fbda7369e7..364ddffe579 100644
--- a/be/src/storage/merger.cpp
+++ b/be/src/storage/merger.cpp
@@ -167,7 +167,8 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type,
// unique_key should consider sequence&delete column
void Merger::vertical_split_columns(const TabletSchema& tablet_schema,
std::vector<std::vector<uint32_t>>* column_groups,
- std::vector<uint32_t>* key_group_cluster_key_idxes) {
+ std::vector<uint32_t>* key_group_cluster_key_idxes,
+ int32_t num_columns_per_group) {
size_t num_key_cols = tablet_schema.num_key_columns();
size_t total_cols = tablet_schema.num_columns();
std::vector<uint32_t> key_columns;
@@ -227,8 +228,7 @@ void Merger::vertical_split_columns(const TabletSchema& tablet_schema,
continue;
}
- if (!value_columns.empty() &&
- value_columns.size() % config::vertical_compaction_num_columns_per_group == 0) {
+ if (!value_columns.empty() && value_columns.size() % num_columns_per_group == 0) {
column_groups->push_back(value_columns);
value_columns.clear();
}
@@ -478,7 +478,33 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t
LOG(INFO) << "Start to do vertical compaction, tablet_id: " << tablet->tablet_id();
std::vector<std::vector<uint32_t>> column_groups;
std::vector<uint32_t> key_group_cluster_key_idxes;
- vertical_split_columns(tablet_schema, &column_groups, &key_group_cluster_key_idxes);
+ // If BE config vertical_compaction_num_columns_per_group has been modified from
+ // its default value (5), use the BE config; otherwise use the tablet meta value.
+ constexpr int32_t default_num_columns_per_group = 5;
+ int32_t num_columns_per_group =
+ config::vertical_compaction_num_columns_per_group != default_num_columns_per_group
+ ? config::vertical_compaction_num_columns_per_group
+ : tablet->tablet_meta()->vertical_compaction_num_columns_per_group();
+
+ DBUG_EXECUTE_IF("Merger.vertical_merge_rowsets.check_num_columns_per_group", {
+ auto expected_value = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group", "expected_value", -1);
+ auto expected_tablet_id = DebugPoints::instance()->get_debug_param_or_default<int64_t>(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group", "tablet_id", -1);
+ if (expected_tablet_id != -1 && expected_tablet_id == tablet->tablet_id()) {
+ if (expected_value != -1 && expected_value != num_columns_per_group) {
+ LOG(FATAL) << "DEBUG_POINT CHECK FAILED: expected num_columns_per_group="
+ << expected_value << " but got " << num_columns_per_group
+ << " for tablet_id=" << tablet->tablet_id();
+ } else {
+ LOG(INFO) << "DEBUG_POINT CHECK PASSED: num_columns_per_group="
+ << num_columns_per_group << ", tablet_id=" << tablet->tablet_id();
+ }
+ }
+ });
+
+ vertical_split_columns(tablet_schema, &column_groups, &key_group_cluster_key_idxes,
+ num_columns_per_group);
// Calculate total rows for density calculation after compaction
int64_t total_rows = 0;
diff --git a/be/src/storage/merger.h b/be/src/storage/merger.h
index 7d346e98b67..62d9e4fed0e 100644
--- a/be/src/storage/merger.h
+++ b/be/src/storage/merger.h
@@ -71,7 +71,8 @@ public:
// for vertical compaction
static void vertical_split_columns(const TabletSchema& tablet_schema,
std::vector<std::vector<uint32_t>>* column_groups,
- std::vector<uint32_t>* key_group_cluster_key_idxes);
+ std::vector<uint32_t>* key_group_cluster_key_idxes,
+ int32_t num_columns_per_group);
static Status vertical_compact_one_group(
BaseTabletSPtr tablet, ReaderType reader_type, const TabletSchema& tablet_schema,
bool is_key, const std::vector<uint32_t>& column_group,
diff --git a/be/src/storage/rowset/segcompaction.cpp b/be/src/storage/rowset/segcompaction.cpp
index df0788b273e..e94fabee96e 100644
--- a/be/src/storage/rowset/segcompaction.cpp
+++ b/be/src/storage/rowset/segcompaction.cpp
@@ -34,6 +34,7 @@
#include "absl/strings/substitute.h"
#include "common/compiler_util.h" // IWYU pragma: keep
+#include "common/config.h"
#include "common/logging.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
@@ -281,8 +282,15 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
std::vector<std::vector<uint32_t>> column_groups;
std::vector<uint32_t> key_group_cluster_key_idxes;
- Merger::vertical_split_columns(*ctx.tablet_schema, &column_groups,
- &key_group_cluster_key_idxes);
+ // If BE config vertical_compaction_num_columns_per_group has been modified from
+ // its default value (5), use the BE config; otherwise use the tablet meta value.
+ constexpr int32_t default_num_columns_per_group = 5;
+ int32_t num_columns_per_group =
+ config::vertical_compaction_num_columns_per_group != default_num_columns_per_group
+ ? config::vertical_compaction_num_columns_per_group
+ : tablet->tablet_meta()->vertical_compaction_num_columns_per_group();
+ Merger::vertical_split_columns(*ctx.tablet_schema, &column_groups, &key_group_cluster_key_idxes,
+ num_columns_per_group);
RowSourcesBuffer row_sources_buf(tablet->tablet_id(), tablet->tablet_path(),
ReaderType::READER_SEGMENT_COMPACTION);
diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp
index 4c032605dbe..b2b6d803d79 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -125,7 +125,10 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
request.time_series_compaction_level_threshold, inverted_index_file_storage_format,
- request.tde_algorithm, storage_format);
+ request.tde_algorithm, storage_format,
+ request.__isset.vertical_compaction_num_columns_per_group
+ ? request.vertical_compaction_num_columns_per_group
+ : 5);
}
TabletMeta::~TabletMeta() {
@@ -154,7 +157,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
int64_t time_series_compaction_level_threshold,
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format,
TEncryptionAlgorithm::type tde_algorithm,
- TStorageFormat::type storage_format)
+ TStorageFormat::type storage_format,
+ int32_t vertical_compaction_num_columns_per_group)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
_delete_bitmap(new DeleteBitmap(tablet_id)),
@@ -187,6 +191,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
time_series_compaction_empty_rowsets_threshold);
tablet_meta_pb.set_time_series_compaction_level_threshold(
time_series_compaction_level_threshold);
+ tablet_meta_pb.set_vertical_compaction_num_columns_per_group(
+ vertical_compaction_num_columns_per_group);
TabletSchemaPB* schema = tablet_meta_pb.mutable_schema();
schema->set_num_short_key_columns(tablet_schema.short_key_column_count);
schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block);
@@ -464,7 +470,9 @@ TabletMeta::TabletMeta(const TabletMeta& b)
b._time_series_compaction_time_threshold_seconds),
_time_series_compaction_empty_rowsets_threshold(
b._time_series_compaction_empty_rowsets_threshold),
- _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold) {};
+ _time_series_compaction_level_threshold(b._time_series_compaction_level_threshold),
+ _vertical_compaction_num_columns_per_group(
+ b._vertical_compaction_num_columns_per_group) {};
void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
ColumnPB* column) {
@@ -857,6 +865,8 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
tablet_meta_pb.time_series_compaction_empty_rowsets_threshold();
_time_series_compaction_level_threshold =
tablet_meta_pb.time_series_compaction_level_threshold();
+ _vertical_compaction_num_columns_per_group =
+ tablet_meta_pb.vertical_compaction_num_columns_per_group();
if (tablet_meta_pb.has_encryption_algorithm()) {
_encryption_algorithm = tablet_meta_pb.encryption_algorithm();
@@ -952,6 +962,8 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_
time_series_compaction_empty_rowsets_threshold());
tablet_meta_pb->set_time_series_compaction_level_threshold(
time_series_compaction_level_threshold());
+ tablet_meta_pb->set_vertical_compaction_num_columns_per_group(
+ vertical_compaction_num_columns_per_group());
tablet_meta_pb->set_encryption_algorithm(_encryption_algorithm);
}
diff --git a/be/src/storage/tablet/tablet_meta.h b/be/src/storage/tablet/tablet_meta.h
index 1cdcba3783d..046f0ca6f8a 100644
--- a/be/src/storage/tablet/tablet_meta.h
+++ b/be/src/storage/tablet/tablet_meta.h
@@ -118,7 +118,8 @@ public:
TInvertedIndexFileStorageFormat::type inverted_index_file_storage_format =
TInvertedIndexFileStorageFormat::V2,
TEncryptionAlgorithm::type tde_algorithm = TEncryptionAlgorithm::PLAINTEXT,
- TStorageFormat::type storage_format = TStorageFormat::V2);
+ TStorageFormat::type storage_format = TStorageFormat::V2,
+ int32_t vertical_compaction_num_columns_per_group = 5);
// If need add a filed in TableMeta, filed init copy in copy construct function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -296,6 +297,13 @@ public:
return _time_series_compaction_level_threshold;
}
+ void set_vertical_compaction_num_columns_per_group(int32_t num) {
+ _vertical_compaction_num_columns_per_group = num;
+ }
+ int32_t vertical_compaction_num_columns_per_group() const {
+ return _vertical_compaction_num_columns_per_group;
+ }
+
int64_t ttl_seconds() const {
std::shared_lock rlock(_meta_lock);
return _ttl_seconds;
@@ -366,6 +374,7 @@ private:
int64_t _time_series_compaction_time_threshold_seconds = 0;
int64_t _time_series_compaction_empty_rowsets_threshold = 0;
int64_t _time_series_compaction_level_threshold = 0;
+ int32_t _vertical_compaction_num_columns_per_group = 5;
int64_t _avg_rs_meta_serialize_size = 0;
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 90ebd4076e1..23927ba99b3 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1245,6 +1245,9 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle
}
}
}
+ } else if (tablet_meta_info.has_vertical_compaction_num_columns_per_group()) {
+ tablet_meta.set_vertical_compaction_num_columns_per_group(
+ tablet_meta_info.vertical_compaction_num_columns_per_group());
}
int64_t table_id = tablet_meta.table_id();
int64_t index_id = tablet_meta.index_id();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index dcf57777f4f..8c6a306aedb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -235,7 +235,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
tbl.variantEnableFlattenNested(), null,
tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true,
- tbl.getColumnSeqMapping());
+ tbl.getColumnSeqMapping(),
+ tbl.getVerticalCompactionNumColumnsPerGroup());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 3fdc176fc5f..9387a93aaa9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -258,7 +258,8 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
tbl.variantEnableFlattenNested(), clusterKeyUids,
tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true,
- columnSeqMapping);
+ columnSeqMapping,
+ tbl.getVerticalCompactionNumColumnsPerGroup());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 66bfaedc8a7..27a1fdcc8df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -272,7 +272,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
tbl.storagePageSize(), tbl.getTDEAlgorithm(),
- tbl.storageDictPageSize(), null);
+ tbl.storageDictPageSize(), null,
+ tbl.getVerticalCompactionNumColumnsPerGroup());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 1c5c6529ba5..113f32d7ae8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -825,6 +825,11 @@ public class SchemaChangeHandler extends AlterHandler {
ColumnType.checkSupportSchemaChangeForComplexType(col.getType(), modColumn.getType(), true);
lightSchemaChange = olapTable.getEnableLightSchemaChange();
}
+ // variant property-only change (e.g. variant_doc_materialization_min_rows)
+ if (columnPos == null && col.getDataType() == PrimitiveType.VARIANT
+ && modColumn.getDataType() == PrimitiveType.VARIANT) {
+ lightSchemaChange = olapTable.getEnableLightSchemaChange();
+ }
if (col.isClusterKey()) {
throw new DdlException("Can not modify cluster key column: " + col.getName());
}
@@ -2572,6 +2577,7 @@ public class SchemaChangeHandler extends AlterHandler {
add(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
add(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD);
add(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_LEVEL_THRESHOLD);
+ add(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP);
add(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY);
add(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM);
add(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT);
@@ -2663,7 +2669,14 @@ public class SchemaChangeHandler extends AlterHandler {
}
+ int verticalCompactionNumColumnsPerGroup = -1; // < 0 means don't update
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP)) {
+ verticalCompactionNumColumnsPerGroup = Integer.parseInt(
+ properties.get(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP));
+ }
+
if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null && timeSeriesCompactionConfig.isEmpty()
+ && verticalCompactionNumColumnsPerGroup < 0
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE)
&& !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
@@ -2711,7 +2724,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(), partition.getName(), storagePolicyId, isInMemory,
null, compactionPolicy, timeSeriesCompactionConfig, enableSingleCompaction, skip,
- disableAutoCompaction);
+ disableAutoCompaction, verticalCompactionNumColumnsPerGroup);
}
olapTable.writeLockOrDdlException();
@@ -2759,8 +2772,8 @@ public class SchemaChangeHandler extends AlterHandler {
for (String partitionName : partitionNames) {
try {
- updatePartitionProperties(db, olapTable.getName(), partitionName, storagePolicyId,
- isInMemory, null, null, null, -1, -1, -1);
+ updatePartitionProperties(db, olapTable.getName(), partitionName,
+ storagePolicyId, isInMemory, null, null, null, -1, -1, -1, -1);
} catch (Exception e) {
String errMsg = "Failed to update partition[" + partitionName + "]'s 'in_memory' property. "
+ "The reason is [" + e.getMessage() + "]";
@@ -2777,7 +2790,8 @@ public class SchemaChangeHandler extends AlterHandler {
int isInMemory, BinlogConfig binlogConfig, String compactionPolicy,
Map<String, Long> timeSeriesCompactionConfig,
int enableSingleCompaction, int skipWriteIndexOnLoad,
- int disableAutoCompaction) throws UserException {
+ int disableAutoCompaction,
+ int verticalCompactionNumColumnsPerGroup) throws UserException {
// be id -> <tablet id,schemaHash>
Map<Long, Set<Pair<Long, Integer>>> beIdToTabletIdWithHash = Maps.newHashMap();
OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP);
@@ -2811,7 +2825,7 @@ public class SchemaChangeHandler extends AlterHandler {
UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(kv.getKey(), kv.getValue(), isInMemory,
storagePolicyId, binlogConfig, countDownLatch, compactionPolicy,
timeSeriesCompactionConfig, enableSingleCompaction, skipWriteIndexOnLoad,
- disableAutoCompaction);
+ disableAutoCompaction, verticalCompactionNumColumnsPerGroup);
batchTask.addTask(task);
}
if (!FeConstants.runningUnitTest) {
@@ -3694,7 +3708,7 @@ public class SchemaChangeHandler extends AlterHandler {
for (Partition partition : partitions) {
updatePartitionProperties(db, olapTable.getName(), partition.getName(), -1, -1,
- newBinlogConfig, null, null, -1, -1, -1);
+ newBinlogConfig, null, null, -1, -1, -1, -1);
}
olapTable.writeLockOrDdlException();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 00cfc7dc8a1..955af7162fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -335,7 +335,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 implements GsonPostProcessable
tbl.variantEnableFlattenNested(),
tbl.storagePageSize(), tbl.getTDEAlgorithm(),
tbl.storageDictPageSize(),
- columnSeqMapping);
+ columnSeqMapping,
+ tbl.getVerticalCompactionNumColumnsPerGroup());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId, shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 2874dc900c8..60142fa694f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1454,7 +1454,8 @@ public class RestoreJob extends AbstractJob implements GsonPostProcessable {
localTbl.variantEnableFlattenNested(),
localTbl.storagePageSize(), localTbl.getTDEAlgorithm(),
localTbl.storageDictPageSize(),
- localTbl.getColumnSeqMapping());
+ localTbl.getColumnSeqMapping(),
+ localTbl.getVerticalCompactionNumColumnsPerGroup());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
if (baseTabletRef != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index e06d0c93d28..d3aa5aa5917 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -977,9 +977,6 @@ public class Column implements GsonPostProcessable {
if (this.getVariantEnableDocMode() != other.getVariantEnableDocMode()) {
throw new DdlException("Can not change variant enable doc snapshot mode");
}
- if (this.getvariantDocMaterializationMinRows() != other.getvariantDocMaterializationMinRows()) {
- throw new DdlException("Can not change variant doc snapshot min rows");
- }
if (this.getVariantDocShardCount() != other.getVariantDocShardCount()) {
throw new DdlException("Can not change variant doc snapshot shard count");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 37e0b7886b4..77c1187effe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3986,6 +3986,14 @@ public class Env {
sb.append(olapTable.getTimeSeriesCompactionLevelThreshold()).append("\"");
}
+ // vertical compaction num columns per group
+ if (olapTable.getVerticalCompactionNumColumnsPerGroup()
+ != PropertyAnalyzer.VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE) {
+ sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP)
+ .append("\" = \"");
+ sb.append(olapTable.getVerticalCompactionNumColumnsPerGroup()).append("\"");
+ }
+
// Storage Vault
if (!Strings.isNullOrEmpty(olapTable.getStorageVaultId())) {
sb.append(",\n\"").append(PropertyAnalyzer
@@ -6222,6 +6230,7 @@ public class Env {
.buildEnableSingleReplicaCompaction()
.buildTimeSeriesCompactionEmptyRowsetsThreshold()
.buildTimeSeriesCompactionLevelThreshold()
+ .buildVerticalCompactionNumColumnsPerGroup()
.buildTTLSeconds()
.buildAutoAnalyzeProperty()
.buildPartitionRetentionCount();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ca3b5cc766c..56d24644b81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2735,6 +2735,21 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
return PropertyAnalyzer.TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE;
}
+ public void setVerticalCompactionNumColumnsPerGroup(int verticalCompactionNumColumnsPerGroup) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+ tableProperty.modifyTableProperties(
+ PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP,
+ Integer.valueOf(verticalCompactionNumColumnsPerGroup).toString());
+ tableProperty.buildVerticalCompactionNumColumnsPerGroup();
+ }
+
+ public int getVerticalCompactionNumColumnsPerGroup() {
+ if (tableProperty != null) {
+ return tableProperty.verticalCompactionNumColumnsPerGroup();
+ }
+ return PropertyAnalyzer.VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE;
+ }
+
public int getIndexSchemaVersion(long indexId) {
MaterializedIndexMeta indexMeta = indexIdToMeta.get(indexId);
return indexMeta.getSchemaVersion();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index ea770b20a3b..f8544de47cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -98,6 +98,8 @@ public class TableProperty implements GsonPostProcessable {
private boolean enableSingleReplicaCompaction = false;
+ private int verticalCompactionNumColumnsPerGroup = 5;
+
private boolean storeRowColumn = false;
private boolean skipWriteIndexOnLoad = false;
@@ -167,6 +169,7 @@ public class TableProperty implements GsonPostProcessable {
buildTimeSeriesCompactionTimeThresholdSeconds();
buildSkipWriteIndexOnLoad();
buildEnableSingleReplicaCompaction();
+ buildVerticalCompactionNumColumnsPerGroup();
buildDisableAutoCompaction();
buildTimeSeriesCompactionEmptyRowsetsThreshold();
buildTimeSeriesCompactionLevelThreshold();
@@ -343,6 +346,16 @@ public class TableProperty implements GsonPostProcessable {
return enableSingleReplicaCompaction;
}
+ public TableProperty buildVerticalCompactionNumColumnsPerGroup() {
+ verticalCompactionNumColumnsPerGroup = Integer.parseInt(
+ properties.getOrDefault(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP, "5"));
+ return this;
+ }
+
+ public int verticalCompactionNumColumnsPerGroup() {
+ return verticalCompactionNumColumnsPerGroup;
+ }
+
public TableProperty buildStoreRowColumn() {
storeRowColumn = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, "false"));
@@ -885,6 +898,7 @@ public class TableProperty implements GsonPostProcessable {
buildTimeSeriesCompactionTimeThresholdSeconds();
buildDisableAutoCompaction();
buildEnableSingleReplicaCompaction();
+ buildVerticalCompactionNumColumnsPerGroup();
buildTimeSeriesCompactionEmptyRowsetsThreshold();
buildTimeSeriesCompactionLevelThreshold();
buildTTLSeconds();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
index d199fcbb10e..caf5f93f772 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java
@@ -113,6 +113,7 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler {
add(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE);
add(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY);
add(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT);
+ add(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP);
}
};
List<String> notAllowedProps = properties.keySet().stream().filter(s -> !allowedProps.contains(s))
@@ -346,6 +347,18 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler {
param.type = UpdatePartitionMetaParam.TabletMetaType.ENABLE_MOW_LIGHT_DELETE;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY)) {
// Do nothing.
+ } else if (properties.containsKey(
+ PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP)) {
+ int verticalCompactionNumColumnsPerGroup = Integer.parseInt(properties.get(
+ PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP));
+ olapTable.readLock();
+ try {
+ partitions.addAll(olapTable.getPartitions());
+ } finally {
+ olapTable.readUnlock();
+ }
+ param.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup;
+ param.type = UpdatePartitionMetaParam.TabletMetaType.VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP;
} else {
LOG.warn("invalid properties:{}", properties);
throw new UserException("invalid properties");
@@ -381,6 +394,7 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler {
TIME_SERIES_COMPACTION_LEVEL_THRESHOLD,
DISABLE_AUTO_COMPACTION,
ENABLE_MOW_LIGHT_DELETE,
+ VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP,
}
TabletMetaType type;
@@ -397,6 +411,7 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler {
long timeSeriesCompactionLevelThreshold = 0;
boolean disableAutoCompaction = false;
boolean enableMowLightDelete = false;
+ int verticalCompactionNumColumnsPerGroup = 5;
}
public void updateCloudPartitionMeta(Database db,
@@ -477,6 +492,10 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler {
param.enableMowLightDelete
);
break;
+ case VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP:
+ infoBuilder.setVerticalCompactionNumColumnsPerGroup(
+ param.verticalCompactionNumColumnsPerGroup);
+ break;
default:
throw new UserException("Unknown TabletMetaType");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index f518d6cc3a4..980e75ee2b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -404,7 +404,8 @@ public class CloudRestoreJob extends RestoreJob {
localTbl.variantEnableFlattenNested(), clusterKeyUids,
localTbl.storagePageSize(), localTbl.getTDEAlgorithmPB(),
localTbl.storageDictPageSize(), false,
- localTbl.getColumnSeqMapping()));
+ localTbl.getColumnSeqMapping(),
+ localTbl.getVerticalCompactionNumColumnsPerGroup()));
// In cloud mode all storage medium will be saved to HDD.
TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 9d47c68b844..173b0a1d57b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -192,7 +192,8 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.variantEnableFlattenNested(), clusterKeyUids,
tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true,
- tbl.getColumnSeqMapping());
+ tbl.getColumnSeqMapping(),
+ tbl.getVerticalCompactionNumColumnsPerGroup());
requestBuilder.addTabletMetas(builder);
}
requestBuilder.setDbId(dbId);
@@ -226,7 +227,8 @@ public class CloudInternalCatalog extends InternalCatalog {
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize,
boolean variantEnableFlattenNested, List<Integer> clusterKeyUids,
long storagePageSize, EncryptionAlgorithmPB encryptionAlgorithm, long storageDictPageSize,
- boolean createInitialRowset, Map<String, List<String>> columnSeqMapping) throws DdlException {
+ boolean createInitialRowset, Map<String, List<String>> columnSeqMapping,
+ int verticalCompactionNumColumnsPerGroup) throws DdlException {
OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -261,6 +263,7 @@ public class CloudInternalCatalog extends InternalCatalog {
builder.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
builder.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
builder.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ builder.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup);
OlapFile.TabletSchemaCloudPB.Builder schemaBuilder = OlapFile.TabletSchemaCloudPB.newBuilder();
schemaBuilder.setSchemaVersion(schemaVersion);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 02347cd38e7..93dc9756f2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -161,6 +161,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION = "enable_single_replica_compaction";
+ public static final String PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP =
+ "vertical_compaction_num_columns_per_group";
+
public static final String PROPERTIES_STORE_ROW_COLUMN = "store_row_column"; // deprecated
public static final String PROPERTIES_ROW_STORE_COLUMNS = "row_store_columns";
@@ -255,6 +258,7 @@ public class PropertyAnalyzer {
public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600;
public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5;
public static final long TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE = 1;
+ public static final int VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE = 5;
public static final String PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT = "variant_max_subcolumns_count";
@@ -2250,4 +2254,27 @@ public class PropertyAnalyzer {
}
throw new AnalysisException("Invalid tde algorithm: " + name + ", only support AES256 and SM4 currently");
}
+
+ public static Integer analyzeVerticalCompactionNumColumnsPerGroup(Map<String, String> properties)
+ throws AnalysisException {
+ if (properties == null || properties.isEmpty()) {
+ return VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE;
+ }
+ String value = properties.get(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP);
+ if (null == value) {
+ return VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE;
+ }
+ properties.remove(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP);
+ try {
+ int num = Integer.parseInt(value);
+ if (num < 1 || num > 50) {
+ throw new AnalysisException(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP
+ + " must be between 1 and 50");
+ }
+ return num;
+ } catch (NumberFormatException e) {
+ throw new AnalysisException(PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP
+ + " must be a valid integer");
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 61d2d2e2294..6a7a2950beb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2130,7 +2130,8 @@ public class InternalCatalog implements CatalogIf<Database> {
tbl.variantEnableFlattenNested(),
tbl.storagePageSize(), tbl.getTDEAlgorithm(),
tbl.storageDictPageSize(),
- tbl.getColumnSeqMapping());
+ tbl.getColumnSeqMapping(),
+ tbl.getVerticalCompactionNumColumnsPerGroup());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat());
@@ -2480,6 +2481,17 @@ public class InternalCatalog implements CatalogIf<Database> {
}
olapTable.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ // set vertical compaction num columns per group
+ int verticalCompactionNumColumnsPerGroup
+ = PropertyAnalyzer.VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP_DEFAULT_VALUE;
+ try {
+ verticalCompactionNumColumnsPerGroup = PropertyAnalyzer
+ .analyzeVerticalCompactionNumColumnsPerGroup(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+ olapTable.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup);
+
boolean variantEnableFlattenNested = false;
try {
variantEnableFlattenNested = PropertyAnalyzer.analyzeVariantFlattenNested(properties);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 93d0d10b055..cf7df808eda 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -1089,7 +1089,8 @@ public class ReportHandler extends Daemon {
olapTable.variantEnableFlattenNested(),
olapTable.storagePageSize(), olapTable.getTDEAlgorithm(),
olapTable.storageDictPageSize(),
- olapTable.getColumnSeqMapping());
+ olapTable.getColumnSeqMapping(),
+ olapTable.getVerticalCompactionNumColumnsPerGroup());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
.getInvertedIndexFileStorageFormat());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index 7dfc89e96e4..e2189ed0696 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -247,6 +247,23 @@ public class ModifyTablePropertiesOp extends AlterTableOp {
}
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
+ } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP)) {
+ int numColumnsPerGroup;
+ String numColumnsPerGroupStr = properties
+ .get(PropertyAnalyzer.PROPERTIES_VERTICAL_COMPACTION_NUM_COLUMNS_PER_GROUP);
+ try {
+ numColumnsPerGroup = Integer.parseInt(numColumnsPerGroupStr);
+ if (numColumnsPerGroup < 1 || numColumnsPerGroup > 50) {
+ throw new AnalysisException(
+ "vertical_compaction_num_columns_per_group must be between 1 and 50: "
+ + numColumnsPerGroupStr);
+ }
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid vertical_compaction_num_columns_per_group format: "
+ + numColumnsPerGroupStr);
+ }
+ this.needTableStable = false;
+ this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) {
throw new AnalysisException(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 0795d66e559..1d03b7bfe9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -125,6 +125,8 @@ public class CreateReplicaTask extends AgentTask {
private long timeSeriesCompactionLevelThreshold;
+ private int verticalCompactionNumColumnsPerGroup;
+
private boolean storeRowColumn;
private BinlogConfig binlogConfig;
@@ -165,7 +167,8 @@ public class CreateReplicaTask extends AgentTask {
long rowStorePageSize,
boolean variantEnableFlattenNested,
long storagePageSize, TEncryptionAlgorithm tdeAlgorithm,
- long storageDictPageSize, Map<String, List<String>> columnSeqMapping) {
+ long storageDictPageSize, Map<String, List<String>> columnSeqMapping,
+ int verticalCompactionNumColumnsPerGroup) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId);
this.replicaId = replicaId;
@@ -208,6 +211,7 @@ public class CreateReplicaTask extends AgentTask {
this.timeSeriesCompactionTimeThresholdSeconds = timeSeriesCompactionTimeThresholdSeconds;
this.timeSeriesCompactionEmptyRowsetsThreshold = timeSeriesCompactionEmptyRowsetsThreshold;
this.timeSeriesCompactionLevelThreshold = timeSeriesCompactionLevelThreshold;
+ this.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup;
this.storeRowColumn = storeRowColumn;
this.binlogConfig = binlogConfig;
this.objectPool = objectPool;
@@ -443,6 +447,7 @@ public class CreateReplicaTask extends AgentTask {
createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
createTabletReq.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ createTabletReq.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup);
createTabletReq.setTdeAlgorithm(tdeAlgorithm);
if (binlogConfig != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
index 7d4c6a3d022..4ba127f4472 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
@@ -54,6 +54,7 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
private int enableSingleReplicaCompaction = -1;
private int skipWriteIndexOnLoad = -1;
private int disableAutoCompaction = -1;
+ private int verticalCompactionNumColumnsPerGroup = -1;
public UpdateTabletMetaInfoTask(long backendId, Set<Pair<Long, Integer>> tableIdWithSchemaHash) {
super(null, backendId, TTaskType.UPDATE_TABLET_META_INFO,
@@ -89,13 +90,15 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
Map<String, Long> timeSeriesCompactionConfig,
int enableSingleReplicaCompaction,
int skipWriteIndexOnLoad,
- int disableAutoCompaction) {
+ int disableAutoCompaction,
+ int verticalCompactionNumColumnsPerGroup) {
this(backendId, tableIdWithSchemaHash, inMemory, storagePolicyId, binlogConfig, latch);
this.compactionPolicy = compactionPolicy;
this.timeSeriesCompactionConfig = timeSeriesCompactionConfig;
this.enableSingleReplicaCompaction = enableSingleReplicaCompaction;
this.skipWriteIndexOnLoad = skipWriteIndexOnLoad;
this.disableAutoCompaction = disableAutoCompaction;
+ this.verticalCompactionNumColumnsPerGroup = verticalCompactionNumColumnsPerGroup;
}
public void countDownLatch(long backendId, Set<Pair<Long, Integer>> tablets) {
@@ -179,6 +182,9 @@ public class UpdateTabletMetaInfoTask extends AgentTask {
if (disableAutoCompaction >= 0) {
metaInfo.setDisableAutoCompaction(disableAutoCompaction > 0);
}
+ if (verticalCompactionNumColumnsPerGroup >= 0) {
+ metaInfo.setVerticalCompactionNumColumnsPerGroup(verticalCompactionNumColumnsPerGroup);
+ }
updateTabletMetaInfoReq.addToTabletMetaInfos(metaInfo);
}
} else {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 799e665039b..a1cef6161bd 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -116,7 +116,7 @@ public class AgentTaskTest {
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null,
TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
- storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>());
+ storagePageSize, TEncryptionAlgorithm.PLAINTEXT, storageDictPageSize, new HashMap<>(), 5);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false);
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index c779506a012..60596a58828 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -663,6 +663,7 @@ message TabletMetaInfoPB { // For update tablet meta
optional int64 time_series_compaction_level_threshold = 12;
optional bool disable_auto_compaction = 13;
optional bool enable_mow_light_delete = 14;
+ optional int32 vertical_compaction_num_columns_per_group = 15;
}
message TabletCompactionJobPB {
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index aadacd37879..0d4359fe162 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -631,6 +631,7 @@ message TabletMetaPB {
optional int64 time_series_compaction_empty_rowsets_threshold = 32 [default = 5];
optional int64 time_series_compaction_level_threshold = 33 [default = 1];
optional EncryptionAlgorithmPB encryption_algorithm = 34;
+ optional int32 vertical_compaction_num_columns_per_group = 35 [default = 5];
// For cloud
optional int64 index_id = 1000;
@@ -688,6 +689,7 @@ message TabletMetaCloudPB {
optional int64 time_series_compaction_empty_rowsets_threshold = 37 [default = 5];
optional int64 time_series_compaction_level_threshold = 38 [default = 1];
optional EncryptionAlgorithmPB encryption_algorithm = 39;
+ optional int32 vertical_compaction_num_columns_per_group = 40 [default = 5];
// Use for selectdb-cloud
optional string table_name = 101;
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index e8044a06c9a..51bd59e7ae7 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -229,6 +229,7 @@ struct TCreateTabletReq {
28: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.DEFAULT // Deprecated
29: optional Types.TInvertedIndexFileStorageFormat inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2
30: optional TEncryptionAlgorithm tde_algorithm
+ 31: optional i32 vertical_compaction_num_columns_per_group = 5
// For cloud
1000: optional bool is_in_memory = false
@@ -543,6 +544,7 @@ struct TTabletMetaInfo {
16: optional bool disable_auto_compaction
17: optional i64 time_series_compaction_empty_rowsets_threshold
18: optional i64 time_series_compaction_level_threshold
+ 19: optional i32 vertical_compaction_num_columns_per_group
}
struct TUpdateTabletMetaInfoReq {
diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out
index a6ff1c341f8..0f8b1a2acaf 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_check_1 --
-108
+111
-- !select_check_2 --
internal test_table_properties_db duplicate_table _auto_bucket false
@@ -38,6 +38,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db duplicate_table variant_enable_flatten_nested false
+internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5
internal test_table_properties_db listtable _auto_bucket false
internal test_table_properties_db listtable binlog.enable false
internal test_table_properties_db listtable binlog.max_bytes 9223372036854775807
@@ -73,6 +74,7 @@ internal test_table_properties_db listtable time_series_compaction_goal_size_mby
internal test_table_properties_db listtable time_series_compaction_level_threshold 1
internal test_table_properties_db listtable time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db listtable variant_enable_flatten_nested false
+internal test_table_properties_db listtable vertical_compaction_num_columns_per_group 5
internal test_table_properties_db unique_table _auto_bucket false
internal test_table_properties_db unique_table binlog.enable false
internal test_table_properties_db unique_table binlog.max_bytes 9223372036854775807
@@ -108,6 +110,7 @@ internal test_table_properties_db unique_table time_series_compaction_goal_size_
internal test_table_properties_db unique_table time_series_compaction_level_threshold 1
internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db unique_table variant_enable_flatten_nested false
+internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5
-- !select_check_3 --
internal test_table_properties_db duplicate_table _auto_bucket false
@@ -145,6 +148,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db duplicate_table variant_enable_flatten_nested false
+internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5
internal test_table_properties_db unique_table _auto_bucket false
internal test_table_properties_db unique_table binlog.enable false
internal test_table_properties_db unique_table binlog.max_bytes 9223372036854775807
@@ -180,6 +184,7 @@ internal test_table_properties_db unique_table time_series_compaction_goal_size_
internal test_table_properties_db unique_table time_series_compaction_level_threshold 1
internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db unique_table variant_enable_flatten_nested false
+internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5
-- !select_check_4 --
@@ -219,6 +224,7 @@ internal test_table_properties_db duplicate_table time_series_compaction_goal_si
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
internal test_table_properties_db duplicate_table variant_enable_flatten_nested false
+internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5
-- !select_check_6 --
diff --git a/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out b/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out
new file mode 100644
index 00000000000..56a06bcf8fa
--- /dev/null
+++ b/regression-test/data/variant_p0/test_variant_modify_doc_materialization_min_rows.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !desc_step1 --
+k int Yes true \N
+v variant<PROPERTIES ("variant_enable_doc_mode" = "true","variant_doc_materialization_min_rows" = "10","variant_doc_hash_shard_count" = "32")> Yes false \N NONE
+
+-- !desc_step2 --
+k int Yes true \N
+v variant<PROPERTIES ("variant_enable_doc_mode" = "true","variant_doc_materialization_min_rows" = "2","variant_doc_hash_shard_count" = "32")> Yes false \N NONE
+v.path_d bigint Yes false \N NONE
+v.path_e text Yes false \N NONE
+v.path_f double Yes false \N NONE
+
+-- !desc_step3 --
+k int Yes true \N
+v variant<PROPERTIES ("variant_enable_doc_mode" = "true","variant_doc_materialization_min_rows" = "2","variant_doc_hash_shard_count" = "32")> Yes false \N NONE
+v.path_a bigint Yes false \N NONE
+v.path_b text Yes false \N NONE
+v.path_c double Yes false \N NONE
+v.path_d bigint Yes false \N NONE
+v.path_e text Yes false \N NONE
+v.path_f double Yes false \N NONE
+
diff --git a/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy b/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy
new file mode 100644
index 00000000000..129cd6d0309
--- /dev/null
+++ b/regression-test/suites/compaction/test_vertical_compaction_num_columns_per_group.groovy
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_vertical_compaction_num_columns_per_group", "nonConcurrent") {
+ def tableName = "test_columns_per_group"
+
+ // Test 1: Create table with property set to 2
+ sql """ DROP TABLE IF EXISTS ${tableName}_2 """
+ sql """ CREATE TABLE ${tableName}_2 (
+ k1 INT,
+ v1 INT, v2 INT, v3 INT, v4 INT, v5 INT,
+ v6 INT, v7 INT, v8 INT, v9 INT, v10 INT
+ ) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "vertical_compaction_num_columns_per_group" = "2",
+ "disable_auto_compaction" = "true"
+ ); """
+
+ // Get tablet info
+ def tablets = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_2; """
+ def tablet_id = tablets[0].TabletId
+ logger.info("Test 1 - tablet_id: ${tablet_id}")
+
+ // Verify SHOW CREATE TABLE contains property
+ def createTableResult = sql """ SHOW CREATE TABLE ${tableName}_2; """
+ def createTableStr = createTableResult[0][1]
+ logger.info("Test 1 - SHOW CREATE TABLE: ${createTableStr}")
+ assertTrue(createTableStr.contains('"vertical_compaction_num_columns_per_group" = "2"'),
+ "SHOW CREATE TABLE should contain vertical_compaction_num_columns_per_group=2 after CREATE")
+
+ // Enable debug point with tablet_id matching - if value doesn't match, BE will LOG(FATAL)
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group",
+ [expected_value: "2", tablet_id: "${tablet_id}"])
+
+ // Insert data to trigger compaction
+ for (int i = 0; i < 5; i++) {
+ sql """ INSERT INTO ${tableName}_2 VALUES
+ (${i}, ${i}, ${i}, ${i}, ${i}, ${i},
+ ${i}, ${i}, ${i}, ${i}, ${i}); """
+ }
+
+ // Trigger and wait for compaction
+ trigger_and_wait_compaction("${tableName}_2", "full")
+ logger.info("Test 1 - Compaction finished, value 2 verified for tablet ${tablet_id}")
+
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group")
+ }
+
+ // Test 2: Create table without setting property (should use default value 5)
+ sql """ DROP TABLE IF EXISTS ${tableName}_default """
+ sql """ CREATE TABLE ${tableName}_default (
+ k1 INT,
+ v1 INT, v2 INT, v3 INT, v4 INT, v5 INT
+ ) DUPLICATE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true"
+ ); """
+
+ def tablets_default = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_default; """
+ def tablet_id_default = tablets_default[0].TabletId
+ logger.info("Test 2 - tablet_id_default: ${tablet_id_default}")
+
+ try {
+ // Verify default value is 5
+ GetDebugPoint().enableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group",
+ [expected_value: "5", tablet_id: "${tablet_id_default}"])
+
+ // Insert data
+ for (int i = 0; i < 5; i++) {
+ sql """ INSERT INTO ${tableName}_default VALUES (${i}, ${i}, ${i}, ${i}, ${i}, ${i}); """
+ }
+
+ // Trigger and wait for compaction
+ trigger_and_wait_compaction("${tableName}_default", "full")
+ logger.info("Test 2 - Compaction finished, default value 5 verified for tablet ${tablet_id_default}")
+
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group")
+ }
+
+ // Test 3: ALTER TABLE to modify property from 2 to 10
+ sql """ ALTER TABLE ${tableName}_2 SET ("vertical_compaction_num_columns_per_group" = "10"); """
+ logger.info("Test 3 - ALTER TABLE executed, changed value from 2 to 10")
+ sql """sync"""
+
+ Thread.sleep(1000)
+ // Verify SHOW CREATE TABLE reflects the change
+ def createTableResult3 = sql """ SHOW CREATE TABLE ${tableName}_2; """
+ def createTableStr3 = createTableResult3[0][1]
+ logger.info("Test 3 - SHOW CREATE TABLE after ALTER: ${createTableStr3}")
+ assertTrue(createTableStr3.contains('"vertical_compaction_num_columns_per_group" = "10"'),
+ "SHOW CREATE TABLE should contain vertical_compaction_num_columns_per_group=10 after ALTER")
+
+
+ // Wait for ALTER TABLE to take effect
+ // In cloud mode, BE syncs from MS which may take longer
+ if (isCloudMode()) {
+ return
+ }
+ try {
+ // Verify modified value is 10
+ GetDebugPoint().enableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group",
+ [expected_value: "10", tablet_id: "${tablet_id}"])
+
+ // Insert more data
+ for (int i = 5; i < 10; i++) {
+ sql """ INSERT INTO ${tableName}_2 VALUES
+ (${i}, ${i}, ${i}, ${i}, ${i}, ${i},
+ ${i}, ${i}, ${i}, ${i}, ${i}); """
+ }
+
+ // Trigger and wait for compaction
+ trigger_and_wait_compaction("${tableName}_2", "full")
+ logger.info("Test 3 - Compaction finished, altered value 10 verified for tablet ${tablet_id}")
+
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+ "Merger.vertical_merge_rowsets.check_num_columns_per_group")
+ }
+
+ // Verify data correctness
+ def result = sql """ SELECT COUNT(*) FROM ${tableName}_2; """
+ assertEquals(10, result[0][0])
+
+ // Check tablet meta shows correct value
+ def tabletMeta = sql_return_maparray """ SHOW TABLETS FROM ${tableName}_2; """
+ logger.info("Test 3 - tablet meta after ALTER: ${tabletMeta[0]}")
+
+ logger.info("=== All tests passed ===")
+ logger.info("If any debug point check failed, the BE would have crashed with LOG(FATAL)")
+}
diff --git a/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy b/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy
new file mode 100644
index 00000000000..4854ec8b8e6
--- /dev/null
+++ b/regression-test/suites/variant_p0/test_variant_modify_doc_materialization_min_rows.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_modify_doc_materialization_min_rows", "p0") {
+ def tableName = "test_variant_modify_doc_min_rows"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ // Step 1: Create table with variant_doc_materialization_min_rows=10
+ sql """
+ CREATE TABLE ${tableName} (
+ k INT,
+ v variant<properties("variant_enable_doc_mode"="true","variant_doc_materialization_min_rows"="10", "variant_doc_hash_shard_count" = "32")>
+ ) ENGINE=OLAP
+ DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ // Verify SHOW CREATE TABLE shows min_rows=5
+ def createResult = sql """ SHOW CREATE TABLE ${tableName} """
+ def createStmt = createResult[0][1]
+ assertTrue(createStmt.contains('"variant_doc_materialization_min_rows" = "10"'),
+ "SHOW CREATE TABLE should show variant_doc_materialization_min_rows=10, got: ${createStmt}")
+
+ // Verify tablet meta shows min_rows=5
+ // Insert 3 rows with different paths (3 < 5, so no sub-columns should be materialized)
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, '{"path_a": 100, "path_b": "hello", "path_c": 1.5}'),
+ (2, '{"path_a": 200, "path_b": "world", "path_c": 2.5}'),
+ (3, '{"path_a": 300, "path_b": "doris", "path_c": 3.5}')
+ """
+
+ // Sync rowsets
+ sql "SELECT * FROM ${tableName} LIMIT 1"
+
+ // Enable describe_extend_variant_column and check DESC
+ sql """set describe_extend_variant_column = true"""
+ // Expect: only k and v columns, NO sub-columns extracted (3 rows < min_rows=5)
+ qt_desc_step1 """desc ${tableName}"""
+
+ // Step 2: ALTER TABLE MODIFY COLUMN to change variant_doc_materialization_min_rows to 2
+ sql """
+ ALTER TABLE ${tableName} MODIFY COLUMN v variant<properties("variant_enable_doc_mode"="true","variant_doc_materialization_min_rows"="2", "variant_doc_hash_shard_count" = "32")>;
+ """
+
+ // Verify SHOW CREATE TABLE shows min_rows=2
+ createResult = sql """ SHOW CREATE TABLE ${tableName} """
+ createStmt = createResult[0][1]
+ assertTrue(createStmt.contains('"variant_doc_materialization_min_rows" = "2"'),
+ "SHOW CREATE TABLE should show variant_doc_materialization_min_rows=2 after ALTER, got: ${createStmt}")
+
+ // Insert 3 more rows with different paths (3 >= 2, so sub-columns should be materialized for new data)
+ sql """ INSERT INTO ${tableName} VALUES
+ (4, '{"path_d": 400, "path_e": "alpha", "path_f": 4.5}'),
+ (5, '{"path_d": 500, "path_e": "beta", "path_f": 5.5}'),
+ (6, '{"path_d": 600, "path_e": "gamma", "path_f": 6.5}')
+ """
+
+ // Sync rowsets
+ sql "SELECT * FROM ${tableName} LIMIT 1"
+
+ // Expect: 3 sub-columns from new data (path_d, path_e, path_f)
+ qt_desc_step2 """desc ${tableName}"""
+
+ // Step 3: Trigger compaction - old data (path_a, path_b, path_c) should also get materialized
+ trigger_and_wait_compaction(tableName, "full")
+
+ // Sync rowsets
+ sql "SELECT * FROM ${tableName} LIMIT 1"
+
+ // Expect: 6 sub-columns total (path_a through path_f all materialized after compaction with min_rows=2)
+ qt_desc_step3 """desc ${tableName}"""
+
+ // Verify data integrity
+ def count = sql """ SELECT COUNT(*) FROM ${tableName} """
+ assertEquals(6, count[0][0])
+
+ // Cleanup
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org