You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2025/12/01 21:05:25 UTC
(doris) branch cs_opt_version-3.1 updated: fix cloud meta for segment v3 (#58590)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch cs_opt_version-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/cs_opt_version-3.1 by this push:
new f3048ce50ad fix cloud meta for segment v3 (#58590)
f3048ce50ad is described below
commit f3048ce50ad61e5270f6e8a0ad7a07d7e11aad40
Author: lihangyu <li...@selectdb.com>
AuthorDate: Tue Dec 2 05:05:16 2025 +0800
fix cloud meta for segment v3 (#58590)
---
.../org/apache/doris/alter/CloudRollupJobV2.java | 43 ++++++++++-----------
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 44 +++++++++++-----------
.../cloud/common/util/CloudPropertyAnalyzer.java | 1 -
.../cloud/datasource/CloudInternalCatalog.java | 15 +++++++-
.../ext_meta/test_storage_format_v2_1.groovy | 6 +--
6 files changed, 62 insertions(+), 49 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index 4ef9ab42780..fcf7409c430 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -214,27 +214,28 @@ public class CloudRollupJobV2 extends RollupJobV2 {
for (Tablet rollupTablet : rollupIndex.getTablets()) {
OlapFile.TabletMetaCloudPB.Builder builder =
((CloudInternalCatalog) Env.getCurrentInternalCatalog())
- .createTabletMetaBuilder(tableId, rollupIndexId,
- partitionId, rollupTablet, tabletType, rollupSchemaHash,
- rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
- tbl.getBfFpp(), null, rollupSchema,
- tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(),
- tbl.isInMemory(), true,
- tbl.getName(), tbl.getTTLSeconds(),
- tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
- tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(),
- tbl.getTimeSeriesCompactionGoalSizeMbytes(),
- tbl.getTimeSeriesCompactionFileCountThreshold(),
- tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
- tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
- tbl.getTimeSeriesCompactionLevelThreshold(),
- tbl.disableAutoCompaction(),
- tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
- tbl.getEnableMowLightDelete(), null,
- tbl.rowStorePageSize(),
- tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
- tbl.storageDictPageSize(), true);
+ .createTabletMetaBuilder(tableId, rollupIndexId,
+ partitionId, rollupTablet, tabletType, rollupSchemaHash,
+ rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
+ tbl.getBfFpp(), null, rollupSchema,
+ tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(),
+ tbl.getStoragePolicy(),
+ tbl.isInMemory(), true,
+ tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
+ tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(),
+ tbl.getTimeSeriesCompactionGoalSizeMbytes(),
+ tbl.getTimeSeriesCompactionFileCountThreshold(),
+ tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+ tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
+ tbl.getTimeSeriesCompactionLevelThreshold(),
+ tbl.disableAutoCompaction(),
+ tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
+ tbl.getEnableMowLightDelete(), null,
+ tbl.rowStorePageSize(),
+ tbl.variantEnableFlattenNested(),
+ tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
+ tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
requestBuilder.setDbId(dbId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index 50a7f2c33da..2a2b32e693d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -258,7 +258,7 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
tbl.getPartitionInfo().getTabletType(partitionId),
shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns,
bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(),
- tbl.getCompressionType(),
+ tbl.getCompressionType(), tbl.getStorageFormat(),
tbl.getStoragePolicy(), tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index 4f0a7bce1b9..1a1bfb296ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -375,27 +375,29 @@ public class CloudRestoreJob extends RestoreJob {
try {
requestBuilder.addTabletMetas(((CloudInternalCatalog) Env.getCurrentInternalCatalog())
.createTabletMetaBuilder(localTbl.getId(), restoredIdx.getId(),
- restorePart.getId(), restoreTablet,
- localTbl.getPartitionInfo().getTabletType(restorePart.getId()),
- indexMeta.getSchemaHash(), indexMeta.getKeysType(),
- indexMeta.getShortKeyColumnCount(), localTbl.getCopiedBfColumns(),
- localTbl.getBfFpp(), indexes, indexMeta.getSchema(), localTbl.getDataSortInfo(),
- localTbl.getCompressionType(), localTbl.getStoragePolicy(),
- localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(),
- localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(),
- localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(),
- localTbl.getTimeSeriesCompactionGoalSizeMbytes(),
- localTbl.getTimeSeriesCompactionFileCountThreshold(),
- localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
- localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
- localTbl.getTimeSeriesCompactionLevelThreshold(), localTbl.disableAutoCompaction(),
- localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
- localTbl.getEnableMowLightDelete(),
- localTbl.getInvertedIndexFileStorageFormat(),
- localTbl.rowStorePageSize(),
- localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize(), localTbl.getTDEAlgorithmPB(),
- localTbl.storageDictPageSize(), false));
+ restorePart.getId(), restoreTablet,
+ localTbl.getPartitionInfo().getTabletType(restorePart.getId()),
+ indexMeta.getSchemaHash(), indexMeta.getKeysType(),
+ indexMeta.getShortKeyColumnCount(), localTbl.getCopiedBfColumns(),
+ localTbl.getBfFpp(), indexes, indexMeta.getSchema(), localTbl.getDataSortInfo(),
+ localTbl.getCompressionType(), localTbl.getStorageFormat(),
+ localTbl.getStoragePolicy(),
+ localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(),
+ localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(),
+ localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(),
+ localTbl.getTimeSeriesCompactionGoalSizeMbytes(),
+ localTbl.getTimeSeriesCompactionFileCountThreshold(),
+ localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+ localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
+ localTbl.getTimeSeriesCompactionLevelThreshold(),
+ localTbl.disableAutoCompaction(),
+ localTbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
+ localTbl.getEnableMowLightDelete(),
+ localTbl.getInvertedIndexFileStorageFormat(),
+ localTbl.rowStorePageSize(),
+ localTbl.variantEnableFlattenNested(),
+ localTbl.storagePageSize(), localTbl.getTDEAlgorithmPB(),
+ localTbl.storageDictPageSize(), false));
// In cloud mode all storage medium will be saved to HDD.
TabletMeta tabletMeta = new TabletMeta(db.getId(), localTbl.getId(), restorePart.getId(),
restoredIdx.getId(), indexMeta.getSchemaHash(), TStorageMedium.HDD);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index b41570a25fc..7ac160f4b18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -30,7 +30,6 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
forceProperties = ImmutableList.of(
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_INMEMORY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM),
- RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 9d516880818..457a5ca1f13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -65,6 +65,7 @@ import org.apache.doris.rpc.RpcException;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
+import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Preconditions;
@@ -165,7 +166,8 @@ public class CloudInternalCatalog extends InternalCatalog {
OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId,
partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount,
bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(),
- tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false,
+ tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(),
tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(),
tbl.getTimeSeriesCompactionFileCountThreshold(),
@@ -203,7 +205,7 @@ public class CloudInternalCatalog extends InternalCatalog {
long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType,
short shortKeyColumnCount, Set<String> bfColumns, double bfFpp, List<Index> indexes,
List<Column> schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType,
- String storagePolicy, boolean isInMemory, boolean isShadow,
+ TStorageFormat storageFormat, String storagePolicy, boolean isInMemory, boolean isShadow,
String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite,
boolean storeRowColumn, int schemaVersion, String compactionPolicy,
Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold,
@@ -313,6 +315,15 @@ public class CloudInternalCatalog extends InternalCatalog {
break;
}
+ // Enable external column meta layout when storage_format is V3 (Cloud mode).
+ switch (storageFormat) {
+ case V3:
+ schemaBuilder.setIsExternalSegmentMetaUsedDefault(true);
+ break;
+ default:
+ break;
+ }
+
schemaBuilder.setSortColNum(dataSortInfo.getColNum());
for (int i = 0; i < schemaColumns.size(); i++) {
Column column = schemaColumns.get(i);
diff --git a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
index d7fe1e48e3a..ece39caed92 100644
--- a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
+++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
@@ -19,9 +19,9 @@ suite("test_storage_format_v2_1") {
def tableName = "test_storage_format_v2_1_table"
// Test 1: Create table with storage_format = V2.1
- sql "DROP TABLE IF EXISTS ${tableName}"
+ sql "DROP TABLE IF EXISTS test_storage_format_v2_1_table"
sql """
- CREATE TABLE ${tableName} (
+ CREATE TABLE test_storage_format_v2_1_table (
k bigint,
v variant
)
@@ -47,7 +47,7 @@ suite("test_storage_format_v2_1") {
// Verify table properties
def result = sql "SHOW CREATE TABLE ${tableName}"
logger.info("Show create table result: ${result}")
- assertTrue(result[0][1].contains("V3"),
+ assertTrue(result[0][1].contains("\"storage_format\" = \"V3\""),
"Table should be created with storage_format V3")
// Cleanup
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org