You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2025/12/04 04:26:38 UTC
(doris) branch master updated: [fix](segmentv3) make it work in cloud (#58681)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d084c6051e8 [fix](segmentv3) make it work in cloud (#58681)
d084c6051e8 is described below
commit d084c6051e81fb4074c837b4125ebc182acae254
Author: lihangyu <li...@selectdb.com>
AuthorDate: Thu Dec 4 12:26:31 2025 +0800
[fix](segmentv3) make it work in cloud (#58681)
---
.../java/org/apache/doris/alter/CloudRollupJobV2.java | 4 ++--
.../org/apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../org/apache/doris/cloud/backup/CloudRestoreJob.java | 3 ++-
.../doris/cloud/common/util/CloudPropertyAnalyzer.java | 1 -
.../doris/cloud/datasource/CloudInternalCatalog.java | 15 +++++++++++++--
.../variant_p0/ext_meta/test_storage_format_v2_1.groovy | 6 +++---
6 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index 82b137a2c02..2b3a1f8937f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -217,8 +217,8 @@ public class CloudRollupJobV2 extends RollupJobV2 {
partitionId, rollupTablet, tabletType, rollupSchemaHash,
rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
tbl.getBfFpp(), null, rollupSchema,
- tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(),
- tbl.isInMemory(), true,
+ tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(),
+ tbl.getStoragePolicy(), tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index e0de2b1e3e2..0d02dc1bc48 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -239,7 +239,7 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
tbl.getPartitionInfo().getTabletType(partitionId),
shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns,
bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(),
- tbl.getCompressionType(),
+ tbl.getCompressionType(), tbl.getStorageFormat(),
tbl.getStoragePolicy(), tbl.isInMemory(), true,
tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index 80bd91d9d54..f9b1840f574 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -386,7 +386,8 @@ public class CloudRestoreJob extends RestoreJob {
indexMeta.getSchemaHash(), indexMeta.getKeysType(),
indexMeta.getShortKeyColumnCount(), localTbl.getCopiedBfColumns(),
localTbl.getBfFpp(), indexes, indexMeta.getSchema(), localTbl.getDataSortInfo(),
- localTbl.getCompressionType(), localTbl.getStoragePolicy(),
+ localTbl.getCompressionType(), localTbl.getStorageFormat(),
+ localTbl.getStoragePolicy(),
localTbl.isInMemory(), false, localTbl.getName(), localTbl.getTTLSeconds(),
localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.storeRowColumn(),
localTbl.getBaseSchemaVersion(), localTbl.getCompactionPolicy(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
index b41570a25fc..7ac160f4b18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
@@ -30,7 +30,6 @@ public class CloudPropertyAnalyzer extends PropertyAnalyzer {
forceProperties = ImmutableList.of(
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_INMEMORY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM),
- RewriteProperty.replace(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, "V2"),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME),
RewriteProperty.delete(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index c2359cc59b2..a3ee47960d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -65,6 +65,7 @@ import org.apache.doris.rpc.RpcException;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
+import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Preconditions;
@@ -171,7 +172,8 @@ public class CloudInternalCatalog extends InternalCatalog {
OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId,
partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount,
bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(),
- tbl.getCompressionType(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false,
+ tbl.getName(), tbl.getTTLSeconds(),
tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(),
tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(),
tbl.getTimeSeriesCompactionFileCountThreshold(),
@@ -208,7 +210,7 @@ public class CloudInternalCatalog extends InternalCatalog {
long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType,
short shortKeyColumnCount, Set<String> bfColumns, double bfFpp, List<Index> indexes,
List<Column> schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType,
- String storagePolicy, boolean isInMemory, boolean isShadow,
+ TStorageFormat storageFormat, String storagePolicy, boolean isInMemory, boolean isShadow,
String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite,
boolean storeRowColumn, int schemaVersion, String compactionPolicy,
Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold,
@@ -319,6 +321,15 @@ public class CloudInternalCatalog extends InternalCatalog {
break;
}
+ // Enable external column meta layout when storage_format is V3 (Cloud mode).
+ switch (storageFormat) {
+ case V3:
+ schemaBuilder.setIsExternalSegmentColumnMetaUsed(true);
+ break;
+ default:
+ break;
+ }
+
schemaBuilder.setSortColNum(dataSortInfo.getColNum());
for (int i = 0; i < schemaColumns.size(); i++) {
Column column = schemaColumns.get(i);
diff --git a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
index d7fe1e48e3a..ece39caed92 100644
--- a/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
+++ b/regression-test/suites/variant_p0/ext_meta/test_storage_format_v2_1.groovy
@@ -19,9 +19,9 @@ suite("test_storage_format_v2_1") {
def tableName = "test_storage_format_v2_1_table"
// Test 1: Create table with storage_format = V2.1
- sql "DROP TABLE IF EXISTS ${tableName}"
+ sql "DROP TABLE IF EXISTS test_storage_format_v2_1_table"
sql """
- CREATE TABLE ${tableName} (
+ CREATE TABLE test_storage_format_v2_1_table (
k bigint,
v variant
)
@@ -47,7 +47,7 @@ suite("test_storage_format_v2_1") {
// Verify table properties
def result = sql "SHOW CREATE TABLE ${tableName}"
logger.info("Show create table result: ${result}")
- assertTrue(result[0][1].contains("V3"),
+ assertTrue(result[0][1].contains("\"storage_format\" = \"V3\""),
"Table should be created with storage_format V3")
// Cleanup
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org