You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by w4...@apache.org on 2026/04/02 02:09:39 UTC
(doris) branch master updated: [fix](cloud) add get_version and get_tablet_stats case (#61915)
This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e29efd9f27c [fix](cloud) add get_version and get_tablet_stats case (#61915)
e29efd9f27c is described below
commit e29efd9f27c2279b22f288bff85a21a684a9dece
Author: meiyi <me...@selectdb.com>
AuthorDate: Thu Apr 2 10:09:27 2026 +0800
[fix](cloud) add get_version and get_tablet_stats case (#61915)
---
.../apache/doris/catalog/CloudTabletStatMgr.java | 8 ++
.../transaction/CloudGlobalTransactionMgr.java | 4 +
.../java/org/apache/doris/qe/SessionVariable.java | 4 +-
.../tablets/test_tablet_stat_syncer.groovy | 150 +++++++++++++++++++++
.../cloud_p0/version/test_version_syncer.groovy | 112 +++++++++++++++
5 files changed, 276 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index 7e23cfe3bbd..b28cc369d46 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -29,6 +29,7 @@ import org.apache.doris.cloud.rpc.MetaServiceProxy;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.rpc.RpcException;
@@ -47,6 +48,7 @@ import org.apache.logging.log4j.Logger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
@@ -582,6 +584,12 @@ public class CloudTabletStatMgr extends MasterDaemon {
if (Config.cloud_get_tablet_stats_version == 1 || tabletIds == null || tabletIds.isEmpty()) {
return;
}
+ List<String> ignoreTablets = Arrays.asList(DebugPointUtil.getDebugParamOrDefault(
+ "FE.CloudTabletStatMgr.addActiveTablets.ignore.tablets", "").split(","));
+ if (!ignoreTablets.isEmpty() && tabletIds.stream().anyMatch(id -> ignoreTablets.contains(String.valueOf(id)))) {
+ LOG.info("ignore adding active tablets: {}, debug param: {}", tabletIds, ignoreTablets);
+ return;
+ }
activeTablets.addAll(tabletIds);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index a8baff54700..0423b9459e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -573,6 +573,10 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
}
private Map<Long, List<Long>> updateVersion(CommitTxnResponse commitTxnResponse) {
+ if (DebugPointUtil.isEnable("FE.CloudGlobalTransactionMgr.updateVersion.disabled")) {
+ LOG.info("FE.CloudGlobalTransactionMgr.updateVersion.disabled");
+ return Collections.emptyMap();
+ }
long dbId = commitTxnResponse.getTxnInfo().getDbId();
long txnId = commitTxnResponse.getTxnInfo().getTxnId();
int totalPartitionNum = commitTxnResponse.getPartitionIdsList().size();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 6598a2e5f9f..48880093347 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -3130,13 +3130,13 @@ public class SessionVariable implements Serializable, Writable {
public String cloudCluster = "";
@VariableMgr.VarAttr(name = DISABLE_EMPTY_PARTITION_PRUNE)
public boolean disableEmptyPartitionPrune = false;
- @VariableMgr.VarAttr(name = CLOUD_FORCE_SYNC_TABLET_STATS)
+ @VariableMgr.VarAttr(name = CLOUD_FORCE_SYNC_TABLET_STATS, needForward = true)
public boolean cloudForceSyncTabletStats = false;
@VariableMgr.VarAttr(name = CLOUD_PARTITION_VERSION_CACHE_TTL_MS)
public long cloudPartitionVersionCacheTtlMs = Long.MAX_VALUE;
@VariableMgr.VarAttr(name = CLOUD_TABLE_VERSION_CACHE_TTL_MS)
public long cloudTableVersionCacheTtlMs = Long.MAX_VALUE;
- @VariableMgr.VarAttr(name = CLOUD_FORCE_SYNC_VERSION)
+ @VariableMgr.VarAttr(name = CLOUD_FORCE_SYNC_VERSION, needForward = true)
public boolean cloudForceSyncVersion = false;
// CLOUD_VARIABLES_END
diff --git a/regression-test/suites/cloud_p0/tablets/test_tablet_stat_syncer.groovy b/regression-test/suites/cloud_p0/tablets/test_tablet_stat_syncer.groovy
new file mode 100644
index 00000000000..1902a68c647
--- /dev/null
+++ b/regression-test/suites/cloud_p0/tablets/test_tablet_stat_syncer.groovy
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite("test_tablet_stat_syncer", "docker") {
+ if (!isCloudMode()) {
+ return
+ }
+
+ def options = new ClusterOptions()
+ options.setFeNum(2)
+ options.setBeNum(1)
+ options.cloudMode = true
+ options.enableDebugPoints()
+ options.connectToFollower = true
+
+ docker(options) {
+ def tbl = 'test_tablet_stat_syncer_tbl'
+ sql """ DROP TABLE IF EXISTS ${tbl} """
+
+ def configResult = sql_return_maparray """ ADMIN SHOW FRONTEND CONFIG LIKE '%cloud_get_tablet_stats_version%' """
+ assertTrue(configResult.size() > 0, "Expected to find cloud_get_tablet_stats_version config")
+ def tabletStatsVersion = configResult[0].Value.toInteger()
+ logger.info("cloud_get_tablet_stats_version = ${tabletStatsVersion}")
+ if (tabletStatsVersion != 2) {
+ logger.info("cloud_get_tablet_stats_version is not 2, skip test")
+ return
+ }
+
+ def intervalConfigResult = sql_return_maparray """ ADMIN SHOW FRONTEND CONFIG LIKE '%tablet_stat_update_interval_second%' """
+ assertTrue(intervalConfigResult.size() > 0, "Expected to find tablet_stat_update_interval_second config")
+ def statUpdateInterval = intervalConfigResult[0].Value.toInteger()
+ logger.info("tablet_stat_update_interval_second = ${statUpdateInterval}")
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbl} (
+ id INT NOT NULL,
+ name VARCHAR(50) NOT NULL,
+ value INT NOT NULL
+ )
+ ENGINE = olap
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ def result = sql_return_maparray """ select * from ${tbl} """
+ assertEquals(0, result.size())
+
+ sql """ INSERT INTO ${tbl} VALUES (1, 'initial', 100) """
+ result = sql_return_maparray """ select * from ${tbl} where id = 1 """
+ assertEquals(1, result.size())
+
+ def tablets = sql_return_maparray """ show tablets from ${tbl} """
+ assertTrue(tablets.size() > 0, "Expected at least one tablet")
+ def tabletIds = tablets.collect { it.TabletId }.join(",")
+ logger.info("Tablet IDs: ${tabletIds}")
+
+ def initialRowCount = tablets[0].RowCount
+ logger.info("Initial RowCount: ${initialRowCount}")
+
+ GetDebugPoint().enableDebugPointForAllFEs("FE.CloudTabletStatMgr.addActiveTablets.ignore.tablets", [value: tabletIds])
+
+ sql """ INSERT INTO ${tbl} VALUES (2, 'test', 200) """
+ result = sql_return_maparray """ select * from ${tbl} where id = 2 """
+ assertEquals(1, result.size(), "Data should be visible in query")
+
+ def tabletsAfterInsert = sql_return_maparray """ show tablets from ${tbl} """
+ def rowCountAfterInsert = tabletsAfterInsert[0].RowCount
+ logger.info("RowCount after insert (before sync): ${rowCountAfterInsert}")
+
+ GetDebugPoint().disableDebugPointForAllFEs("FE.CloudTabletStatMgr.addActiveTablets.ignore.tablets")
+
+ def maxWaitMs = (statUpdateInterval * 3 + 5) * 1000
+ def intervalMs = 1000
+ def startTime = System.currentTimeMillis()
+ def updated = false
+ while (true) {
+ def tabletsNow = sql_return_maparray """ show tablets from ${tbl} """
+ def rowCountNow = tabletsNow[0].RowCount.toInteger()
+ logger.info("RowCount get: ${rowCountNow}")
+ if (rowCountNow == 2) {
+ logger.info("RowCount updated: ${rowCountNow}")
+ updated = true
+ break
+ }
+ if (System.currentTimeMillis() - startTime > maxWaitMs) {
+ throw new IllegalStateException("Timeout waiting for tablet stat update. Waited ${maxWaitMs}ms, RowCount still ${rowCountNow}")
+ }
+ Thread.sleep(intervalMs)
+ }
+ assertTrue(updated, "Tablet stat should eventually update after debug point disabled")
+ tabletsAfterInsert = sql_return_maparray """ show tablets from ${tbl} """
+ rowCountAfterInsert = tabletsAfterInsert[0].RowCount.toInteger()
+ logger.info("RowCount after stat sync: ${rowCountAfterInsert}")
+ assertEquals(2, rowCountAfterInsert, "RowCount should reflect the new row after stat sync")
+
+ sql """ INSERT INTO ${tbl} VALUES (3, 'force_sync', 300) """
+ def tabletsBeforeSync = sql_return_maparray """ show tablets from ${tbl} """
+ def localDataSizeBeforeSync = tabletsBeforeSync[0].LocalDataSize
+ def rowCountBeforeSync = tabletsBeforeSync[0].RowCount
+ logger.info("LocalDataSize before force sync: ${localDataSizeBeforeSync}, RowCount: ${rowCountBeforeSync}")
+
+ sql """ set cloud_force_sync_tablet_stats = true """
+ def tabletsAfterForceSync = sql_return_maparray """ show tablets from ${tbl} """
+ def localDataSizeAfterForceSync = tabletsAfterForceSync[0].LocalDataSize
+ def rowCountAfterForceSync = tabletsAfterForceSync[0].RowCount
+ logger.info("LocalDataSize after force sync: ${localDataSizeAfterForceSync}, RowCount: ${rowCountAfterForceSync}")
+ sql """ set cloud_force_sync_tablet_stats = false """
+
+ startTime = System.currentTimeMillis()
+ updated = false
+ while (true) {
+ def tabletsNow = sql_return_maparray """ show tablets from ${tbl} """
+ def rowCountNow = tabletsNow[0].RowCount.toInteger()
+ logger.info("RowCount get 3: ${rowCountNow}")
+ if (rowCountNow == 3) {
+ logger.info("RowCount updated 3: ${rowCountNow}")
+ updated = true
+ break
+ }
+ if (System.currentTimeMillis() - startTime > maxWaitMs) {
+ throw new IllegalStateException("Timeout waiting for tablet stat update. Waited ${maxWaitMs}ms, RowCount still ${rowCountNow}")
+ }
+ Thread.sleep(intervalMs)
+ }
+ assertTrue(updated, "Tablet stat should eventually update after debug point disabled")
+ tabletsAfterInsert = sql_return_maparray """ show tablets from ${tbl} """
+ rowCountAfterInsert = tabletsAfterInsert[0].RowCount.toInteger()
+ logger.info("RowCount after stat sync: ${rowCountAfterInsert}")
+ assertEquals(3, rowCountAfterInsert, "RowCount should reflect the new row after stat sync")
+ }
+}
diff --git a/regression-test/suites/cloud_p0/version/test_version_syncer.groovy b/regression-test/suites/cloud_p0/version/test_version_syncer.groovy
new file mode 100644
index 00000000000..dbfab8c3048
--- /dev/null
+++ b/regression-test/suites/cloud_p0/version/test_version_syncer.groovy
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+
+suite("test_version_syncer", "docker") {
+ if (!isCloudMode()) {
+ return
+ }
+
+ def options = new ClusterOptions()
+ options.setFeNum(2)
+ options.setBeNum(1)
+ options.cloudMode = true
+ options.enableDebugPoints()
+ options.connectToFollower = true
+
+ docker(options) {
+ def tbl = 'test_version_syncer_tbl'
+ sql """ DROP TABLE IF EXISTS ${tbl} """
+
+ def enableSyncerConfig = sql_return_maparray """ ADMIN SHOW FRONTEND CONFIG LIKE '%cloud_enable_version_syncer%' """
+ assertTrue(enableSyncerConfig.size() > 0, "Expected to find cloud_enable_version_syncer config")
+ def enableSyncer = enableSyncerConfig[0].Value.toBoolean()
+ logger.info("cloud_enable_version_syncer = ${enableSyncer}")
+ if (!enableSyncer) {
+ logger.info("FE.CloudGlobalTransactionMgr.updateVersion.disabled")
+ return
+ }
+ def configResult = sql_return_maparray """ ADMIN SHOW FRONTEND CONFIG LIKE '%cloud_version_syncer_interval_second%' """
+ assertTrue(configResult.size() > 0, "Expected to find cloud_version_syncer_interval_second config")
+ def syncerInterval = configResult[0].Value.toInteger()
+ logger.info("cloud_version_syncer_interval_second = ${syncerInterval}")
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbl} (
+ id INT NOT NULL,
+ name VARCHAR(50) NOT NULL,
+ value INT NOT NULL
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ def result = sql_return_maparray """ select * from ${tbl} """
+ assertEquals(0, result.size())
+
+ sql """ INSERT INTO ${tbl} VALUES (1, 'initial', 100) """
+ result = sql_return_maparray """ select * from ${tbl} where id = 1 """
+ assertEquals(1, result.size())
+
+ DebugPoint.enableDebugPointForAllFEs('FE.CloudGlobalTransactionMgr.updateVersion.disabled')
+
+ sql """ INSERT INTO ${tbl} VALUES (2, 'test', 200) """
+ result = sql_return_maparray """ select * from ${tbl} where id = 2 """
+ assertEquals(0, result.size(), "Data should not be visible before version sync")
+
+ def maxWaitMs = (syncerInterval * 2 + 5) * 1000
+ def intervalMs = 1000
+ def startTime = System.currentTimeMillis()
+ def found = false
+ while (true) {
+ result = sql_return_maparray """ select * from ${tbl} where id = 2 """
+ if (result.size() == 1) {
+ assertEquals('test', result[0].name)
+ assertEquals(200, result[0].value)
+ found = true
+ break
+ }
+ if (System.currentTimeMillis() - startTime > maxWaitMs) {
+ throw new IllegalStateException("Timeout waiting for data to be visible. Waited ${maxWaitMs}ms")
+ }
+ Thread.sleep(intervalMs)
+ }
+ assertTrue(found, "Data should eventually be visible after version syncer runs")
+
+ result = sql_return_maparray """ select * from ${tbl} order by id """
+ assertEquals(2, result.size())
+
+ // Test cloud_force_sync_version session variable
+ // Step 1: Insert data with version sync disabled, query should return no data
+ sql """ INSERT INTO ${tbl} VALUES (3, 'force_sync', 300) """
+ result = sql_return_maparray """ select * from ${tbl} where id = 3 """
+ assertEquals(0, result.size(), "Data should not be visible before force sync")
+
+ // Step 2: Enable session variable cloud_force_sync_version = true and execute show partitions
+ sql """ set cloud_force_sync_version = true """
+ def showPartitionsResult = sql_return_maparray """ show partitions from ${tbl} """
+ logger.info("Show partitions result: ${showPartitionsResult}")
+
+ // Step 3: Query again, data should now be visible
+ result = sql_return_maparray """ select * from ${tbl} where id = 3 """
+ assertEquals(1, result.size(), "Data should be visible after force sync")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org