You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2025/05/24 02:50:45 UTC
(doris) branch branch-3.0 updated: branch-3.0: [opt](cloud) load data no call partition.getVisibleVersion in cloud mode #51111 (#51150)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new b251837f632 branch-3.0: [opt](cloud) load data no call partition.getVisibleVersion in cloud mode #51111 (#51150)
b251837f632 is described below
commit b251837f632d51f1fbc4718a23f78566ee0406e3
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat May 24 10:50:38 2025 +0800
branch-3.0: [opt](cloud) load data no call partition.getVisibleVersion in cloud mode #51111 (#51150)
Cherry-picked from #51111
Co-authored-by: yujun <yu...@selectdb.com>
---
.../org/apache/doris/planner/OlapTableSink.java | 44 +++++++++++-----------
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index 31c15d8a2c1..0f4f1f9f17d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -46,6 +46,7 @@ import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.cloud.qe.ComputeGroupException;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -55,7 +56,6 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.DebugPointUtil.DebugPoint;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.rpc.RpcException;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
@@ -649,31 +649,30 @@ public class OlapTableSink extends DataSink {
TOlapTableLocationParam slaveLocationParam = new TOlapTableLocationParam();
// BE id -> path hash
Multimap<Long, Long> allBePathsMap = HashMultimap.create();
- List<Partition> partitions = partitionIds.stream().map(partitionId -> table.getPartition(partitionId))
- .collect(Collectors.toList());
- List<Long> visibleVersions = null;
- try {
- visibleVersions = Partition.getVisibleVersions(partitions);
- } catch (RpcException e) {
- throw new UserException("OlapTableSink get partition visible version failed", e);
- }
- for (int i = 0; i < partitions.size(); i++) {
- Partition partition = partitions.get(i);
- long visibleVersion = visibleVersions.get(i);
+ for (long partitionId : partitionIds) {
+ Partition partition = table.getPartition(partitionId);
int loadRequiredReplicaNum = table.getLoadRequiredReplicaNum(partition.getId());
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) {
// we should ensure the replica backend is alive
// otherwise, there will be a 'unknown node id, id=xxx' error for stream load
for (Tablet tablet : index.getTablets()) {
- String errMsg = "";
+ StringBuilder errMsgBuilder = new StringBuilder();
Multimap<Long, Long> bePathsMap = HashMultimap.create();
try {
bePathsMap = tablet.getNormalReplicaBackendPathMap();
if (bePathsMap.keySet().size() < loadRequiredReplicaNum) {
- errMsg = "tablet " + tablet.getId() + " alive replica num " + bePathsMap.keySet().size()
- + " < load required replica num " + loadRequiredReplicaNum
- + ", alive backends: [" + StringUtils.join(bePathsMap.keySet(), ",") + "]"
- + ", detail: " + tablet.getDetailsStatusForQuery(visibleVersion);
+ errMsgBuilder.append("tablet ").append(tablet.getId())
+ .append(" alive replica num ").append(bePathsMap.keySet().size())
+ .append(" < load required replica num ").append(loadRequiredReplicaNum)
+ .append(", alive backends: [")
+ .append(StringUtils.join(bePathsMap.keySet(), ","))
+ .append("]");
+ if (!Config.isCloudMode()) {
+ // in cloud mode, partition get visible version is a rpc,
+ // and each cluster has only one replica, no need to detail the replicas in cloud mode.
+ errMsgBuilder.append(", detail: ")
+ .append(tablet.getDetailsStatusForQuery(partition.getVisibleVersion()));
+ }
long now = System.currentTimeMillis();
long lastLoadFailedTime = tablet.getLastLoadFailedTime();
tablet.setLastLoadFailedTime(now);
@@ -681,15 +680,16 @@ public class OlapTableSink extends DataSink {
Env.getCurrentEnv().getTabletScheduler().tryAddRepairTablet(
tablet, dbId, table, partition, index, 0);
}
- throw new UserException(InternalErrorCode.REPLICA_FEW_ERR, errMsg);
+ throw new UserException(InternalErrorCode.REPLICA_FEW_ERR, errMsgBuilder.toString());
}
} catch (ComputeGroupException e) {
LOG.warn("failed to get replica backend path for tablet " + tablet.getId(), e);
- errMsg += e.toString();
- throw new UserException(InternalErrorCode.INTERNAL_ERR, errMsg);
+ errMsgBuilder.append(", ").append(e.toString());
+ throw new UserException(InternalErrorCode.INTERNAL_ERR, errMsgBuilder.toString());
+ }
+ if (!Config.isCloudMode()) {
+ debugWriteRandomChooseSink(tablet, partition.getVisibleVersion(), bePathsMap);
}
-
- debugWriteRandomChooseSink(tablet, visibleVersion, bePathsMap);
if (bePathsMap.keySet().isEmpty()) {
throw new UserException(InternalErrorCode.REPLICA_FEW_ERR,
"tablet " + tablet.getId() + " no available replica");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org