You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/12 07:09:55 UTC
(doris) 09/27: [fix](restore) Fix the conflict IDs between two cluster (#33423)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fe772c76e7d8d0309838be18d8c0382c330dd4af
Author: walter <w4...@gmail.com>
AuthorDate: Wed Apr 10 18:57:10 2024 +0800
[fix](restore) Fix the conflict IDs between two cluster (#33423)
The meta of the restore may come from different clusters, so the
original ID in the meta may conflict with the ID of the new cluster. For
example, if a newly allocated ID happens to be the same as an original ID,
the original one may be overwritten when executing `put`.
---
.../java/org/apache/doris/catalog/OlapTable.java | 51 ++++++++++------------
.../org/apache/doris/catalog/PartitionInfo.java | 30 ++++++++++---
2 files changed, 47 insertions(+), 34 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 723f7dba065..c79a71461b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -566,7 +566,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
}
public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restoreReplicaAlloc,
- boolean reserveReplica) {
+ boolean reserveReplica) {
+ // ATTN: The meta of the restore may come from different clusters, so the
+ // original ID in the meta may conflict with the ID of the new cluster. For
+ // example, if a newly allocated ID happens to be the same as an original ID,
+ // the original one may be overwritten when executing `put`, then causes a
+ // NullPointerException.
+
// table id
id = env.getNextId();
@@ -577,13 +583,15 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
}
// reset all 'indexIdToXXX' map
+ Map<Long, MaterializedIndexMeta> origIdxIdToMeta = indexIdToMeta;
+ indexIdToMeta = Maps.newHashMap();
for (Map.Entry<Long, String> entry : origIdxIdToName.entrySet()) {
long newIdxId = env.getNextId();
if (entry.getValue().equals(name)) {
// base index
baseIndexId = newIdxId;
}
- indexIdToMeta.put(newIdxId, indexIdToMeta.remove(entry.getKey()));
+ indexIdToMeta.put(newIdxId, origIdxIdToMeta.get(entry.getKey()));
indexNameToId.put(entry.getValue(), newIdxId);
}
@@ -594,36 +602,25 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
}
// reset partition info and idToPartition map
- if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) {
- for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
- long newPartId = env.getNextId();
- if (reserveReplica) {
- ReplicaAllocation originReplicaAlloc = partitionInfo.getReplicaAllocation(entry.getValue());
- partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), originReplicaAlloc, false);
- } else {
- partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), restoreReplicaAlloc, false);
- }
- idToPartition.put(newPartId, idToPartition.remove(entry.getValue()));
- }
- } else {
- // Single partitioned
+ Map<Long, Long> partitionMap = Maps.newHashMap();
+ Map<Long, Partition> origIdToPartition = idToPartition;
+ idToPartition = Maps.newHashMap();
+ for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
long newPartId = env.getNextId();
- for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
- if (reserveReplica) {
- ReplicaAllocation originReplicaAlloc = partitionInfo.getReplicaAllocation(entry.getValue());
- partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), originReplicaAlloc, true);
- } else {
- partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), restoreReplicaAlloc, true);
- }
- idToPartition.put(newPartId, idToPartition.remove(entry.getValue()));
- }
+ idToPartition.put(newPartId, origIdToPartition.get(entry.getValue()));
+ partitionMap.put(newPartId, entry.getValue());
}
+ boolean isSinglePartition = partitionInfo.getType() != PartitionType.RANGE
+ && partitionInfo.getType() != PartitionType.LIST;
+ partitionInfo.resetPartitionIdForRestore(partitionMap,
+ reserveReplica ? null : restoreReplicaAlloc, isSinglePartition);
// for each partition, reset rollup index map
- Map<Tag, Integer> nextIndexs = Maps.newHashMap();
+ Map<Tag, Integer> nextIndexes = Maps.newHashMap();
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
Partition partition = entry.getValue();
- // entry.getKey() is the new partition id, use it to get the restore specified replica allocation
+ // entry.getKey() is the new partition id, use it to get the restore specified
+ // replica allocation
ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(entry.getKey());
for (Map.Entry<Long, String> entry2 : origIdxIdToName.entrySet()) {
MaterializedIndex idx = partition.getIndex(entry2.getKey());
@@ -648,7 +645,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
try {
Pair<Map<Tag, List<Long>>, TStorageMedium> tag2beIdsAndMedium =
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(
- replicaAlloc, nextIndexs, null, false, false);
+ replicaAlloc, nextIndexes, null, false, false);
Map<Tag, List<Long>> tag2beIds = tag2beIdsAndMedium.first;
for (Map.Entry<Tag, List<Long>> entry3 : tag2beIds.entrySet()) {
for (Long beId : entry3.getValue()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 70aee07fad2..c899a4e8917 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -390,15 +390,31 @@ public class PartitionInfo implements Writable {
}
}
- public void resetPartitionIdForRestore(long newPartitionId, long oldPartitionId,
+ public void resetPartitionIdForRestore(
+ Map<Long, Long> partitionIdMap,
ReplicaAllocation restoreReplicaAlloc, boolean isSinglePartitioned) {
- idToDataProperty.put(newPartitionId, idToDataProperty.remove(oldPartitionId));
- idToReplicaAllocation.remove(oldPartitionId);
- idToReplicaAllocation.put(newPartitionId, restoreReplicaAlloc);
- if (!isSinglePartitioned) {
- idToItem.put(newPartitionId, idToItem.remove(oldPartitionId));
+ Map<Long, DataProperty> origIdToDataProperty = idToDataProperty;
+ Map<Long, ReplicaAllocation> origIdToReplicaAllocation = idToReplicaAllocation;
+ Map<Long, PartitionItem> origIdToItem = idToItem;
+ Map<Long, Boolean> origIdToInMemory = idToInMemory;
+ Map<Long, String> origIdToStoragePolicy = idToStoragePolicy;
+ idToDataProperty = Maps.newHashMap();
+ idToReplicaAllocation = Maps.newHashMap();
+ idToItem = Maps.newHashMap();
+ idToInMemory = Maps.newHashMap();
+ idToStoragePolicy = Maps.newHashMap();
+
+ for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
+ idToDataProperty.put(entry.getKey(), origIdToDataProperty.get(entry.getValue()));
+ idToReplicaAllocation.put(entry.getKey(),
+ restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(entry.getValue())
+ : restoreReplicaAlloc);
+ if (!isSinglePartitioned) {
+ idToItem.put(entry.getKey(), origIdToItem.get(entry.getValue()));
+ }
+ idToInMemory.put(entry.getKey(), origIdToInMemory.get(entry.getValue()));
+ idToStoragePolicy.put(entry.getKey(), origIdToStoragePolicy.get(entry.getValue()));
}
- idToInMemory.put(newPartitionId, idToInMemory.remove(oldPartitionId));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org