You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ca...@apache.org on 2022/04/17 02:01:40 UTC
[incubator-doris] branch master updated: [Refactor](ReportHandler) Remove some unused schema_hash code in fe (#9005)
This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0f8a7ff985 [Refactor](ReportHandler) Remove some unused schema_hash code in fe (#9005)
0f8a7ff985 is described below
commit 0f8a7ff98584557d547c6bbdbd3c1569676486e8
Author: caiconghui <55...@users.noreply.github.com>
AuthorDate: Sun Apr 17 10:01:34 2022 +0800
[Refactor](ReportHandler) Remove some unused schema_hash code in fe (#9005)
---
.../java/org/apache/doris/catalog/Replica.java | 16 --
.../main/java/org/apache/doris/catalog/Tablet.java | 6 +-
.../apache/doris/catalog/TabletInvertedIndex.java | 225 ++++++++-------------
.../java/org/apache/doris/catalog/TabletMeta.java | 45 -----
.../java/org/apache/doris/master/MasterImpl.java | 17 --
.../org/apache/doris/master/ReportHandler.java | 98 ++++-----
.../org/apache/doris/planner/OlapScanNode.java | 9 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
8 files changed, 129 insertions(+), 289 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
index a42cebdc8a..b8660d1848 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -260,22 +260,6 @@ public class Replica implements Writable {
updateReplicaInfo(newVersion, lastFailedVersion, lastSuccessVersion, dataSize, rowCount);
}
- public void updateVersionInfoForRecovery(
- long newVersion,
- long lastFailedVersion,
- long lastSuccessVersion) {
-
- LOG.warn("update replica {} on backend {}'s version for recovery. version: {}:{}."
- + " last failed version: {}:{}, last success version: {}:{}",
- this.id, this.backendId, this.version, newVersion,
- this.lastFailedVersion, lastFailedVersion,
- this.lastSuccessVersion, lastSuccessVersion);
-
- this.version = newVersion;
- this.lastFailedVersion = lastFailedVersion;
- this.lastSuccessVersion = lastSuccessVersion;
- }
-
/* last failed version: LFV
* last success version: LSV
* version: V
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index 23852cf398..7f1cc537e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -213,7 +213,7 @@ public class Tablet extends MetaObject implements Writable {
}
// for query
- public List<Replica> getQueryableReplicas(long visibleVersion, int schemaHash) {
+ public List<Replica> getQueryableReplicas(long visibleVersion) {
List<Replica> allQueryableReplica = Lists.newArrayListWithCapacity(replicas.size());
for (Replica replica : replicas) {
if (replica.isBad()) {
@@ -227,9 +227,7 @@ public class Tablet extends MetaObject implements Writable {
ReplicaState state = replica.getState();
if (state.canQuery()) {
- // replica.getSchemaHash() == -1 is for compatibility
- if (replica.checkVersionCatchUp(visibleVersion, false)
- && (replica.getSchemaHash() == -1 || replica.getSchemaHash() == schemaHash)) {
+ if (replica.checkVersionCatchUp(visibleVersion, false)) {
allQueryableReplica.add(replica);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index 28e2569ddb..5f60c9e054 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -117,8 +117,7 @@ public class TabletInvertedIndex {
final HashMap<Long, TStorageMedium> storageMediumMap,
ListMultimap<Long, Long> tabletSyncMap,
ListMultimap<Long, Long> tabletDeleteFromMeta,
- Set<Long> foundTabletsWithValidSchema,
- Map<Long, TTabletInfo> foundTabletsWithInvalidSchema,
+ Set<Long> tabletFoundInMeta,
ListMultimap<TStorageMedium, Long> tabletMigrationMap,
Map<Long, ListMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
ListMultimap<Long, Long> transactionsToClear,
@@ -139,105 +138,99 @@ public class TabletInvertedIndex {
if (backendTablets.containsKey(tabletId)) {
TTablet backendTablet = backendTablets.get(tabletId);
Replica replica = entry.getValue();
- for (TTabletInfo backendTabletInfo : backendTablet.getTabletInfos()) {
- if (tabletMeta.containsSchemaHash(backendTabletInfo.getSchemaHash())) {
- foundTabletsWithValidSchema.add(tabletId);
- if (partitionIdInMemorySet.contains(backendTabletInfo.getPartitionId()) != backendTabletInfo.isIsInMemory()) {
- synchronized (tabletToInMemory) {
- tabletToInMemory.add(new ImmutableTriple<>(tabletId, backendTabletInfo.getSchemaHash(), !backendTabletInfo.isIsInMemory()));
- }
- }
- // 1. (intersection)
- if (needSync(replica, backendTabletInfo)) {
- // need sync
- synchronized (tabletSyncMap) {
- tabletSyncMap.put(tabletMeta.getDbId(), tabletId);
- }
- }
+ tabletFoundInMeta.add(tabletId);
+ TTabletInfo backendTabletInfo = backendTablet.getTabletInfos().get(0);
+ if (partitionIdInMemorySet.contains(backendTabletInfo.getPartitionId()) != backendTabletInfo.isIsInMemory()) {
+ synchronized (tabletToInMemory) {
+ tabletToInMemory.add(new ImmutableTriple<>(tabletId, backendTabletInfo.getSchemaHash(), !backendTabletInfo.isIsInMemory()));
+ }
+ }
+ // 1. (intersection)
+ if (needSync(replica, backendTabletInfo)) {
+ // need sync
+ synchronized (tabletSyncMap) {
+ tabletSyncMap.put(tabletMeta.getDbId(), tabletId);
+ }
+ }
- // check and set path
- // path info of replica is only saved in Master FE
- if (backendTabletInfo.isSetPathHash() &&
- replica.getPathHash() != backendTabletInfo.getPathHash()) {
- replica.setPathHash(backendTabletInfo.getPathHash());
- }
+ // check and set path
+ // path info of replica is only saved in Master FE
+ if (backendTabletInfo.isSetPathHash() &&
+ replica.getPathHash() != backendTabletInfo.getPathHash()) {
+ replica.setPathHash(backendTabletInfo.getPathHash());
+ }
- if (backendTabletInfo.isSetSchemaHash() && replica.getState() == ReplicaState.NORMAL
- && replica.getSchemaHash() != backendTabletInfo.getSchemaHash()) {
- // update the schema hash only when replica is normal
- replica.setSchemaHash(backendTabletInfo.getSchemaHash());
- }
+ if (backendTabletInfo.isSetSchemaHash() && replica.getState() == ReplicaState.NORMAL
+ && replica.getSchemaHash() != backendTabletInfo.getSchemaHash()) {
+ // update the schema hash only when replica is normal
+ replica.setSchemaHash(backendTabletInfo.getSchemaHash());
+ }
- if (needRecover(replica, tabletMeta.getOldSchemaHash(), backendTabletInfo)) {
- LOG.warn("replica {} of tablet {} on backend {} need recovery. "
- + "replica in FE: {}, report version {}, report schema hash: {},"
- + " is bad: {}, is version missing: {}",
- replica.getId(), tabletId, backendId, replica,
- backendTabletInfo.getVersion(),
- backendTabletInfo.getSchemaHash(),
- backendTabletInfo.isSetUsed() ? !backendTabletInfo.isUsed() : "false",
- backendTabletInfo.isSetVersionMiss() ? backendTabletInfo.isVersionMiss() : "unset");
- synchronized (tabletRecoveryMap) {
- tabletRecoveryMap.put(tabletMeta.getDbId(), tabletId);
- }
- }
+ if (needRecover(replica, tabletMeta.getOldSchemaHash(), backendTabletInfo)) {
+ LOG.warn("replica {} of tablet {} on backend {} need recovery. "
+ + "replica in FE: {}, report version {}, report schema hash: {},"
+ + " is bad: {}, is version missing: {}",
+ replica.getId(), tabletId, backendId, replica,
+ backendTabletInfo.getVersion(),
+ backendTabletInfo.getSchemaHash(),
+ backendTabletInfo.isSetUsed() ? !backendTabletInfo.isUsed() : "false",
+ backendTabletInfo.isSetVersionMiss() ? backendTabletInfo.isVersionMiss() : "unset");
+ synchronized (tabletRecoveryMap) {
+ tabletRecoveryMap.put(tabletMeta.getDbId(), tabletId);
+ }
+ }
- long partitionId = tabletMeta.getPartitionId();
- if (!Config.disable_storage_medium_check) {
- // check if need migration
- TStorageMedium storageMedium = storageMediumMap.get(partitionId);
- if (storageMedium != null && backendTabletInfo.isSetStorageMedium()) {
- if (storageMedium != backendTabletInfo.getStorageMedium()) {
- synchronized (tabletMigrationMap) {
- tabletMigrationMap.put(storageMedium, tabletId);
- }
- }
- if (storageMedium != tabletMeta.getStorageMedium()) {
- tabletMeta.setStorageMedium(storageMedium);
- }
+ long partitionId = tabletMeta.getPartitionId();
+ if (!Config.disable_storage_medium_check) {
+ // check if need migration
+ TStorageMedium storageMedium = storageMediumMap.get(partitionId);
+ if (storageMedium != null && backendTabletInfo.isSetStorageMedium()) {
+ if (storageMedium != backendTabletInfo.getStorageMedium()) {
+ synchronized (tabletMigrationMap) {
+ tabletMigrationMap.put(storageMedium, tabletId);
}
}
+ if (storageMedium != tabletMeta.getStorageMedium()) {
+ tabletMeta.setStorageMedium(storageMedium);
+ }
+ }
+ }
- // check if should clear transactions
- if (backendTabletInfo.isSetTransactionIds()) {
- List<Long> transactionIds = backendTabletInfo.getTransactionIds();
- GlobalTransactionMgr transactionMgr = Catalog.getCurrentGlobalTransactionMgr();
- for (Long transactionId : transactionIds) {
- TransactionState transactionState = transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId);
- if (transactionState == null || transactionState.getTransactionStatus() == TransactionStatus.ABORTED) {
- synchronized (transactionsToClear) {
- transactionsToClear.put(transactionId, tabletMeta.getPartitionId());
- }
- LOG.debug("transaction id [{}] is not valid any more, "
- + "clear it from backend [{}]", transactionId, backendId);
- } else if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
- TableCommitInfo tableCommitInfo = transactionState.getTableCommitInfo(tabletMeta.getTableId());
- PartitionCommitInfo partitionCommitInfo = tableCommitInfo == null ? null : tableCommitInfo.getPartitionCommitInfo(partitionId);
- if (partitionCommitInfo != null) {
- TPartitionVersionInfo versionInfo = new TPartitionVersionInfo(tabletMeta.getPartitionId(),
- partitionCommitInfo.getVersion(), 0);
- synchronized (transactionsToPublish) {
- ListMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(transactionState.getDbId());
- if (map == null) {
- map = ArrayListMultimap.create();
- transactionsToPublish.put(transactionState.getDbId(), map);
- }
- map.put(transactionId, versionInfo);
- }
+ // check if should clear transactions
+ if (backendTabletInfo.isSetTransactionIds()) {
+ List<Long> transactionIds = backendTabletInfo.getTransactionIds();
+ GlobalTransactionMgr transactionMgr = Catalog.getCurrentGlobalTransactionMgr();
+ for (Long transactionId : transactionIds) {
+ TransactionState transactionState = transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId);
+ if (transactionState == null || transactionState.getTransactionStatus() == TransactionStatus.ABORTED) {
+ synchronized (transactionsToClear) {
+ transactionsToClear.put(transactionId, tabletMeta.getPartitionId());
+ }
+ LOG.debug("transaction id [{}] is not valid any more, "
+ + "clear it from backend [{}]", transactionId, backendId);
+ } else if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
+ TableCommitInfo tableCommitInfo = transactionState.getTableCommitInfo(tabletMeta.getTableId());
+ PartitionCommitInfo partitionCommitInfo = tableCommitInfo == null ? null : tableCommitInfo.getPartitionCommitInfo(partitionId);
+ if (partitionCommitInfo != null) {
+ TPartitionVersionInfo versionInfo = new TPartitionVersionInfo(tabletMeta.getPartitionId(),
+ partitionCommitInfo.getVersion(), 0);
+ synchronized (transactionsToPublish) {
+ ListMultimap<Long, TPartitionVersionInfo> map = transactionsToPublish.get(transactionState.getDbId());
+ if (map == null) {
+ map = ArrayListMultimap.create();
+ transactionsToPublish.put(transactionState.getDbId(), map);
}
+ map.put(transactionId, versionInfo);
}
}
- } // end for txn id
-
- // update replicase's version count
- // no need to write log, and no need to get db lock.
- if (backendTabletInfo.isSetVersionCount()) {
- replica.setVersionCount(backendTabletInfo.getVersionCount());
}
- } else {
- // tablet with invalid schemahash
- foundTabletsWithInvalidSchema.put(tabletId, backendTabletInfo);
- } // end for be tablet info
+ }
+ } // end for txn id
+
+ // update replicase's version count
+ // no need to write log, and no need to get db lock.
+ if (backendTabletInfo.isSetVersionCount()) {
+ replica.setVersionCount(backendTabletInfo.getVersionCount());
}
} else {
// 2. (meta - be)
@@ -254,12 +247,11 @@ public class TabletInvertedIndex {
}
long end = System.currentTimeMillis();
- LOG.info("finished to do tablet diff with backend[{}]. sync: {}. metaDel: {}. foundValid: {}. foundInvalid: {}."
- + " migration: {}. found invalid transactions {}. found republish transactions {}. tabletInMemorySync: {}."
- + " need recovery: {}. cost: {} ms", backendId, tabletSyncMap.size(),
- tabletDeleteFromMeta.size(), foundTabletsWithValidSchema.size(), foundTabletsWithInvalidSchema.size(),
- tabletMigrationMap.size(), transactionsToClear.size(), transactionsToPublish.size(), tabletToInMemory.size(),
- tabletRecoveryMap.size(), (end - start));
+ LOG.info("finished to do tablet diff with backend[{}]. sync: {}. metaDel: {}. foundInMeta: {}. migration: {}. "
+ + "found invalid transactions {}. found republish transactions {}. tabletInMemorySync: {}."
+ + " need recovery: {}. cost: {} ms", backendId, tabletSyncMap.size(), tabletDeleteFromMeta.size(),
+ tabletFoundInMeta.size(), tabletMigrationMap.size(), transactionsToClear.size(), transactionsToPublish.size(),
+ tabletToInMemory.size(), tabletRecoveryMap.size(), (end - start));
}
public Long getTabletIdByReplica(long replicaId) {
@@ -466,47 +458,6 @@ public class TabletInvertedIndex {
}
}
- public void setNewSchemaHash(long partitionId, long indexId, int newSchemaHash) {
- if (Catalog.isCheckpointThread()) {
- return;
- }
- readLock();
- try {
- Preconditions.checkState(tabletMetaTable.contains(partitionId, indexId));
- tabletMetaTable.get(partitionId, indexId).setNewSchemaHash(newSchemaHash);
- } finally {
- readUnlock();
- }
- }
-
- public void updateToNewSchemaHash(long partitionId, long indexId) {
- if (Catalog.isCheckpointThread()) {
- return;
- }
- readLock();
- try {
- Preconditions.checkState(tabletMetaTable.contains(partitionId, indexId));
- tabletMetaTable.get(partitionId, indexId).updateToNewSchemaHash();
- } finally {
- readUnlock();
- }
- }
-
- public void deleteNewSchemaHash(long partitionId, long indexId) {
- if (Catalog.isCheckpointThread()) {
- return;
- }
- readLock();
- try {
- TabletMeta tabletMeta = tabletMetaTable.get(partitionId, indexId);
- if (tabletMeta != null) {
- tabletMeta.deleteNewSchemaHash();
- }
- } finally {
- readUnlock();
- }
- }
-
public List<Long> getTabletIdsByBackendId(long backendId) {
List<Long> tabletIds = Lists.newArrayList();
readLock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletMeta.java
index 0c695ca14d..2679977b6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletMeta.java
@@ -19,8 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.thrift.TStorageMedium;
-import com.google.common.base.Preconditions;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -80,40 +78,6 @@ public class TabletMeta {
this.storageMedium = storageMedium;
}
- public void setNewSchemaHash(int newSchemaHash) {
- lock.writeLock().lock();
- try {
- Preconditions.checkState(this.newSchemaHash == -1);
- this.newSchemaHash = newSchemaHash;
- LOG.debug("setNewSchemaHash: {}", toString());
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- public void updateToNewSchemaHash() {
- lock.writeLock().lock();
- try {
- Preconditions.checkState(this.newSchemaHash != -1);
- int tmp = this.oldSchemaHash;
- this.oldSchemaHash = this.newSchemaHash;
- this.newSchemaHash = tmp;
- LOG.debug("updateToNewSchemaHash: " + toString());
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- public void deleteNewSchemaHash() {
- lock.writeLock().lock();
- try {
- LOG.debug("deleteNewSchemaHash: " + toString());
- this.newSchemaHash = -1;
- } finally {
- lock.writeLock().unlock();
- }
- }
-
public int getOldSchemaHash() {
lock.readLock().lock();
try {
@@ -123,15 +87,6 @@ public class TabletMeta {
}
}
- public boolean containsSchemaHash(int schemaHash) {
- lock.readLock().lock();
- try {
- return this.oldSchemaHash == schemaHash || this.newSchemaHash == schemaHash;
- } finally {
- lock.readLock().unlock();
- }
- }
-
@Override
public String toString() {
lock.readLock().lock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index f0ae6a3ee2..b9d1202c5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -420,7 +420,6 @@ public class MasterImpl {
private void checkReplica(TTabletInfo tTabletInfo, TabletMeta tabletMeta)
throws MetaNotFoundException {
long tabletId = tTabletInfo.getTabletId();
- int schemaHash = tTabletInfo.getSchemaHash();
// during finishing stage, index's schema hash switched, when old schema hash finished
// current index hash != old schema hash and alter job's new schema hash != old schema hash
// the check replica will failed
@@ -430,11 +429,6 @@ public class MasterImpl {
// rollup may be dropped
throw new MetaNotFoundException("tablet " + tabletId + " does not exist");
}
- if (!tabletMeta.containsSchemaHash(schemaHash)) {
- throw new MetaNotFoundException("tablet[" + tabletId
- + "] schemaHash is not equal to index's switchSchemaHash. "
- + tabletMeta.toString()+ " vs. " + schemaHash);
- }
}
private Replica findRelatedReplica(OlapTable olapTable, Partition partition,
@@ -656,17 +650,6 @@ public class MasterImpl {
return null;
}
- int currentSchemaHash = olapTable.getSchemaHashByIndexId(pushIndexId);
- if (schemaHash != currentSchemaHash) {
- if (pushState == PartitionState.SCHEMA_CHANGE) {
- // Alter job is always null, so that not deal with it
- } else {
- // this should not happen. observe(cmy)
- throw new MetaNotFoundException("Diff tablet[" + tabletId + "] schemaHash. index[" + pushIndexId + "]: "
- + currentSchemaHash + " vs. " + schemaHash);
- }
- }
-
MaterializedIndex materializedIndex = partition.getIndex(pushIndexId);
if (materializedIndex == null) {
throw new MetaNotFoundException("Cannot find index[" + pushIndexId + "]");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 4cf6176cdf..62cff85f96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -18,6 +18,7 @@
package org.apache.doris.master;
+import com.google.common.collect.Sets;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.MaterializedIndex;
@@ -79,7 +80,6 @@ import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
-import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Triple;
@@ -257,10 +257,8 @@ public class ReportHandler extends Daemon {
ListMultimap<Long, Long> tabletSyncMap = LinkedListMultimap.create();
// db id -> tablet id
ListMultimap<Long, Long> tabletDeleteFromMeta = LinkedListMultimap.create();
- // tablet ids which schema hash is valid
- Set<Long> foundTabletsWithValidSchema = Sets.newConcurrentHashSet();
- // tablet ids which schema hash is invalid
- Map<Long, TTabletInfo> foundTabletsWithInvalidSchema = Maps.newConcurrentMap();
+ // tablet ids which both in fe and be
+ Set<Long> tabletFoundInMeta = Sets.newConcurrentHashSet();
// storage medium -> tablet id
ListMultimap<TStorageMedium, Long> tabletMigrationMap = LinkedListMultimap.create();
@@ -277,8 +275,7 @@ public class ReportHandler extends Daemon {
Catalog.getCurrentInvertedIndex().tabletReport(backendId, backendTablets, storageMediumMap,
tabletSyncMap,
tabletDeleteFromMeta,
- foundTabletsWithValidSchema,
- foundTabletsWithInvalidSchema,
+ tabletFoundInMeta,
tabletMigrationMap,
transactionsToPublish,
transactionsToClear,
@@ -297,8 +294,8 @@ public class ReportHandler extends Daemon {
}
// 4. handle (be - meta)
- if (foundTabletsWithValidSchema.size() != backendTablets.size()) {
- deleteFromBackend(backendTablets, foundTabletsWithValidSchema, foundTabletsWithInvalidSchema, backendId);
+ if (tabletFoundInMeta.size() != backendTablets.size()) {
+ deleteFromBackend(backendTablets, tabletFoundInMeta, backendId);
}
// 5. migration (ssd <-> hdd)
@@ -665,67 +662,42 @@ public class ReportHandler extends Daemon {
}
private static void deleteFromBackend(Map<Long, TTablet> backendTablets,
- Set<Long> foundTabletsWithValidSchema,
- Map<Long, TTabletInfo> foundTabletsWithInvalidSchema,
+ Set<Long> tabletFoundInMeta,
long backendId) {
int deleteFromBackendCounter = 0;
int addToMetaCounter = 0;
AgentBatchTask batchTask = new AgentBatchTask();
- // This means that the meta of all backend tablets can be found in fe, we only need to process tablets with invalid Schema
- if (foundTabletsWithValidSchema.size() + foundTabletsWithInvalidSchema.size() == backendTablets.size()) {
- for (Long tabletId : foundTabletsWithInvalidSchema.keySet()) {
- // this tablet is found in meta but with invalid schema hash. delete it.
- int schemaHash = foundTabletsWithInvalidSchema.get(tabletId).getSchemaHash();
- DropReplicaTask task = new DropReplicaTask(backendId, tabletId, schemaHash);
+ for (Long tabletId : backendTablets.keySet()) {
+ TTablet backendTablet = backendTablets.get(tabletId);
+ TTabletInfo backendTabletInfo = backendTablet.getTabletInfos().get(0);
+ boolean needDelete = false;
+ if (!tabletFoundInMeta.contains(tabletId)) {
+ if (isBackendReplicaHealthy(backendTabletInfo)) {
+ // if this tablet is not in meta. try adding it.
+ // if add failed. delete this tablet from backend.
+ try {
+ addReplica(tabletId, backendTabletInfo, backendId);
+ // update counter
+ needDelete = false;
+ ++addToMetaCounter;
+ } catch (MetaNotFoundException e) {
+ LOG.warn("failed add to meta. tablet[{}], backend[{}]. {}",
+ tabletId, backendId, e.getMessage());
+ needDelete = true;
+ }
+ } else {
+ needDelete = true;
+ }
+ }
+
+ if (needDelete) {
+ // drop replica
+ DropReplicaTask task = new DropReplicaTask(backendId, tabletId, backendTabletInfo.getSchemaHash());
batchTask.addTask(task);
- LOG.warn("delete tablet[" + tabletId + " - " + schemaHash + "] from backend[" + backendId
- + "] because invalid schema hash");
+ LOG.warn("delete tablet[" + tabletId + "] from backend[" + backendId + "] because not found in meta");
++deleteFromBackendCounter;
}
- } else {
- for (Long tabletId : backendTablets.keySet()) {
- if (foundTabletsWithInvalidSchema.containsKey(tabletId)) {
- int schemaHash = foundTabletsWithInvalidSchema.get(tabletId).getSchemaHash();
- DropReplicaTask task = new DropReplicaTask(backendId, tabletId, schemaHash);
- batchTask.addTask(task);
- LOG.warn("delete tablet[" + tabletId + " - " + schemaHash + "] from backend[" + backendId
- + "] because invalid schema hash");
- ++deleteFromBackendCounter;
- continue;
- }
- TTablet backendTablet = backendTablets.get(tabletId);
- for (TTabletInfo backendTabletInfo : backendTablet.getTabletInfos()) {
- boolean needDelete = false;
- if (!foundTabletsWithValidSchema.contains(tabletId)) {
- if (isBackendReplicaHealthy(backendTabletInfo)) {
- // if this tablet is not in meta. try adding it.
- // if add failed. delete this tablet from backend.
- try {
- addReplica(tabletId, backendTabletInfo, backendId);
- // update counter
- needDelete = false;
- ++addToMetaCounter;
- } catch (MetaNotFoundException e) {
- LOG.info("failed add to meta. tablet[{}], backend[{}]. {}",
- tabletId, backendId, e.getMessage());
- needDelete = true;
- }
- } else {
- needDelete = true;
- }
- }
-
- if (needDelete) {
- // drop replica
- DropReplicaTask task = new DropReplicaTask(backendId, tabletId, backendTabletInfo.getSchemaHash());
- batchTask.addTask(task);
- LOG.info("delete tablet[" + tabletId + " - " + backendTabletInfo.getSchemaHash()
- + "] from backend[" + backendId + "] because not found in meta");
- ++deleteFromBackendCounter;
- }
- } // end for tabletInfos
- } // end for backendTabletIds
- }
+ } // end for backendTabletIds
if (batchTask.getTaskNum() != 0) {
AgentTaskExecutor.submit(batchTask);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 0ba2e7a693..f74666a18c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -482,10 +482,7 @@ public class OlapScanNode extends ScanNode {
}
private void addScanRangeLocations(Partition partition,
- MaterializedIndex index,
List<Tablet> tablets) throws UserException {
- int schemaHash = olapTable.getSchemaHashByIndexId(index.getId());
- String schemaHashStr = String.valueOf(schemaHash);
long visibleVersion = partition.getVisibleVersion();
String visibleVersionStr = String.valueOf(visibleVersion);
@@ -500,13 +497,13 @@ public class OlapScanNode extends ScanNode {
TScanRangeLocations scanRangeLocations = new TScanRangeLocations();
TPaloScanRange paloRange = new TPaloScanRange();
paloRange.setDbName("");
- paloRange.setSchemaHash(schemaHashStr);
+ paloRange.setSchemaHash("");
paloRange.setVersion(visibleVersionStr);
paloRange.setVersionHash("");
paloRange.setTabletId(tabletId);
// random shuffle List && only collect one copy
- List<Replica> replicas = tablet.getQueryableReplicas(visibleVersion, schemaHash);
+ List<Replica> replicas = tablet.getQueryableReplicas(visibleVersion);
if (replicas.isEmpty()) {
LOG.error("no queryable replica found in tablet {}. visible version {}",
tabletId, visibleVersion);
@@ -670,7 +667,7 @@ public class OlapScanNode extends ScanNode {
totalTabletsNum += selectedTable.getTablets().size();
selectedTabletsNum += tablets.size();
- addScanRangeLocations(partition, selectedTable, tablets);
+ addScanRangeLocations(partition, tablets);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 92e68b353c..fd5d2cd4ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1422,7 +1422,7 @@ public class ShowExecutor {
}
dbName = db.getFullName();
Table table = db.getTableNullable(tableId);
- if (table == null || !(table instanceof OlapTable)) {
+ if (!(table instanceof OlapTable)) {
isSync = false;
break;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org