This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 188f664359d7ccd804c36ba6189d6e6e941158af
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Thu Jun 8 13:43:10 2023 +0800
[Fix](hive-catalog) Fallback to refresh catalog when hms events are missing (#20227)
This error can not be recovered (the relevant events in hms may have been deleted and can not recovered), so we need a fallback.
---
.../org/apache/doris/datasource/HMSExternalCatalog.java | 16 +++++++++++++++-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 2 +-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
index 531c626e3d..4d6c8b86fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
@@ -35,6 +35,7 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.security.UserGroupInformation;
@@ -215,7 +216,20 @@ public class HMSExternalCatalog extends ExternalCatalog {
LOG.info("Event id not updated when pulling events on catalog [{}]", hmsExternalCatalog.getName());
return null;
}
- return client.getNextNotification(lastSyncedEventId, Config.hms_events_batch_size_per_rpc, null);
+
+ try {
+ return client.getNextNotification(lastSyncedEventId, Config.hms_events_batch_size_per_rpc, null);
+ } catch (IllegalStateException e) {
+ // Need a fallback to handle this because this error state can not be recovered until restarting FE
+ if (HiveMetaStoreClient.REPL_EVENTS_MISSING_IN_METASTORE.equals(e.getMessage())) {
+ lastSyncedEventId = getCurrentEventId();
+ refreshCatalog(hmsExternalCatalog);
+ LOG.warn("Notification events are missing, maybe an event can not be handled "
+ + "or processing rate is too low, fallback to refresh the catalog");
+ return null;
+ }
+ throw e;
+ }
}
private void refreshCatalog(HMSExternalCatalog hmsExternalCatalog) {
diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index f3568a98a7..21ade60d07 100644
--- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -320,7 +320,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
static final protected Logger LOG = LoggerFactory.getLogger(HiveMetaStoreClient.class);
//copied from ErrorMsg.java
- private static final String REPL_EVENTS_MISSING_IN_METASTORE = "Notification events are missing in the meta store.";
+ public static final String REPL_EVENTS_MISSING_IN_METASTORE = "Notification events are missing in the meta store.";
public HiveMetaStoreClient(Configuration conf) throws MetaException {
this(conf, null, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org