[hbase] branch branch-2.4 updated: HBASE-27491 Do not clear cache on RejectedExecutionException (#4914)

[bb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

bbeaudreault pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
     new f1bfda1aa3a HBASE-27491 Do not clear cache on RejectedExecutionException (#4914)
f1bfda1aa3a is described below

commit f1bfda1aa3a99384f4ebc787d50dcb1d659219aa
Author: Bri Augenreich <bb...@vt.edu>
AuthorDate: Fri Dec 9 08:45:40 2022 -0500

    HBASE-27491 Do not clear cache on RejectedExecutionException (#4914)
    
    Signed-off-by: Duo Zhang <zh...@apache.org>
    Signed-off-by: Bryan Beaudreault <bb...@apache.org>
---
 .../hbase/client/AsyncRequestFutureImpl.java       | 26 +++++++++++++++-------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java
index 78d6748498b..12e6a61d29f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java
@@ -219,13 +219,13 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
         } catch (IOException e) {
           // The service itself failed . It may be an error coming from the communication
           // layer, but, as well, a functional error raised by the server.
-          receiveGlobalFailure(multiAction, server, numAttempt, e);
+          receiveGlobalFailure(multiAction, server, numAttempt, e, true);
           return;
         } catch (Throwable t) {
           // This should not happen. Let's log & retry anyway.
           LOG.error("id=" + asyncProcess.id + ", caught throwable. Unexpected."
             + " Retrying. Server=" + server + ", tableName=" + tableName, t);
-          receiveGlobalFailure(multiAction, server, numAttempt, t);
+          receiveGlobalFailure(multiAction, server, numAttempt, t, true);
           return;
         }
         if (res.type() == AbstractResponse.ResponseType.MULTI) {
@@ -521,6 +521,7 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
    */
   void sendMultiAction(Map<ServerName, MultiAction> actionsByServer, int numAttempt,
     List<Action> actionsForReplicaThread, boolean reuseThread) {
+    boolean clearServerCache = true;
     // Run the last item on the same thread if we are already on a send thread.
     // We hope most of the time it will be the only item, so we can cut down on threads.
     int actionsRemaining = actionsByServer.size();
@@ -555,6 +556,8 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
               // let's secure this a little.
               LOG.warn("id=" + asyncProcess.id + ", task rejected by pool. Unexpected." + " Server="
                 + server.getServerName(), t);
+              // Do not update cache if exception is from failing to submit action to thread pool
+              clearServerCache = false;
             } else {
               // see #HBASE-14359 for more details
               LOG.warn("Caught unexpected exception/error: ", t);
@@ -562,7 +565,7 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
             asyncProcess.decTaskCounters(multiAction.getRegions(), server);
             // We're likely to fail again, but this will increment the attempt counter,
             // so it will finish.
-            receiveGlobalFailure(multiAction, server, numAttempt, t);
+            receiveGlobalFailure(multiAction, server, numAttempt, t, clearServerCache);
           }
         }
       }
@@ -717,11 +720,15 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
    * @param t          the throwable (if any) that caused the resubmit
    */
   private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int numAttempt,
-    Throwable t) {
+    Throwable t, boolean clearServerCache) {
     errorsByServer.reportServerError(server);
     Retry canRetry = errorsByServer.canTryMore(numAttempt) ? Retry.YES : Retry.NO_RETRIES_EXHAUSTED;
 
-    cleanServerCache(server, t);
+    // Do not update cache if exception is from failing to submit action to thread pool
+    if (clearServerCache) {
+      cleanServerCache(server, t);
+    }
+
     int failed = 0;
     int stopped = 0;
     List<Action> toReplay = new ArrayList<>();
@@ -729,9 +736,12 @@ class AsyncRequestFutureImpl<CResult> implements AsyncRequestFuture {
       byte[] regionName = e.getKey();
       byte[] row = e.getValue().get(0).getAction().getRow();
       // Do not use the exception for updating cache because it might be coming from
-      // any of the regions in the MultiAction.
-      updateCachedLocations(server, regionName, row,
-        ClientExceptionsUtil.isMetaClearingException(t) ? null : t);
+      // any of the regions in the MultiAction and do not update cache if exception is
+      // from failing to submit action to thread pool
+      if (clearServerCache) {
+        updateCachedLocations(server, regionName, row,
+          ClientExceptionsUtil.isMetaClearingException(t) ? null : t);
+      }
       for (Action action : e.getValue()) {
         Retry retry =
           manageError(action.getOriginalIndex(), action.getAction(), canRetry, t, server);