[accumulo] branch 1.10 updated: Handle infinite blocking loops blocking for tservers to register (#3235)

[ed...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.10 by this push:
     new 22a120f36f Handle infinite blocking loops blocking for tservers to register (#3235)
22a120f36f is described below

commit 22a120f36fd2b0adf8b53ae4c8936e96ac11ab0d
Author: Daniel Roberts <dd...@gmail.com>
AuthorDate: Mon Mar 13 09:51:55 2023 -0400

    Handle infinite blocking loops blocking for tservers to register (#3235)
    
    This change keeps the calculated timeouts for a defined maxWait loop while still allowing the infinite loop case to have an incremental backoff.
    
    * Handle infinite blocking loops
    * Change variable naming and collapse time units
    * Use static imports for TimeUnits
    * Log messages still show up in a semi-frequent manner to alert users of the continued blocking.
---
 .../java/org/apache/accumulo/master/Master.java    | 32 +++++++++++++++-------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9617032d82..7da6ed7f7c 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.master;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
 
 import java.io.IOException;
@@ -789,7 +790,7 @@ public class Master extends AccumuloServerContext
         return MasterGoalState.valueOf(new String(data));
       } catch (Exception e) {
         log.error("Problem getting real goal state from zookeeper: " + e);
-        sleepUninterruptibly(1_000, MILLISECONDS);
+        sleepUninterruptibly(1, SECONDS);
       }
   }
 
@@ -1490,20 +1491,31 @@ public class Master extends AccumuloServerContext
           tserverSet.size(), Property.MASTER_STARTUP_TSERVER_AVAIL_MIN_COUNT.getKey());
       return;
     }
-    long maxWait = accConfig.getTimeInMillis(Property.MASTER_STARTUP_TSERVER_AVAIL_MAX_WAIT);
+    long userWait = MILLISECONDS
+        .toSeconds(accConfig.getTimeInMillis(Property.MASTER_STARTUP_TSERVER_AVAIL_MAX_WAIT));
 
-    if (maxWait <= 0) {
+    // Setting retry values for defined wait timeouts
+    long retries = 10;
+    // Set these to the same value so the max possible wait time always matches the provided maxWait
+    long initialWait = userWait / retries;
+    long maxWaitPeriod = initialWait;
+    long waitIncrement = 0;
+
+    if (userWait <= 0) {
       log.info("tserver availability check set to block indefinitely, To change, set {} > 0.",
           Property.MASTER_STARTUP_TSERVER_AVAIL_MAX_WAIT.getKey());
-      maxWait = Long.MAX_VALUE;
-    }
+      userWait = Long.MAX_VALUE;
 
-    long retries = 10;
-    long waitPeriod = maxWait / retries;
+      // If indefinitely blocking, change retry values to support incremental backoff and logging.
+      retries = userWait;
+      initialWait = 1;
+      maxWaitPeriod = 30;
+      waitIncrement = 5;
+    }
 
-    Retry tserverRetry = Retry.builder().maxRetries(retries).retryAfter(waitPeriod, MILLISECONDS)
-        .incrementBy(0, MILLISECONDS).maxWait(waitPeriod, MILLISECONDS)
-        .logInterval(30_000, MILLISECONDS).createRetry();
+    Retry tserverRetry = Retry.builder().maxRetries(retries).retryAfter(initialWait, SECONDS)
+        .incrementBy(waitIncrement, SECONDS).maxWait(maxWaitPeriod, SECONDS)
+        .logInterval(30, SECONDS).createRetry();
 
     log.info("Checking for tserver availability - need to reach {} servers. Have {}",
         minTserverCount, tserverSet.size());