You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2024/06/19 19:23:30 UTC
(hbase) branch branch-2.5 updated: HBASE-28663 Graceful shutdown of CanaryTool timeouts (#5991)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new da5cfc4c692 HBASE-28663 Graceful shutdown of CanaryTool timeouts (#5991)
da5cfc4c692 is described below
commit da5cfc4c69244814c433ad36f981364ff81c20a7
Author: d-c-manning <67...@users.noreply.github.com>
AuthorDate: Wed Jun 19 12:21:18 2024 -0700
HBASE-28663 Graceful shutdown of CanaryTool timeouts (#5991)
Signed-off-by: Viraj Jasani <vj...@apache.org>
Signed-off-by: Mihir Monani <mm...@salesforce.com>
---
.../org/apache/hadoop/hbase/tool/CanaryTool.java | 41 ++++++++++++++++--
.../apache/hadoop/hbase/tool/TestCanaryTool.java | 48 ++++++++++++++++++++++
2 files changed, 86 insertions(+), 3 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java
index a54bed8935e..979fdeb0bb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java
@@ -198,6 +198,10 @@ public class CanaryTool implements Tool, Canary {
long getWriteSuccessCount();
long incWriteSuccessCount();
+
+ void stop();
+
+ boolean isStopped();
}
/**
@@ -208,6 +212,7 @@ public class CanaryTool implements Tool, Canary {
readSuccessCount = new AtomicLong(0), writeSuccessCount = new AtomicLong(0);
private Map<String, String> readFailures = new ConcurrentHashMap<>();
private Map<String, String> writeFailures = new ConcurrentHashMap<>();
+ private volatile boolean stopped = false;
@Override
public long getReadFailureCount() {
@@ -268,6 +273,15 @@ public class CanaryTool implements Tool, Canary {
public long incWriteSuccessCount() {
return writeSuccessCount.incrementAndGet();
}
+
+ public void stop() {
+ stopped = true;
+ }
+
+ @Override
+ public boolean isStopped() {
+ return stopped;
+ }
}
/**
@@ -444,6 +458,9 @@ public class CanaryTool implements Tool, Canary {
@Override
public Void call() throws Exception {
+ if (this.sink.isStopped()) {
+ return null;
+ }
ZooKeeper zooKeeper = null;
try {
zooKeeper = new ZooKeeper(host, timeout, EmptyWatcher.instance);
@@ -498,6 +515,9 @@ public class CanaryTool implements Tool, Canary {
@Override
public Void call() {
+ if (this.sink.isStopped()) {
+ return null;
+ }
switch (taskType) {
case READ:
return read();
@@ -685,6 +705,9 @@ public class CanaryTool implements Tool, Canary {
@Override
public Void call() {
+ if (this.sink.isStopped()) {
+ return null;
+ }
TableName tableName = null;
Table table = null;
Get get = null;
@@ -1075,6 +1098,7 @@ public class CanaryTool implements Tool, Canary {
if (currentTimeLength > timeout) {
LOG.error("The monitor is running too long (" + currentTimeLength
+ ") after timeout limit:" + timeout + " will be killed itself !!");
+ monitorThread.interrupt();
if (monitor.initialized) {
return TIMEOUT_ERROR_EXIT_CODE;
} else {
@@ -1113,6 +1137,15 @@ public class CanaryTool implements Tool, Canary {
return sink.getWriteFailures();
}
+ /**
+ * Return a CanaryTool.Sink object containing the detailed results of the canary run. The Sink may
+ * not have been created if a Monitor thread is not yet running.
+ * @return the active Sink if one exists, null otherwise.
+ */
+ public Sink getActiveSink() {
+ return sink;
+ }
+
private void printUsageAndExit() {
System.err.println(
"Usage: canary [OPTIONS] [<TABLE1> [<TABLE2]...] | [<REGIONSERVER1> [<REGIONSERVER2]..]");
@@ -1159,10 +1192,11 @@ public class CanaryTool implements Tool, Canary {
Sink getSink(Configuration configuration, Class clazz) {
// In test context, this.sink might be set. Use it if non-null. For testing.
- return this.sink != null
- ? this.sink
- : (Sink) ReflectionUtils
+ if (this.sink == null) {
+ this.sink = (Sink) ReflectionUtils
.newInstance(configuration.getClass("hbase.canary.sink.class", clazz, Sink.class));
+ }
+ return this.sink;
}
/**
@@ -1366,6 +1400,7 @@ public class CanaryTool implements Tool, Canary {
@Override
public void close() throws IOException {
+ this.sink.stop();
if (this.admin != null) {
this.admin.close();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index dc68ee59982..2c22a0b1104 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.tool;
+import static org.apache.hadoop.hbase.regionserver.TestRegionServerNoMaster.closeRegion;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -38,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.ToolRunner;
@@ -130,6 +133,51 @@ public class TestCanaryTool {
isA(ColumnFamilyDescriptor.class), anyLong());
}
+ /**
+ * When CanaryTool times out, it should stop scanning and shutdown quickly and gracefully. This
+ * test helps to confirm that threadpools do not continue executing work after the canary
+ * finishes. It also verifies sink behavior and measures correct failure counts in the sink.
+ * @throws Exception if it can't create a table, communicate with minicluster, or run the canary.
+ */
+ @Test
+ public void testCanaryStopsScanningAfterTimeout() throws Exception {
+ // Prepare a table with multiple regions, and close those regions on the regionserver.
+ // Do not notify HMaster or META. CanaryTool will scan and receive NotServingRegionExceptions.
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ // Close the unused Table reference returned by createMultiRegionTable.
+ testingUtility.createMultiRegionTable(tableName, new byte[][] { FAMILY }).close();
+ List<RegionInfo> regions = testingUtility.getAdmin().getRegions(tableName);
+ assertTrue("verify table has multiple regions", regions.size() > 1);
+ HRegionServer regionserver = testingUtility.getMiniHBaseCluster().getRegionServer(0);
+ for (RegionInfo region : regions) {
+ closeRegion(testingUtility, regionserver, region);
+ }
+
+ // Run CanaryTool with 1 thread. This thread will attempt to scan the first region.
+ // It will use default rpc retries and receive NotServingRegionExceptions for many seconds
+ // according to HConstants.RETRY_BACKOFF. The CanaryTool timeout is set to 4 seconds, so it
+ // will time out before the first region scan is complete.
+ ExecutorService executor = new ScheduledThreadPoolExecutor(1);
+ CanaryTool canary = new CanaryTool(executor);
+ String[] args = { "-t", "4000", tableName.getNameAsString() };
+ int retCode = ToolRunner.run(testingUtility.getConfiguration(), canary, args);
+ executor.shutdown();
+ try {
+ if (!executor.awaitTermination(3, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ executor.shutdownNow();
+ }
+
+ CanaryTool.Sink sink = canary.getActiveSink();
+ assertEquals("verify canary timed out with TIMEOUT_ERROR_EXIT_CODE", 3, retCode);
+ assertEquals("verify only the first region failed", 1, sink.getReadFailureCount());
+ assertEquals("verify no successful reads", 0, sink.getReadSuccessCount());
+ assertEquals("verify we were attempting to scan all regions", regions.size(),
+ ((CanaryTool.RegionStdOutSink) sink).getTotalExpectedRegions());
+ }
+
@Test
public void testCanaryRegionTaskReadAllCF() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());