You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/09/02 17:43:31 UTC
hbase git commit: HBASE-16399 Provide an API to get list of failed
regions and servername in Canary (Vishal Khandelwal)
Repository: hbase
Updated Branches:
refs/heads/branch-1 8be6f95f9 -> 8691520e3
HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8691520e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8691520e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8691520e
Branch: refs/heads/branch-1
Commit: 8691520e335aa9a2001c21aac5248bf7daed2139
Parents: 8be6f95
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Sep 2 10:13:35 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Sep 2 10:43:20 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/tool/Canary.java | 64 ++++++++++++++++++--
.../hadoop/hbase/tool/TestCanaryTool.java | 12 ++--
2 files changed, 66 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8691520e/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index 1e27dcb..4e87c8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -39,6 +39,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -113,11 +114,15 @@ public final class Canary implements Tool {
public long incReadFailureCount();
public void publishReadFailure(HRegionInfo region, Exception e);
public void publishReadFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
+ public void updateReadFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getReadFailures();
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
public long getWriteFailureCount();
public void publishWriteFailure(HRegionInfo region, Exception e);
public void publishWriteFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
public void publishWriteTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getWriteFailures();
}
// new extended sink for output regionserver mode info
// do not change the Sink interface directly due to maintaining the API
@@ -132,6 +137,9 @@ public final class Canary implements Tool {
private AtomicLong readFailureCount = new AtomicLong(0),
writeFailureCount = new AtomicLong(0);
+ private Map<String, String> readFailures = new ConcurrentHashMap<String, String>();
+ private Map<String, String> writeFailures = new ConcurrentHashMap<String, String>();
+
@Override
public long getReadFailureCount() {
return readFailureCount.get();
@@ -156,9 +164,24 @@ public final class Canary implements Tool {
}
@Override
+ public void updateReadFailedHostList(HRegionInfo region, String serverName) {
+ readFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
+ @Override
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) {
LOG.info(String.format("read from region %s column family %s in %dms",
- region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ }
+
+ @Override
+ public Map<String, String> getReadFailures() {
+ return readFailures;
+ }
+
+ @Override
+ public Map<String, String> getWriteFailures() {
+ return writeFailures;
}
@Override
@@ -184,6 +207,12 @@ public final class Canary implements Tool {
LOG.info(String.format("write to region %s column family %s in %dms",
region.getRegionNameAsString(), column.getNameAsString(), msTime));
}
+
+ @Override
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName) {
+ writeFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
}
// a ExtendedSink implementation
public static class RegionServerStdOutSink extends StdOutSink implements ExtendedSink {
@@ -263,11 +292,13 @@ public final class Canary implements Tool {
private Sink sink;
private TaskType taskType;
private boolean rawScanEnabled;
+ private ServerName serverName;
- RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType taskType,
- boolean rawScanEnabled) {
+ RegionTask(Connection connection, HRegionInfo region, ServerName serverName, Sink sink,
+ TaskType taskType, boolean rawScanEnabled) {
this.connection = connection;
this.region = region;
+ this.serverName = serverName;
this.sink = sink;
this.taskType = taskType;
this.rawScanEnabled = rawScanEnabled;
@@ -354,6 +385,7 @@ public final class Canary implements Tool {
sink.publishReadTiming(region, column, stopWatch.getTime());
} catch (Exception e) {
sink.publishReadFailure(region, column, e);
+ sink.updateReadFailedHostList(region, serverName.getHostname());
} finally {
if (rs != null) {
rs.close();
@@ -410,6 +442,7 @@ public final class Canary implements Tool {
table.close();
} catch (IOException e) {
sink.publishWriteFailure(region, e);
+ sink.updateWriteFailedHostList(region, serverName.getHostname());
}
return null;
}
@@ -731,6 +764,14 @@ public final class Canary implements Tool {
return monitor.errorCode;
}
+ public Map<String, String> getReadFailures() {
+ return sink.getReadFailures();
+ }
+
+ public Map<String, String> getWriteFailures() {
+ return sink.getWriteFailures();
+ }
+
private void printUsageAndExit() {
System.err.printf(
"Usage: bin/hbase %s [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]%n",
@@ -1145,14 +1186,25 @@ public final class Canary implements Tool {
table = admin.getConnection().getTable(tableDesc.getTableName());
} catch (TableNotFoundException e) {
return new ArrayList<Future<Void>>();
+ } finally {
+ if (table != null) {
+ table.close();
+ }
}
+
List<RegionTask> tasks = new ArrayList<RegionTask>();
+ RegionLocator regionLocator = null;
try {
- for (HRegionInfo region : admin.getTableRegions(tableDesc.getTableName())) {
- tasks.add(new RegionTask(admin.getConnection(), region, sink, taskType, rawScanEnabled));
+ regionLocator = admin.getConnection().getRegionLocator(tableDesc.getTableName());
+ for (HRegionLocation location : regionLocator.getAllRegionLocations()) {
+ ServerName rs = location.getServerName();
+ HRegionInfo region = location.getRegionInfo();
+ tasks.add(new RegionTask(admin.getConnection(), region, rs, sink, taskType, rawScanEnabled));
}
} finally {
- table.close();
+ if (regionLocator != null) {
+ regionLocator.close();
+ }
}
return executor.invokeAll(tasks);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8691520e/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index b5b0d37..f910668 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -54,6 +54,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
+import static org.junit.Assert.assertEquals;
@RunWith(MockitoJUnitRunner.class)
@Category({MediumTests.class})
@@ -111,10 +112,11 @@ public class TestCanaryTool {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink());
Canary canary = new Canary(executor, sink);
- String[] args = { "-t", "10000", "testTable" };
+ String[] args = { "-writeSniffing", "-t", "10000", "testTable" };
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
- verify(sink, atLeastOnce())
- .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
+ assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
+ verify(sink, atLeastOnce()).publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
}
//no table created, so there should be no regions
@@ -163,13 +165,15 @@ public class TestCanaryTool {
ToolRunner.run(conf, canary, args);
verify(sink, atLeastOnce())
.publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
-
+
private void runRegionserverCanary() throws Exception {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink());
String[] args = { "-t", "10000", "-regionserver"};
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
}