You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/09/02 17:43:27 UTC
[1/2] hbase git commit: HBASE-16399 Provide an API to get list of
failed regions and servername in Canary (Vishal Khandelwal)
Repository: hbase
Updated Branches:
refs/heads/0.98 51a5fe850 -> 0ac78f440
refs/heads/master 0b6eccf4c -> 5e905c02d
HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e905c02
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e905c02
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e905c02
Branch: refs/heads/master
Commit: 5e905c02d9ddbedc06cb53e94c8c873aa8032145
Parents: 0b6eccf
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Sep 2 10:13:35 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Sep 2 10:13:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/tool/Canary.java | 68 +++++++++++++++++---
.../hadoop/hbase/tool/TestCanaryTool.java | 12 ++--
2 files changed, 67 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e905c02/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index bbf858d..10bc139 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -39,6 +39,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -115,11 +116,15 @@ public final class Canary implements Tool {
public long incReadFailureCount();
public void publishReadFailure(HRegionInfo region, Exception e);
public void publishReadFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
+ public void updateReadFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getReadFailures();
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
public long getWriteFailureCount();
public void publishWriteFailure(HRegionInfo region, Exception e);
public void publishWriteFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
public void publishWriteTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getWriteFailures();
}
// new extended sink for output regionserver mode info
// do not change the Sink interface directly due to maintaining the API
@@ -134,6 +139,9 @@ public final class Canary implements Tool {
private AtomicLong readFailureCount = new AtomicLong(0),
writeFailureCount = new AtomicLong(0);
+ private Map<String, String> readFailures = new ConcurrentHashMap<String, String>();
+ private Map<String, String> writeFailures = new ConcurrentHashMap<String, String>();
+
@Override
public long getReadFailureCount() {
return readFailureCount.get();
@@ -158,9 +166,24 @@ public final class Canary implements Tool {
}
@Override
+ public void updateReadFailedHostList(HRegionInfo region, String serverName) {
+ readFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
+ @Override
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) {
LOG.info(String.format("read from region %s column family %s in %dms",
- region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ }
+
+ @Override
+ public Map<String, String> getReadFailures() {
+ return readFailures;
+ }
+
+ @Override
+ public Map<String, String> getWriteFailures() {
+ return writeFailures;
}
@Override
@@ -186,6 +209,12 @@ public final class Canary implements Tool {
LOG.info(String.format("write to region %s column family %s in %dms",
region.getRegionNameAsString(), column.getNameAsString(), msTime));
}
+
+ @Override
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName) {
+ writeFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
}
// a ExtendedSink implementation
public static class RegionServerStdOutSink extends StdOutSink implements ExtendedSink {
@@ -265,11 +294,13 @@ public final class Canary implements Tool {
private Sink sink;
private TaskType taskType;
private boolean rawScanEnabled;
+ private ServerName serverName;
- RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType taskType,
- boolean rawScanEnabled) {
+ RegionTask(Connection connection, HRegionInfo region, ServerName serverName, Sink sink,
+ TaskType taskType, boolean rawScanEnabled) {
this.connection = connection;
this.region = region;
+ this.serverName = serverName;
this.sink = sink;
this.taskType = taskType;
this.rawScanEnabled = rawScanEnabled;
@@ -356,6 +387,7 @@ public final class Canary implements Tool {
sink.publishReadTiming(region, column, stopWatch.getTime());
} catch (Exception e) {
sink.publishReadFailure(region, column, e);
+ sink.updateReadFailedHostList(region, serverName.getHostname());
} finally {
if (rs != null) {
rs.close();
@@ -412,6 +444,7 @@ public final class Canary implements Tool {
table.close();
} catch (IOException e) {
sink.publishWriteFailure(region, e);
+ sink.updateWriteFailedHostList(region, serverName.getHostname());
}
return null;
}
@@ -733,6 +766,14 @@ public final class Canary implements Tool {
return monitor.errorCode;
}
+ public Map<String, String> getReadFailures() {
+ return sink.getReadFailures();
+ }
+
+ public Map<String, String> getWriteFailures() {
+ return sink.getWriteFailures();
+ }
+
private void printUsageAndExit() {
System.err.printf(
"Usage: bin/hbase %s [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]%n",
@@ -1143,16 +1184,25 @@ public final class Canary implements Tool {
} catch (TableNotFoundException e) {
return new ArrayList<Future<Void>>();
}
+ finally {
+ if (table !=null) {
+ table.close();
+ }
+ }
+
List<RegionTask> tasks = new ArrayList<RegionTask>();
+ RegionLocator regionLocator = null;
try {
- List<HRegionInfo> hris = admin.getTableRegions(tableDesc.getTableName());
- if (hris != null) {
- for (HRegionInfo region : hris) {
- tasks.add(new RegionTask(admin.getConnection(), region, sink, taskType, rawScanEnabled));
- }
+ regionLocator = admin.getConnection().getRegionLocator(tableDesc.getTableName());
+ for (HRegionLocation location : regionLocator.getAllRegionLocations()) {
+ ServerName rs = location.getServerName();
+ HRegionInfo region = location.getRegionInfo();
+ tasks.add(new RegionTask(admin.getConnection(), region, rs, sink, taskType, rawScanEnabled));
}
} finally {
- table.close();
+ if (regionLocator != null) {
+ regionLocator.close();
+ }
}
return executor.invokeAll(tasks);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e905c02/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index fb33d7e..c1d6e20 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -53,6 +53,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
+import static org.junit.Assert.assertEquals;
@RunWith(MockitoJUnitRunner.class)
@Category({MediumTests.class})
@@ -110,10 +111,11 @@ public class TestCanaryTool {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink());
Canary canary = new Canary(executor, sink);
- String[] args = { "-t", "10000", "testTable" };
+ String[] args = { "-writeSniffing", "-t", "10000", "testTable" };
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
- verify(sink, atLeastOnce())
- .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
+ assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
+ verify(sink, atLeastOnce()).publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
}
//no table created, so there should be no regions
@@ -162,13 +164,15 @@ public class TestCanaryTool {
ToolRunner.run(conf, canary, args);
verify(sink, atLeastOnce())
.publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
-
+
private void runRegionserverCanary() throws Exception {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink());
String[] args = { "-t", "10000", "-regionserver"};
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
}
[2/2] hbase git commit: HBASE-16399 Provide an API to get list of
failed regions and servername in Canary (Vishal Khandelwal)
Posted by ap...@apache.org.
HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ac78f44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ac78f44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ac78f44
Branch: refs/heads/0.98
Commit: 0ac78f440cdfc233711a5eb8a3db58db1a837d37
Parents: 51a5fe8
Author: Andrew Purtell <ap...@apache.org>
Authored: Fri Sep 2 10:13:35 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Fri Sep 2 10:37:22 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/tool/Canary.java | 53 ++++++++++++++++++--
.../hadoop/hbase/tool/TestCanaryTool.java | 12 +++--
2 files changed, 56 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ac78f44/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index 51f162f..d887e4d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -40,6 +40,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -113,11 +114,15 @@ public final class Canary implements Tool {
public long incReadFailureCount();
public void publishReadFailure(HRegionInfo region, Exception e);
public void publishReadFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
+ public void updateReadFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getReadFailures();
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
public long getWriteFailureCount();
public void publishWriteFailure(HRegionInfo region, Exception e);
public void publishWriteFailure(HRegionInfo region, HColumnDescriptor column, Exception e);
public void publishWriteTiming(HRegionInfo region, HColumnDescriptor column, long msTime);
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName);
+ public Map<String,String> getWriteFailures();
}
// new extended sink for output regionserver mode info
// do not change the Sink interface directly due to maintaining the API
@@ -132,6 +137,9 @@ public final class Canary implements Tool {
private AtomicLong readFailureCount = new AtomicLong(0),
writeFailureCount = new AtomicLong(0);
+ private Map<String, String> readFailures = new ConcurrentHashMap<String, String>();
+ private Map<String, String> writeFailures = new ConcurrentHashMap<String, String>();
+
@Override
public long getReadFailureCount() {
return readFailureCount.get();
@@ -156,9 +164,24 @@ public final class Canary implements Tool {
}
@Override
+ public void updateReadFailedHostList(HRegionInfo region, String serverName) {
+ readFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
+ @Override
public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) {
LOG.info(String.format("read from region %s column family %s in %dms",
- region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ region.getRegionNameAsString(), column.getNameAsString(), msTime));
+ }
+
+ @Override
+ public Map<String, String> getReadFailures() {
+ return readFailures;
+ }
+
+ @Override
+ public Map<String, String> getWriteFailures() {
+ return writeFailures;
}
@Override
@@ -184,6 +207,12 @@ public final class Canary implements Tool {
LOG.info(String.format("write to region %s column family %s in %dms",
region.getRegionNameAsString(), column.getNameAsString(), msTime));
}
+
+ @Override
+ public void updateWriteFailedHostList(HRegionInfo region, String serverName) {
+ writeFailures.put(region.getRegionNameAsString(), serverName);
+ }
+
}
// a ExtendedSink implementation
public static class RegionServerStdOutSink extends StdOutSink implements ExtendedSink {
@@ -265,14 +294,16 @@ public final class Canary implements Tool {
private Sink sink;
private TaskType taskType;
private boolean rawScanEnabled;
+ private ServerName serverName;
- RegionTask(HConnection connection, HRegionInfo region, Sink sink, TaskType taskType,
- boolean rawScanEnabled) {
+ RegionTask(HConnection connection, HRegionInfo region, ServerName serverName, Sink sink,
+ TaskType taskType, boolean rawScanEnabled) {
this.connection = connection;
this.region = region;
this.sink = sink;
this.taskType = taskType;
this.rawScanEnabled = rawScanEnabled;
+ this.serverName = serverName;
}
@Override
@@ -356,6 +387,7 @@ public final class Canary implements Tool {
sink.publishReadTiming(region, column, stopWatch.getTime());
} catch (Exception e) {
sink.publishReadFailure(region, column, e);
+ sink.updateReadFailedHostList(region, serverName.getHostname());
} finally {
if (rs != null) {
rs.close();
@@ -412,6 +444,7 @@ public final class Canary implements Tool {
table.close();
} catch (IOException e) {
sink.publishWriteFailure(region, e);
+ sink.updateWriteFailedHostList(region, serverName.getHostname());
}
return null;
}
@@ -483,9 +516,11 @@ public final class Canary implements Tool {
LOG.debug("The targeted table was disabled. Assuming success.");
} catch (DoNotRetryIOException dnrioe) {
sink.publishReadFailure(tableName.getNameAsString(), serverName);
+ sink.updateReadFailedHostList(region, serverName);
LOG.error(dnrioe);
} catch (IOException e) {
sink.publishReadFailure(tableName.getNameAsString(), serverName);
+ sink.updateReadFailedHostList(region, serverName);
LOG.error(e);
} finally {
if (table != null) {
@@ -720,6 +755,14 @@ public final class Canary implements Tool {
return monitor.errorCode;
}
+ public Map<String, String> getReadFailures() {
+ return sink.getReadFailures();
+ }
+
+ public Map<String, String> getWriteFailures() {
+ return sink.getWriteFailures();
+ }
+
private void printUsageAndExit() {
System.err.printf(
"Usage: bin/hbase %s [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]%n",
@@ -1103,8 +1146,8 @@ public final class Canary implements Tool {
}
List<RegionTask> tasks = new ArrayList<RegionTask>();
try {
- for (HRegionInfo region : ((HTable)table).getRegionLocations().keySet()) {
- tasks.add(new RegionTask(connection, region, sink, taskType, rawScanEnabled));
+ for (Map.Entry<HRegionInfo, ServerName> region : ((HTable)table).getRegionLocations().entrySet()) {
+ tasks.add(new RegionTask(connection, region.getKey(), region.getValue(), sink, taskType, rawScanEnabled));
}
} finally {
table.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ac78f44/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index 9d1dd9e..877eb66 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -52,6 +52,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
+import static org.junit.Assert.assertEquals;
@RunWith(MockitoJUnitRunner.class)
@Category({MediumTests.class})
@@ -108,10 +109,11 @@ public class TestCanaryTool {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink());
Canary canary = new Canary(executor, sink);
- String[] args = { "-t", "10000", "testTable" };
+ String[] args = { "-writeSniffing", "-t", "10000", "testTable" };
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
- verify(sink, atLeastOnce())
- .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
+ assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
+ verify(sink, atLeastOnce()).publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
}
//no table created, so there should be no regions
@@ -160,13 +162,15 @@ public class TestCanaryTool {
ToolRunner.run(conf, canary, args);
verify(sink, atLeastOnce())
.publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong());
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
-
+
private void runRegionserverCanary() throws Exception {
ExecutorService executor = new ScheduledThreadPoolExecutor(1);
Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink());
String[] args = { "-t", "10000", "-regionserver"};
ToolRunner.run(testingUtility.getConfiguration(), canary, args);
+ assertEquals("verify no read error count", 0, canary.getReadFailures().size());
}
}