You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/06/07 23:06:54 UTC
[6/8] hbase git commit: HBASE-15727 Canary Tool for Zookeeper (churro
morales)
HBASE-15727 Canary Tool for Zookeeper (churro morales)
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/60958f59
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/60958f59
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/60958f59
Branch: refs/heads/0.98
Commit: 60958f591258fabca627ec087286112793fe6dbf
Parents: fee078d
Author: tedyu <yu...@gmail.com>
Authored: Thu Jun 2 10:17:42 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Jun 7 15:10:58 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/tool/Canary.java | 145 ++++++++++++++++++-
.../hadoop/hbase/tool/TestCanaryTool.java | 19 +++
2 files changed, 162 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/60958f59/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
index cfb9860..4f2e5fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java
@@ -19,8 +19,14 @@
package org.apache.hadoop.hbase.tool;
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
+
+import com.google.common.collect.Lists;
+
import java.io.Closeable;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -74,9 +80,15 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.util.RegionSplitter;
+import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.client.ConnectStringParser;
+import org.apache.zookeeper.data.Stat;
import com.google.protobuf.ServiceException;
@@ -84,12 +96,15 @@ import com.google.protobuf.ServiceException;
* HBase Canary Tool, that that can be used to do
* "canary monitoring" of a running HBase cluster.
*
- * Here are two modes
+ * Here are three modes
* 1. region mode - Foreach region tries to get one row per column family
* and outputs some information about failure or latency.
*
* 2. regionserver mode - Foreach regionserver tries to get one row from one table
* selected randomly and outputs some information about failure or latency.
+ *
+ * 3. zookeeper mode - for each zookeeper instance, selects a zNode and
+ * outputs some information about failure or latency.
*/
public final class Canary implements Tool {
// Sink interface used by the canary to outputs information
@@ -186,6 +201,57 @@ public final class Canary implements Tool {
}
}
+ public static class ZookeeperStdOutSink extends StdOutSink implements ExtendedSink {
+ @Override public void publishReadFailure(String zNode, String server) {
+ incReadFailureCount();
+ LOG.error(String.format("Read from zNode:%s on zookeeper instance:%s", zNode, server));
+ }
+
+ @Override public void publishReadTiming(String znode, String server, long msTime) {
+ LOG.info(String.format("Read from zNode:%s on zookeeper instance:%s in %dms",
+ znode, server, msTime));
+ }
+ }
+
+ static class ZookeeperTask implements Callable<Void> {
+ private final HConnection connection;
+ private final String host;
+ private String znode;
+ private final int timeout;
+ private ZookeeperStdOutSink sink;
+
+ public ZookeeperTask(HConnection connection, String host, String znode, int timeout,
+ ZookeeperStdOutSink sink) {
+ this.connection = connection;
+ this.host = host;
+ this.znode = znode;
+ this.timeout = timeout;
+ this.sink = sink;
+ }
+
+ @Override public Void call() throws Exception {
+ ZooKeeper zooKeeper = null;
+ try {
+ zooKeeper = new ZooKeeper(host, timeout, EmptyWatcher.instance);
+ Stat exists = zooKeeper.exists(znode, false);
+ StopWatch stopwatch = new StopWatch();
+ stopwatch.start();
+ zooKeeper.getData(znode, false, exists);
+ stopwatch.stop();
+ sink.publishReadTiming(znode, host, stopwatch.getTime());
+ } catch (KeeperException e) {
+ sink.publishReadFailure(znode, host);
+ } catch (InterruptedException e) {
+ sink.publishReadFailure(znode, host);
+ } finally {
+ if (zooKeeper != null) {
+ zooKeeper.close();
+ }
+ }
+ return null;
+ }
+ }
+
/**
* For each column family of the region tries to get one row and outputs the latency, or the
* failure.
@@ -455,6 +521,8 @@ public final class Canary implements Tool {
private long timeout = DEFAULT_TIMEOUT;
private boolean failOnError = true;
private boolean regionServerMode = false;
+ private boolean zookeeperMode = false;
+
private boolean writeSniffing = false;
private boolean treatFailureAsError = false;
private TableName writeTableName = DEFAULT_WRITE_TABLE_NAME;
@@ -514,6 +582,8 @@ public final class Canary implements Tool {
System.err.println("-interval needs a numeric value argument.");
printUsageAndExit();
}
+ } else if (cmd.equals("-zookeeper")) {
+ this.zookeeperMode = true;
} else if(cmd.equals("-regionserver")) {
this.regionServerMode = true;
} else if(cmd.equals("-writeSniffing")) {
@@ -564,6 +634,15 @@ public final class Canary implements Tool {
index = i;
}
}
+
+ if (this.zookeeperMode) {
+ if (this.regionServerMode || this.writeSniffing) {
+ System.err.println("-zookeeper is exclusive and cannot be combined with "
+ + "other modes.");
+ printUsageAndExit();
+ }
+ }
+
return index;
}
@@ -641,6 +720,8 @@ public final class Canary implements Tool {
System.err.println(" -help Show this help and exit.");
System.err.println(" -regionserver replace the table argument to regionserver,");
System.err.println(" which means to enable regionserver mode");
+ System.err.println(" -zookeeper Tries to grab zookeeper.znode.parent ");
+ System.err.println(" on each zookeeper instance");
System.err.println(" -daemon Continuous check at defined intervals.");
System.err.println(" -interval <N> Interval between checks (sec)");
System.err.println(" -e Use table/regionserver as regular expression");
@@ -678,6 +759,10 @@ public final class Canary implements Tool {
monitor =
new RegionServerMonitor(connection, monitorTargets, this.useRegExp,
(ExtendedSink) this.sink, this.executor, this.treatFailureAsError);
+ } else if (this.zookeeperMode) {
+ monitor =
+ new ZookeeperMonitor(connection, monitorTargets, this.useRegExp,
+ (ZookeeperStdOutSink) this.sink, this.executor, this.treatFailureAsError);
} else {
monitor =
new RegionMonitor(connection, monitorTargets, this.useRegExp, this.sink, this.executor,
@@ -1060,6 +1145,62 @@ public final class Canary implements Tool {
}
}
}
+
+ // monitor for zookeeper mode
+ private static class ZookeeperMonitor extends Monitor {
+ private List<String> hosts;
+ private final String znode;
+ private final int timeout;
+
+ protected ZookeeperMonitor(HConnection connection, String[] monitorTargets, boolean useRegExp,
+ ExtendedSink sink, ExecutorService executor, boolean treatFailureAsError) {
+ super(connection, monitorTargets, useRegExp, sink, executor, treatFailureAsError);
+ Configuration configuration = connection.getConfiguration();
+ znode =
+ configuration.get(ZOOKEEPER_ZNODE_PARENT,
+ DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+ timeout = configuration
+ .getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);
+ ConnectStringParser parser =
+ new ConnectStringParser(ZKConfig.getZKQuorumServersString(configuration));
+ hosts = Lists.newArrayList();
+ for (InetSocketAddress server : parser.getServerAddresses()) {
+ hosts.add(server.toString());
+ }
+ }
+
+ @Override public void run() {
+ List<ZookeeperTask> tasks = Lists.newArrayList();
+ for (final String host : hosts) {
+ tasks.add(new ZookeeperTask(connection, host, znode, timeout, getSink()));
+ }
+ try {
+ for (Future<Void> future : this.executor.invokeAll(tasks)) {
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ LOG.error("Sniff zookeeper failed!", e);
+ this.errorCode = ERROR_EXIT_CODE;
+ }
+ }
+ } catch (InterruptedException e) {
+ this.errorCode = ERROR_EXIT_CODE;
+ Thread.currentThread().interrupt();
+ LOG.error("Sniff zookeeper interrupted!", e);
+ }
+ this.done = true;
+ }
+
+
+ private ZookeeperStdOutSink getSink() {
+ if (!(sink instanceof ZookeeperStdOutSink)) {
+ throw new RuntimeException("Can only write to zookeeper sink");
+ }
+ return ((ZookeeperStdOutSink) sink);
+ }
+ }
+
+
// a monitor for regionserver mode
private static class RegionServerMonitor extends Monitor {
@@ -1249,7 +1390,7 @@ public final class Canary implements Tool {
AuthUtil.launchAuthChore(conf);
int numThreads = conf.getInt("hbase.canary.threads.num", MAX_THREADS_NUM);
- LOG.info("Number of exection threads " + numThreads);
+ LOG.info("Number of execution threads " + numThreads);
ExecutorService executor = new ScheduledThreadPoolExecutor(numThreads);
http://git-wip-us.apache.org/repos/asf/hbase/blob/60958f59/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
index b765177..18c269a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Appender;
import org.apache.log4j.LogManager;
import org.apache.log4j.spi.LoggingEvent;
+import org.apache.hadoop.hbase.HConstants;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -44,6 +45,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.spy;
@@ -76,6 +78,23 @@ public class TestCanaryTool {
Appender mockAppender;
@Test
+ public void testBasicZookeeperCanaryWorks() throws Exception {
+ int port = testingUtility.getZkCluster().getClientPort();
+ testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM,
+ "localhost:" + port + "/hbase");
+ ExecutorService executor = new ScheduledThreadPoolExecutor(2);
+ Canary.ZookeeperStdOutSink sink = spy(new Canary.ZookeeperStdOutSink());
+ Canary canary = new Canary(executor, sink);
+ String[] args = { "-t", "10000", "-zookeeper" };
+ ToolRunner.run(testingUtility.getConfiguration(), canary, args);
+
+ String baseZnode = testingUtility.getConfiguration()
+ .get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+ verify(sink, atLeastOnce())
+ .publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
+ }
+
+ @Test
public void testBasicCanaryWorks() throws Exception {
TableName tableName = TableName.valueOf("testTable");
HTable table = testingUtility.createTable(tableName, new byte[][] { FAMILY });