You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/11/28 19:12:03 UTC
svn commit: r1546423 - in /hbase/branches/0.89-fb: bin/
src/main/java/org/apache/hadoop/hbase/zookeeper/
src/test/java/org/apache/hadoop/hbase/zookeeper/
Author: liyin
Date: Thu Nov 28 18:12:02 2013
New Revision: 1546423
URL: http://svn.apache.org/r1546423
Log:
[HBASE-10006] utility to remove multiple Zookeeper nodes given a time range
Author: pervyshev
Test Plan: TestZooKeeperWrapper unit test; try executing commands from console as well
Reviewers: manukranthk, aaiyer, rshroff, liyintang
Reviewed By: manukranthk
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1037504
Task ID: 3069166
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapperMain.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperWrapper.java
Modified:
hbase/branches/0.89-fb/bin/hbase
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
Modified: hbase/branches/0.89-fb/bin/hbase
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/bin/hbase?rev=1546423&r1=1546422&r2=1546423&view=diff
==============================================================================
--- hbase/branches/0.89-fb/bin/hbase (original)
+++ hbase/branches/0.89-fb/bin/hbase Thu Nov 28 18:12:02 2013
@@ -68,6 +68,7 @@ if [ $# = 0 ]; then
echo " shell-tests run the HBase shell tests"
fi
echo " zkcli run the ZooKeeper shell"
+ echo " zkwrap run the ZooKeeperWrapper shell"
echo " master run an HBase HMaster node"
echo " regionserver run an HBase HRegionServer node"
echo " zookeeper run a Zookeeper server"
@@ -281,6 +282,8 @@ elif [ "$COMMAND" = "zookeeper" ] ; then
fi
elif [ "$COMMAND" = "zkcli" ] ; then
CLASS='org.apache.zookeeper.ZooKeeperMain'
+elif [ "$COMMAND" = "zkwrap" ] ; then
+ CLASS='org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapperMain'
elif [ "$COMMAND" = "classpath" ] ; then
# print the computed classpath and exit
CLASS=
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=1546423&r1=1546422&r2=1546423&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Thu Nov 28 18:12:02 2013
@@ -21,7 +21,9 @@ package org.apache.hadoop.hbase.zookeepe
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
@@ -33,8 +35,10 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -65,6 +69,7 @@ import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper.States;
import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
/**
@@ -124,8 +129,8 @@ public class ZooKeeperWrapper implements
/** Specifies the RS hosting root (host, port, start code) */
private final String rootRegionZNode;
-
- /** A znode containing root regionserver host:port only for compatibility with old clients */
+
+ /** A znode containing root regionserver host:port only for compatibility with old clients */
private final String legacyRootRegionZNode;
/**
@@ -175,6 +180,12 @@ public class ZooKeeperWrapper implements
*/
private static volatile boolean closedUnknownZKWrapper = false;
+ private static SimpleDateFormat format = new SimpleDateFormat("yy/MM/dd HH:mm:ss");
+
+ private static String format(long date) {
+ return format.format(new Date(date));
+ }
+
// return the singleton given the name of the instance
public static ZooKeeperWrapper getInstance(Configuration conf, String name) {
name = getZookeeperClusterKey(conf, name);
@@ -259,7 +270,7 @@ public class ZooKeeperWrapper implements
String rootServerZNodeName =
conf.get("zookeeper.znode.rootserver.complete", "root-region-server-complete");
- String legacyRootServerZNodeName =
+ String legacyRootServerZNodeName =
conf.get("zookeeper.znode.rootserver", "root-region-server");
String rsZNodeName = conf.get("zookeeper.znode.rs", "rs");
String masterAddressZNodeName = conf.get("zookeeper.znode.master", "master");
@@ -1044,7 +1055,7 @@ public class ZooKeeperWrapper implements
}
private String joinPath(String parent, String child) {
- return parent + ZNODE_PATH_SEPARATOR + child;
+ return (parent != "/" ? parent : "") + ZNODE_PATH_SEPARATOR + child;
}
/**
@@ -1804,7 +1815,7 @@ public class ZooKeeperWrapper implements
throws KeeperException {
try {
List<String> children = listChildrenNoWatch(node);
- if (!children.isEmpty()) {
+ if (children != null && !children.isEmpty()) {
for (String child : children) {
deleteNodeRecursively(joinPath(node, child));
}
@@ -1832,6 +1843,89 @@ public class ZooKeeperWrapper implements
}
/**
+ * Return the children of the node recursively; the result is sorted
+ */
+ public List<String> getChildrenRecursively(String path)
+ throws KeeperException, InterruptedException {
+ List<String> children = recoverableZK.getChildren(path, false);
+ Collections.sort(children);
+ List<String> result = new LinkedList<String>();
+ for (String child: children) {
+ String childPath = joinPath(path, child);
+ result.add(childPath);
+ List<String> childResult = getChildrenRecursively(childPath);
+ result.addAll(childResult);
+ }
+ return result;
+ }
+
+ /**
+ * Node filter
+ */
+ public static class NodeFilter {
+ private long st;
+ private long en;
+
+ public NodeFilter() {
+ this.st = -1;
+ this.en = -1;
+ }
+
+ public NodeFilter(long st, long en) {
+ this.st = st;
+ this.en = en;
+ }
+
+ public boolean matches(Stat stat) {
+ if (st >= 0 && en >= 0) {
+ return st <= stat.getMtime() && stat.getMtime() < en;
+ }
+ else if (st >= 0) {
+ return st <= stat.getMtime();
+ }
+ else if (en >= 0) {
+ return stat.getMtime() < en;
+ }
+ else {
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Delete the node if it passes the given filter
+ */
+ public void delete(String path, NodeFilter filter)
+ throws InterruptedException, KeeperException {
+ Stat stat = recoverableZK.exists(path, false);
+ if (stat == null) {
+ return;
+ }
+ if (filter != null && !filter.matches(stat)) {
+ return;
+ }
+ LOG.info("Deleting " + path + " Mtime = " + format(stat.getMtime()));
+ recoverableZK.delete(path, -1);
+ }
+
+ /**
+ * Delete all those descendants of the node that pass the filter
+ * (a descendant non-leaf node may be deleted only if all its descendants have been deleted)
+ */
+ public void deleteChildrenRecursively(String path, NodeFilter filter)
+ throws KeeperException, InterruptedException {
+ List<String> children = recoverableZK.getChildren(path, false);
+ for (String child : children) {
+ String childPath = joinPath(path, child);
+ deleteChildrenRecursively(childPath, filter);
+ List<String> childChildren = recoverableZK.getChildren(childPath, false);
+ if (childChildren.isEmpty()) {
+ delete(childPath, filter);
+ }
+ }
+ }
+
+ /**
* Handles InterruptedExceptions in client calls.
* <p>
* This may be temporary but for now this gives one place to deal with these.
@@ -1957,4 +2051,8 @@ public class ZooKeeperWrapper implements
public String getIdentifier() {
return recoverableZK.getIdentifier();
}
+
+ public RecoverableZooKeeper getRecoverableZooKeeper() {
+ return recoverableZK;
+ }
}
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapperMain.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapperMain.java?rev=1546423&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapperMain.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapperMain.java Thu Nov 28 18:12:02 2013
@@ -0,0 +1,189 @@
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.util.RuntimeExceptionAbortStrategy;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper.NodeFilter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+public class ZooKeeperWrapperMain {
+
+ @SuppressWarnings("serial")
+ private static class UsageException extends Exception {
+
+ public UsageException() {
+ super();
+ }
+
+ public UsageException(String message) {
+ super(message);
+ }
+
+ public UsageException(Throwable cause) {
+ super(cause);
+ }
+
+ }
+
+ private static class TimeParser {
+
+ static final SimpleDateFormat format = new SimpleDateFormat("yy/MM/dd HH:mm:ss");
+
+ public static long parse(String time)
+ throws UsageException {
+ try {
+ return format.parse(time).getTime();
+ } catch (java.text.ParseException e) {
+ // nothing to do
+ }
+ try {
+ return Long.parseLong(time);
+ } catch (NumberFormatException e) {
+ // nothing to do
+ }
+ throw new UsageException("Failed to parse a time string");
+ }
+
+ public static void help() {
+ Date now = new Date();
+ System.err.println("timestamp format is either '" + format.format(now) + "' or '" + now.getTime() + "'");
+ }
+ }
+
+ private static ZooKeeperWrapper createZooKeeperWrapper() {
+ Configuration conf = HBaseConfiguration.create();
+ try {
+ return ZooKeeperWrapper.createInstance(conf, "zkwrap", new RuntimeExceptionAbortStrategy());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to create a ZooKeeperWrapper instance", e);
+ }
+ }
+
+ static abstract class Command {
+
+ private static Options opts = new Options();
+
+ protected static Options getOptions() {
+ return opts;
+ }
+
+ protected abstract void help();
+
+ protected abstract void run(CommandLine line)
+ throws UsageException;
+
+ public void run(String[] args) {
+ try {
+ CommandLineParser parser = new BasicParser();
+ CommandLine line;
+ try {
+ line = parser.parse(opts, args);
+ } catch (ParseException e) {
+ throw new UsageException(e);
+ }
+ run(line);
+ } catch (UsageException e) {
+ if (e.getMessage() != null) {
+ System.err.println(e.getMessage());
+ }
+ else {
+ help();
+ }
+ }
+ }
+
+ }
+
+ static class DeleteCommand extends Command {
+
+ static {
+ getOptions().addOption("r", false, "delete children recursively (a required option)");
+ getOptions().addOption("st", true, "starting timestamp");
+ getOptions().addOption("en", true, "ending timestamp (not including)");
+ }
+
+ protected void help() {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ "delete [options] <path>",
+ getOptions()
+ );
+ TimeParser.help();
+ }
+
+ protected void run(CommandLine line) throws UsageException {
+ ZooKeeperWrapper zkWrapper = createZooKeeperWrapper();
+ if (line.hasOption("r")) {
+ if (line.getArgs().length != 1) {
+ throw new UsageException("one path is expected");
+ }
+ String path = line.getArgs()[0];
+ long st = TimeParser.parse(line.getOptionValue("st", "-1"));
+ long en = TimeParser.parse(line.getOptionValue("en", "-1"));
+ try {
+ zkWrapper.deleteChildrenRecursively(path, new NodeFilter(st, en));
+ }
+ catch (KeeperException e) {
+ throw new RuntimeException(e);
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ else {
+ throw new UsageException();
+ }
+ }
+
+ }
+
+ private void help() {
+ String message =
+ "supported commands:\n" +
+ " delete\n";
+ System.err.print(message);
+ }
+
+ private void run(String args[]) {
+ if (args.length == 0) {
+ help();
+ return;
+ }
+
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.WARN);
+ Logger.getLogger("org.apache.hadoop.hbase.zookeeper").setLevel(Level.WARN);
+
+ String commandName = args[0];
+ String[] commandArgs = Arrays.copyOfRange(args, 1, args.length);
+
+ Map<String, Command> commands = new HashMap<String, Command>();
+ commands.put("delete", new DeleteCommand());
+
+ if (commands.containsKey(commandName)) {
+ commands.get(commandName).run(commandArgs);
+ }
+ else {
+ help();
+ }
+ }
+
+ public static void main(String args[]) {
+ (new ZooKeeperWrapperMain()).run(args);
+ }
+
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperWrapper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperWrapper.java?rev=1546423&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperWrapper.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperWrapper.java Thu Nov 28 18:12:02 2013
@@ -0,0 +1,136 @@
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper.NodeFilter;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestZooKeeperWrapper {
+
+ private static final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+
+ private static final Log LOG = LogFactory.getLog(TestZooKeeperWrapper.class);
+
+ private ZooKeeperWrapper zkWrapper;
+ private RecoverableZooKeeper recoverableZK;
+
+ @BeforeClass
+ public static void setUpBeforeClass()
+ throws IOException, InterruptedException {
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass()
+ throws IOException {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUpBeforeTest()
+ throws IOException {
+ zkWrapper = ZooKeeperWrapper.createInstance(
+ TEST_UTIL.getConfiguration(), "test instance");
+ recoverableZK = zkWrapper.getRecoverableZooKeeper();
+ }
+
+ @After
+ public void tearDownAfterTest() {
+ // nothing to do
+ }
+
+ @SuppressWarnings("unused")
+ private static void print(String[] lines) {
+ System.out.println();
+ for (String line : lines) {
+ System.out.println(line);
+ }
+ System.out.println();
+ }
+
+ private String dump(String path)
+ throws KeeperException, InterruptedException {
+ List<String> result = zkWrapper.getChildrenRecursively(path);
+ StringBuilder sb = new StringBuilder();
+ for (String s : result) {
+ if (sb.length() > 0) {
+ sb.append(':');
+ }
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
+ private Stat create(String path)
+ throws KeeperException, InterruptedException {
+ recoverableZK.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ return recoverableZK.exists(path, false);
+ }
+
+ private Stat create(String path, long lastTime)
+ throws KeeperException, InterruptedException {
+ Stat stat = create(path);
+ while (stat.getMtime() <= lastTime) {
+ Thread.sleep(1);
+ stat = recoverableZK.setData(path, null, -1);
+ }
+ return stat;
+ }
+
+ private void delete(String path)
+ throws KeeperException {
+ if (zkWrapper.checkExists(path) == -1)
+ return;
+ zkWrapper.deleteNodeRecursively(path);
+ }
+
+ @Test
+ public void testDeleteChildrenRecursively()
+ throws KeeperException, InterruptedException {
+ delete("/test");
+
+ create("/test");
+ create("/test/a");
+ create("/test/a/b");
+ create("/test/a/c");
+
+ Assert.assertEquals("/test/a:/test/a/b:/test/a/c", dump("/test"));
+
+ zkWrapper.deleteChildrenRecursively("/test", new NodeFilter());
+
+ Assert.assertEquals("", dump("/test"));
+ }
+
+ @Test
+ public void testDeleteTimeRange()
+ throws KeeperException, InterruptedException {
+ delete("/test");
+
+ create("/test");
+ long t1 = create("/test/a").getMtime();
+ long t2 = create("/test/b", t1).getMtime();
+ long t3 = create("/test/b/c", t2).getMtime();
+ long t4 = create("/test/d", t3).getMtime();
+ long t5 = create("/test/d/e", t4).getMtime();
+
+ Assert.assertEquals("/test/a:/test/b:/test/b/c:/test/d:/test/d/e", dump("/test"));
+
+ zkWrapper.deleteChildrenRecursively("/test", new NodeFilter(t2, t5));
+
+ Assert.assertEquals("/test/a:/test/d:/test/d/e", dump("/test"));
+ }
+}