You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2014/09/25 18:28:27 UTC
git commit: HBASE-11405 Multiple invocations of hbck in parallel
disables balancer permanently (Sean Busbey and bharath v)
Repository: hbase
Updated Branches:
refs/heads/0.98 be48e80d4 -> f73f18abb
HBASE-11405 Multiple invocations of hbck in parallel disables balancer permanently (Sean Busbey and bharath v)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f73f18ab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f73f18ab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f73f18ab
Branch: refs/heads/0.98
Commit: f73f18abbb26e37e955e77e1a7698d1a50aaa32c
Parents: be48e80
Author: Ted Yu <te...@apache.org>
Authored: Thu Sep 25 16:28:17 2014 +0000
Committer: Ted Yu <te...@apache.org>
Committed: Thu Sep 25 16:28:17 2014 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 82 +++++++++++++++++++-
.../hadoop/hbase/HBaseTestingUtility.java | 58 ++++++++++++++
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 50 ++++++++++++
3 files changed, 189 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f73f18ab/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index edd8b66..d0a19ba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.net.InetAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@@ -45,6 +46,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -52,10 +54,12 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.ClusterStatus;
@@ -105,7 +109,10 @@ import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
@@ -174,6 +181,8 @@ public class HBaseFsck extends Configured {
private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
private static final int DEFAULT_MAX_MERGE = 5;
private static final String TO_BE_LOADED = "to_be_loaded";
+ private static final String HBCK_LOCK_FILE = "hbase-hbck.lock";
+
/**********************
* Internal resources
@@ -188,6 +197,12 @@ public class HBaseFsck extends Configured {
private long startMillis = System.currentTimeMillis();
private HFileCorruptionChecker hfcc;
private int retcode = 0;
+ private Path HBCK_LOCK_PATH;
+ private FSDataOutputStream hbckOutFd;
+ // This lock is to prevent cleanup of balancer resources twice between
+ // ShutdownHook and the main code. We cleanup only if the connect() is
+ // successful
+ private final AtomicBoolean hbckLockCleanup = new AtomicBoolean(false);
/***********
* Options
@@ -297,10 +312,73 @@ public class HBaseFsck extends Configured {
}
/**
+ * This method maintains a lock using a file. If the creation fails we return null
+ *
+ * @return FSDataOutputStream object corresponding to the newly opened lock file
+ * @throws IOException
+ */
+ private FSDataOutputStream checkAndMarkRunningHbck() throws IOException {
+ try {
+ FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
+ FsPermission defaultPerms = FSUtils.getFilePermissions(fs, getConf(),
+ HConstants.DATA_FILE_UMASK_KEY);
+ Path tmpDir = new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY);
+ fs.mkdirs(tmpDir);
+ HBCK_LOCK_PATH = new Path(tmpDir, HBCK_LOCK_FILE);
+ final FSDataOutputStream out = FSUtils.create(fs, HBCK_LOCK_PATH, defaultPerms, false);
+ out.writeBytes(InetAddress.getLocalHost().toString());
+ out.flush();
+ return out;
+ } catch(RemoteException e) {
+ if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
+ return null;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ private void unlockHbck() {
+ if(hbckLockCleanup.compareAndSet(true, false)){
+ IOUtils.closeStream(hbckOutFd);
+ try{
+ FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true);
+ } catch(IOException ioe) {
+ LOG.warn("Failed to delete " + HBCK_LOCK_PATH);
+ LOG.debug(ioe);
+ }
+ }
+ }
+
+ /**
* To repair region consistency, one must call connect() in order to repair
* online state.
*/
public void connect() throws IOException {
+
+ // Check if another instance of balancer is running
+ hbckOutFd = checkAndMarkRunningHbck();
+ if (hbckOutFd == null) {
+ setRetCode(-1);
+ LOG.error("Another instance of hbck is running, exiting this instance.[If you are sure" +
+ " no other instance is running, delete the lock file " +
+ HBCK_LOCK_PATH + " and rerun the tool]");
+ throw new IOException("Duplicate hbck - Abort");
+ }
+
+ // Make sure to cleanup the lock
+ hbckLockCleanup.set(true);
+
+ // Add a shutdown hook to this thread, incase user tries to
+ // kill the hbck with a ctrl-c, we want to cleanup the lock so that
+ // it is available for further calls
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ public void run() {
+ unlockHbck();
+ }
+ });
+ LOG.debug("Launching hbck");
+
connection = HConnectionManager.createConnection(getConf());
admin = new HBaseAdmin(connection);
meta = new HTable(TableName.META_TABLE_NAME, connection);
@@ -495,6 +573,9 @@ public class HBaseFsck extends Configured {
checkAndFixTableLocks();
+ // Remove the hbck lock
+ unlockHbck();
+
// Print table summary
printTableSummary(tablesInfo);
return errors.summarize();
@@ -3825,7 +3906,6 @@ public class HBaseFsck extends Configured {
Path hbasedir = FSUtils.getRootDir(conf);
URI defaultFs = hbasedir.getFileSystem(conf).getUri();
FSUtils.setFsDefault(conf, new Path(defaultFs));
-
int ret = ToolRunner.run(new HBaseFsckTool(conf), args);
System.exit(ret);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f73f18ab/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index d4f4888..b93e565 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -27,6 +27,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.UnknownHostException;
@@ -103,6 +104,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.TaskLog;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.WatchedEvent;
@@ -262,6 +264,18 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
}
/**
+ * Controls how many attempts we will make in the face of failures in HDFS.
+ * @deprecated to be removed with Hadoop 1.x support
+ */
+ @Deprecated
+ public void setHDFSClientRetry(final int retries) {
+ this.conf.setInt("hdfs.client.retries.number", retries);
+ if (0 == retries) {
+ makeDFSClientNonRetrying();
+ }
+ }
+
+ /**
* Returns this classes's instance of {@link Configuration}. Be careful how
* you use the returned Configuration since {@link HConnection} instances
* can be shared. The Map of HConnections is keyed by the Configuration. If
@@ -1927,6 +1941,50 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
return createMultiRegions(c, table, columnFamily, KEYS);
}
+ void makeDFSClientNonRetrying() {
+ if (null == this.dfsCluster) {
+ LOG.debug("dfsCluster has not started, can't make client non-retrying.");
+ return;
+ }
+ try {
+ final FileSystem filesystem = this.dfsCluster.getFileSystem();
+ if (!(filesystem instanceof DistributedFileSystem)) {
+ LOG.debug("dfsCluster is not backed by a DistributedFileSystem, can't make client non-retrying.");
+ return;
+ }
+ // rely on FileSystem.CACHE to alter how we talk via DFSClient
+ final DistributedFileSystem fs = (DistributedFileSystem)filesystem;
+ // retrieve the backing DFSClient instance
+ final Field dfsField = fs.getClass().getDeclaredField("dfs");
+ dfsField.setAccessible(true);
+ final Class<?> dfsClazz = dfsField.getType();
+ final DFSClient dfs = DFSClient.class.cast(dfsField.get(fs));
+
+ // expose the method for creating direct RPC connections.
+ final Method createRPCNamenode = dfsClazz.getDeclaredMethod("createRPCNamenode", InetSocketAddress.class, Configuration.class, UserGroupInformation.class);
+ createRPCNamenode.setAccessible(true);
+
+ // grab the DFSClient instance's backing connection information
+ final Field nnField = dfsClazz.getDeclaredField("nnAddress");
+ nnField.setAccessible(true);
+ final InetSocketAddress nnAddress = InetSocketAddress.class.cast(nnField.get(dfs));
+ final Field confField = dfsClazz.getDeclaredField("conf");
+ confField.setAccessible(true);
+ final Configuration conf = Configuration.class.cast(confField.get(dfs));
+ final Field ugiField = dfsClazz.getDeclaredField("ugi");
+ ugiField.setAccessible(true);
+ final UserGroupInformation ugi = UserGroupInformation.class.cast(ugiField.get(dfs));
+
+ // replace the proxy for the namenode rpc with a direct instance
+ final Field namenodeField = dfsClazz.getDeclaredField("namenode");
+ namenodeField.setAccessible(true);
+ namenodeField.set(dfs, createRPCNamenode.invoke(null, nnAddress, conf, ugi));
+ LOG.debug("Set DSFClient namenode to bare RPC");
+ } catch (Exception exception) {
+ LOG.info("Could not alter DFSClient to be non-retrying.", exception);
+ }
+ }
+
/**
* Creates the specified number of regions in the specified table.
* @param c
http://git-wip-us.apache.org/repos/asf/hbase/blob/f73f18ab/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 62149e5..c903c12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -36,8 +36,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -127,11 +130,13 @@ public class TestHBaseFsck {
Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
+ @SuppressWarnings("deprecation")
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.handler.count", 2);
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.metahandler.count", 2);
TEST_UTIL.startMiniCluster(3);
+ TEST_UTIL.setHDFSClientRetry(0);
executorService = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS,
new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
@@ -513,6 +518,51 @@ public class TestHBaseFsck {
}
/**
+ * This test makes sure that parallel instances of Hbck is disabled.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testParallelHbck() throws Exception {
+ final ExecutorService service;
+ final Future<HBaseFsck> hbck1,hbck2;
+
+ class RunHbck implements Callable<HBaseFsck>{
+ boolean fail = true;
+ public HBaseFsck call(){
+ try{
+ return doFsck(conf, false);
+ } catch(Exception e){
+ if (e.getMessage().contains("Duplicate hbck")) {
+ fail = false;
+ } else {
+ LOG.fatal("hbck failed.", e);
+ }
+ }
+ // If we reach here, then an exception was caught
+ if (fail) fail();
+ return null;
+ }
+ }
+ service = Executors.newFixedThreadPool(2);
+ hbck1 = service.submit(new RunHbck());
+ hbck2 = service.submit(new RunHbck());
+ service.shutdown();
+ //wait for 15 seconds, for both hbck calls finish
+ service.awaitTermination(15, TimeUnit.SECONDS);
+ HBaseFsck h1 = hbck1.get();
+ HBaseFsck h2 = hbck2.get();
+ // Make sure only one of the calls was successful
+ assert(h1 == null || h2 == null);
+ if (h1 != null) {
+ assert(h1.getRetCode() >= 0);
+ }
+ if (h2 != null) {
+ assert(h2.getRetCode() >= 0);
+ }
+ }
+
+ /**
* This create and fixes a bad table with regions that have a duplicate
* start key
*/