You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gc...@apache.org on 2012/10/06 01:16:36 UTC
svn commit: r1394857 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/ipc/
test/java/org/apache/hadoop/hbase/client/
test/java/org/apache/hadoop/hbase/ipc/
Author: gchanan
Date: Fri Oct 5 23:16:35 2012
New Revision: 1394857
URL: http://svn.apache.org/viewvc?rev=1394857&view=rev
Log:
HBASE-6920 On timeout connecting to master, client can get stuck and never make progress
Added:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Fri Oct 5 23:16:35 2012
@@ -616,6 +616,23 @@ public class HConnectionManager {
return this.conf;
}
+ /**
+ * Log failure of getMaster attempt
+ * @return true if should retry
+ */
+ private boolean shouldRetryGetMaster(int tries, Exception e) {
+ if (tries == numRetries - 1) {
+ // This was our last chance - don't bother sleeping
+ LOG.info("getMaster attempt " + tries + " of " + numRetries +
+ " failed; no more retrying.", e);
+ return false;
+ }
+ LOG.info("getMaster attempt " + tries + " of " + numRetries +
+ " failed; retrying after sleep of " +
+ ConnectionUtils.getPauseTime(this.pause, tries), e);
+ return true;
+ }
+
public HMasterInterface getMaster()
throws MasterNotRunningException, ZooKeeperConnectionException {
// TODO: REMOVE. MOVE TO HBaseAdmin and redo as a Callable!!!
@@ -673,15 +690,9 @@ public class HConnectionManager {
}
} catch (IOException e) {
- if (tries == numRetries - 1) {
- // This was our last chance - don't bother sleeping
- LOG.info("getMaster attempt " + tries + " of " + numRetries +
- " failed; no more retrying.", e);
- break;
- }
- LOG.info("getMaster attempt " + tries + " of " + numRetries +
- " failed; retrying after sleep of " +
- ConnectionUtils.getPauseTime(this.pause, tries), e);
+ if (!shouldRetryGetMaster(tries, e)) break;
+ } catch (UndeclaredThrowableException ute) {
+ if (!shouldRetryGetMaster(tries, ute)) break;
}
// Cannot connect to master or it is not running. Sleep & retry
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java Fri Oct 5 23:16:35 2012
@@ -129,7 +129,7 @@ public class HBaseRPC {
}
// return the RpcEngine that handles a proxy object
- private static synchronized RpcEngine getProxyEngine(Object proxy) {
+ static synchronized RpcEngine getProxyEngine(Object proxy) {
return PROXY_ENGINES.get(proxy.getClass());
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java Fri Oct 5 23:16:35 2012
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.ipc;
+import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -37,6 +38,13 @@ interface RpcEngine {
User ticket, Configuration conf,
SocketFactory factory, int rpcTimeout) throws IOException;
+ /** Construct a client-side proxy object, specifying an InvocationHandler for testing purposes */
+ VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+ long clientVersion, InetSocketAddress addr,
+ User ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout, InvocationHandler handler)
+ throws IOException;
+
/** Stop this proxy. */
void stopProxy(VersionedProtocol proxy);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java Fri Oct 5 23:16:35 2012
@@ -121,7 +121,7 @@ class WritableRpcEngine implements RpcEn
protected final static ClientCache CLIENTS = new ClientCache();
- private static class Invoker implements InvocationHandler {
+ static class Invoker implements InvocationHandler {
private Class<? extends VersionedProtocol> protocol;
private InetSocketAddress address;
private User ticket;
@@ -172,13 +172,12 @@ class WritableRpcEngine implements RpcEn
public VersionedProtocol getProxy(
Class<? extends VersionedProtocol> protocol, long clientVersion,
InetSocketAddress addr, User ticket,
- Configuration conf, SocketFactory factory, int rpcTimeout)
+ Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
throws IOException {
VersionedProtocol proxy =
(VersionedProtocol) Proxy.newProxyInstance(
- protocol.getClassLoader(), new Class[] { protocol },
- new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+ protocol.getClassLoader(), new Class[] { protocol }, handler);
if (proxy instanceof VersionedProtocol) {
long serverVersion = ((VersionedProtocol)proxy)
.getProtocolVersion(protocol.getName(), clientVersion);
@@ -190,6 +189,14 @@ class WritableRpcEngine implements RpcEn
return proxy;
}
+ public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+ long clientVersion, InetSocketAddress addr,
+ User ticket, Configuration conf,
+ SocketFactory factory, int rpcTimeout) throws IOException {
+ return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
+ new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+ }
+
/**
* Stop this proxy and release its invoker's resource
* @param proxy the proxy to be stopped
Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java?rev=1394857&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java Fri Oct 5 23:16:35 2012
@@ -0,0 +1,89 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.RandomTimeoutRpcEngine;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestClientTimeouts {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ protected static int SLAVES = 1;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ RandomTimeoutRpcEngine.setProtocolEngine(conf, HMasterInterface.class);
+ TEST_UTIL.startMiniCluster(SLAVES);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Test that a client that fails an RPC to the master retries properly and
+ * doesn't throw any unexpected exceptions.
+ * @throws Exception
+ */
+ @Test
+ public void testClientGetMasterFailure() throws Exception {
+ long lastLimit = HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT;
+ HConnection lastConnection = null;
+ for (int i = 0; i < 5; ++i) {
+ // Ensure the HBaseAdmin uses a new connection by changing Configuration.
+ Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+ conf.setLong(HConstants.HBASE_CLIENT_PREFETCH_LIMIT, ++lastLimit);
+ try {
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ HConnection connection = admin.getConnection();
+ assertFalse(connection == lastConnection);
+ // Ensure the RandomTimeoutRpcEngine is actually being used.
+ assertTrue(RandomTimeoutRpcEngine.isProxyForObject((admin.getMaster())));
+ } catch (MasterNotRunningException ex) {
+ // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get
+ // a MasterNotRunningException. We only care about other exceptions -- i.e.
+ // UndeclaredThrowableException.
+ }
+ }
+ }
+}
Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java?rev=1394857&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java Fri Oct 5 23:16:35 2012
@@ -0,0 +1,77 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.Random;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import javax.net.SocketFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.security.User;
+
+/**
+ * RpcEngine that random throws a SocketTimeoutEngine for testing.
+ * Make sure to call setProtocolEngine to have the client actually use the RpcEngine
+ * for a specific protocol
+ */
+public class RandomTimeoutRpcEngine extends WritableRpcEngine {
+
+ private static final Random RANDOM = new Random(System.currentTimeMillis());
+ public static double chanceOfTimeout = 0.3;
+
+ public VersionedProtocol getProxy(
+ Class<? extends VersionedProtocol> protocol, long clientVersion,
+ InetSocketAddress addr, User ticket,
+ Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException {
+ RandomTimeoutInvoker handler =
+ new RandomTimeoutInvoker(protocol, addr, ticket, conf, factory, rpcTimeout);
+ return super.getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout, handler);
+ }
+
+ // Call this in order to set this class to run as the RpcEngine for the given protocol
+ public static void setProtocolEngine(Configuration conf, Class protocol) {
+ HBaseRPC.setProtocolEngine(conf, protocol, RandomTimeoutRpcEngine.class);
+ }
+
+ public static boolean isProxyForObject(Object proxy) {
+ return HBaseRPC.getProxyEngine(proxy).getClass().equals(RandomTimeoutRpcEngine.class);
+ }
+
+ static class RandomTimeoutInvoker extends Invoker {
+
+ public RandomTimeoutInvoker(Class<? extends VersionedProtocol> protocol,
+ InetSocketAddress address, User ticket,
+ Configuration conf, SocketFactory factory, int rpcTimeout) {
+ super(protocol, address, ticket, conf, factory, rpcTimeout);
+ }
+
+ public Object invoke(Object proxy, Method method, Object[] args)
+ throws Throwable {
+ if (RANDOM.nextFloat() < chanceOfTimeout) {
+ throw new SocketTimeoutException("fake timeout");
+ }
+ return super.invoke(proxy, method, args);
+ }
+ }
+}