You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2014/05/13 18:21:21 UTC
svn commit: r1594264 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/
src/main/java/org/apache/hadoop/hdfs/server/namenod...
Author: kihwal
Date: Tue May 13 16:21:20 2014
New Revision: 1594264
URL: http://svn.apache.org/r1594264
Log:
svn merge -c 1594263 merging from trunk to branch-2 to fix:HDFS-6334. Client failover proxy provider for IP failover based NN HA. Contributed by Kihwal Lee.
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
- copied unchanged from r1594263, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
- copied unchanged from r1594263, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/WrappedFailoverProxyProvider.java
- copied unchanged from r1594263, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/WrappedFailoverProxyProvider.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue May 13 16:21:20 2014
@@ -16,6 +16,9 @@ Release 2.5.0 - UNRELEASED
HDFS-5168. Add cross node dependency support to BlockPlacementPolicy.
(Nikola Vujic via szetszwo)
+ HDFS-6334. Client failover proxy provider for IP failover based NN HA.
+ (kihwal)
+
IMPROVEMENTS
HDFS-6007. Update documentation about short-circuit local reads (iwasakims
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java Tue May 13 16:21:20 2014
@@ -37,9 +37,11 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
+import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
@@ -203,18 +205,55 @@ public class HAUtil {
/**
* @return true if the given nameNodeUri appears to be a logical URI.
- * This is the case if there is a failover proxy provider configured
- * for it in the given configuration.
*/
public static boolean isLogicalUri(
Configuration conf, URI nameNodeUri) {
String host = nameNodeUri.getHost();
+ // A logical name must be one of the service IDs.
+ return DFSUtil.getNameServiceIds(conf).contains(host);
+ }
+
+ /**
+ * Check whether the client has a failover proxy provider configured
+ * for the namenode/nameservice.
+ *
+ * @param conf Configuration
+ * @param nameNodeUri The URI of namenode
+ * @return true if failover is configured.
+ */
+ public static boolean isClientFailoverConfigured(
+ Configuration conf, URI nameNodeUri) {
+ String host = nameNodeUri.getHost();
String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
+ host;
return conf.get(configKey) != null;
}
/**
+ * Check whether logical URI is needed for the namenode and
+ * the corresponding failover proxy provider in the config.
+ *
+ * @param conf Configuration
+ * @param nameNodeUri The URI of namenode
+ * @return true if logical URI is needed. false, if not needed.
+ * @throws IOException most likely due to misconfiguration.
+ */
+ public static boolean useLogicalUri(Configuration conf, URI nameNodeUri)
+ throws IOException {
+ // Create the proxy provider. Actual proxy is not created.
+ AbstractNNFailoverProxyProvider<ClientProtocol> provider = NameNodeProxies
+ .createFailoverProxyProvider(conf, nameNodeUri, ClientProtocol.class,
+ false);
+
+ // No need to use logical URI since failover is not configured.
+ if (provider == null) {
+ return false;
+ }
+ // Check whether the failover proxy provider uses logical URI.
+ return provider.useLogicalURI();
+ }
+
+ /**
* Parse the file system URI out of the provided token.
*/
public static URI getServiceUriFromToken(final String scheme,
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java Tue May 13 16:21:20 2014
@@ -50,6 +50,8 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
+import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
+import org.apache.hadoop.hdfs.server.namenode.ha.WrappedFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
@@ -136,26 +138,29 @@ public class NameNodeProxies {
@SuppressWarnings("unchecked")
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
URI nameNodeUri, Class<T> xface) throws IOException {
- Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
- getFailoverProxyProviderClass(conf, nameNodeUri, xface);
+ AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
+ createFailoverProxyProvider(conf, nameNodeUri, xface, true);
- if (failoverProxyProviderClass == null) {
+ if (failoverProxyProvider == null) {
// Non-HA case
return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
UserGroupInformation.getCurrentUser(), true);
} else {
// HA case
- FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies
- .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface,
- nameNodeUri);
Conf config = new Conf(conf);
T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
RetryPolicies.failoverOnNetworkException(
RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
config.maxRetryAttempts, config.failoverSleepBaseMillis,
config.failoverSleepMaxMillis));
-
- Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
+
+ Text dtService;
+ if (failoverProxyProvider.useLogicalURI()) {
+ dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
+ } else {
+ dtService = SecurityUtil.buildTokenService(
+ NameNode.getAddress(nameNodeUri));
+ }
return new ProxyAndInfo<T>(proxy, dtService);
}
}
@@ -183,12 +188,10 @@ public class NameNodeProxies {
Configuration config, URI nameNodeUri, Class<T> xface,
int numResponseToDrop) throws IOException {
Preconditions.checkArgument(numResponseToDrop > 0);
- Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
- getFailoverProxyProviderClass(config, nameNodeUri, xface);
- if (failoverProxyProviderClass != null) { // HA case
- FailoverProxyProvider<T> failoverProxyProvider =
- createFailoverProxyProvider(config, failoverProxyProviderClass,
- xface, nameNodeUri);
+ AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
+ createFailoverProxyProvider(config, nameNodeUri, xface, true);
+
+ if (failoverProxyProvider != null) { // HA case
int delay = config.getInt(
DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
@@ -211,7 +214,13 @@ public class NameNodeProxies {
T proxy = (T) Proxy.newProxyInstance(
failoverProxyProvider.getInterface().getClassLoader(),
new Class[] { xface }, dummyHandler);
- Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
+ Text dtService;
+ if (failoverProxyProvider.useLogicalURI()) {
+ dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
+ } else {
+ dtService = SecurityUtil.buildTokenService(
+ NameNode.getAddress(nameNodeUri));
+ }
return new ProxyAndInfo<T>(proxy, dtService);
} else {
LOG.warn("Currently creating proxy using " +
@@ -396,7 +405,7 @@ public class NameNodeProxies {
/** Gets the configured Failover proxy provider's class */
@VisibleForTesting
public static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass(
- Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException {
+ Configuration conf, URI nameNodeUri) throws IOException {
if (nameNodeUri == null) {
return null;
}
@@ -408,17 +417,6 @@ public class NameNodeProxies {
@SuppressWarnings("unchecked")
Class<FailoverProxyProvider<T>> ret = (Class<FailoverProxyProvider<T>>) conf
.getClass(configKey, null, FailoverProxyProvider.class);
- if (ret != null) {
- // If we found a proxy provider, then this URI should be a logical NN.
- // Given that, it shouldn't have a non-default port number.
- int port = nameNodeUri.getPort();
- if (port > 0 && port != NameNode.DEFAULT_PORT) {
- throw new IOException("Port " + port + " specified in URI "
- + nameNodeUri + " but host '" + host
- + "' is a logical (HA) namenode"
- + " and does not use port information.");
- }
- }
return ret;
} catch (RuntimeException e) {
if (e.getCause() instanceof ClassNotFoundException) {
@@ -433,18 +431,33 @@ public class NameNodeProxies {
/** Creates the Failover proxy provider instance*/
@VisibleForTesting
- public static <T> FailoverProxyProvider<T> createFailoverProxyProvider(
- Configuration conf, Class<FailoverProxyProvider<T>> failoverProxyProviderClass,
- Class<T> xface, URI nameNodeUri) throws IOException {
+ public static <T> AbstractNNFailoverProxyProvider<T> createFailoverProxyProvider(
+ Configuration conf, URI nameNodeUri, Class<T> xface, boolean checkPort)
+ throws IOException {
+ Class<FailoverProxyProvider<T>> failoverProxyProviderClass = null;
+ AbstractNNFailoverProxyProvider<T> providerNN;
Preconditions.checkArgument(
xface.isAssignableFrom(NamenodeProtocols.class),
"Interface %s is not a NameNode protocol", xface);
try {
+ // Obtain the class of the proxy provider
+ failoverProxyProviderClass = getFailoverProxyProviderClass(conf,
+ nameNodeUri);
+ if (failoverProxyProviderClass == null) {
+ return null;
+ }
+ // Create a proxy provider instance.
Constructor<FailoverProxyProvider<T>> ctor = failoverProxyProviderClass
.getConstructor(Configuration.class, URI.class, Class.class);
FailoverProxyProvider<T> provider = ctor.newInstance(conf, nameNodeUri,
xface);
- return provider;
+
+ // If the proxy provider is of an old implementation, wrap it.
+ if (!(provider instanceof AbstractNNFailoverProxyProvider)) {
+ providerNN = new WrappedFailoverProxyProvider<T>(provider);
+ } else {
+ providerNN = (AbstractNNFailoverProxyProvider<T>)provider;
+ }
} catch (Exception e) {
String message = "Couldn't create proxy provider " + failoverProxyProviderClass;
if (LOG.isDebugEnabled()) {
@@ -456,6 +469,20 @@ public class NameNodeProxies {
throw new IOException(message, e);
}
}
+
+ // Check the port in the URI, if it is logical.
+ if (checkPort && providerNN.useLogicalURI()) {
+ int port = nameNodeUri.getPort();
+ if (port > 0 && port != NameNode.DEFAULT_PORT) {
+ // Throwing here without any cleanup is fine since we have not
+ // actually created the underlying proxies yet.
+ throw new IOException("Port " + port + " specified in URI "
+ + nameNodeUri + " but host '" + nameNodeUri.getHost()
+ + "' is a logical (HA) namenode"
+ + " and does not use port information.");
+ }
+ }
+ return providerNN;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Tue May 13 16:21:20 2014
@@ -127,8 +127,8 @@ public class DatanodeWebHdfsMethods {
token.decodeFromUrlString(delegation);
URI nnUri = URI.create(HdfsConstants.HDFS_URI_SCHEME +
"://" + nnId);
- boolean isHA = HAUtil.isLogicalUri(conf, nnUri);
- if (isHA) {
+ boolean isLogical = HAUtil.isLogicalUri(conf, nnUri);
+ if (isLogical) {
token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri));
} else {
token.setService(SecurityUtil.buildTokenService(nnUri));
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java Tue May 13 16:21:20 2014
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.UserGroupInformation;
@@ -46,8 +46,8 @@ import com.google.common.base.Preconditi
* to connect to during fail-over. The first configured address is tried first,
* and on a fail-over event the other address is tried.
*/
-public class ConfiguredFailoverProxyProvider<T> implements
- FailoverProxyProvider<T> {
+public class ConfiguredFailoverProxyProvider<T> extends
+ AbstractNNFailoverProxyProvider<T> {
private static final Log LOG =
LogFactory.getLog(ConfiguredFailoverProxyProvider.class);
@@ -165,4 +165,12 @@ public class ConfiguredFailoverProxyProv
}
}
}
+
+ /**
+ * Logical URI is required for this failover proxy provider.
+ */
+ @Override
+ public boolean useLogicalURI() {
+ return true;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Tue May 13 16:21:20 2014
@@ -913,9 +913,10 @@ public class DFSAdmin extends FsShell {
Configuration dfsConf = dfs.getConf();
URI dfsUri = dfs.getUri();
- boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);
- if (isHaEnabled) {
- // In the case of HA, run finalizeUpgrade for all NNs in this nameservice
+ boolean isHaAndLogicalUri = HAUtil.isLogicalUri(dfsConf, dfsUri);
+ if (isHaAndLogicalUri) {
+ // In the case of HA and logical URI, run finalizeUpgrade for all
+ // NNs in this nameservice.
String nsId = dfsUri.getHost();
List<ClientProtocol> namenodes =
HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId);
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Tue May 13 16:21:20 2014
@@ -176,10 +176,13 @@ public class WebHdfsFileSystem extends F
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.nnAddrs = resolveNNAddr();
- boolean isHA = HAUtil.isLogicalUri(conf, this.uri);
- // In non-HA case, the code needs to call getCanonicalUri() in order to
- // handle the case where no port is specified in the URI
- this.tokenServiceName = isHA ? HAUtil.buildTokenServiceForLogicalUri(uri)
+ boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.uri);
+ boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.uri);
+ // In non-HA or non-logical URI case, the code needs to call
+ // getCanonicalUri() in order to handle the case where no port is
+ // specified in the URI
+ this.tokenServiceName = isLogicalUri ?
+ HAUtil.buildTokenServiceForLogicalUri(uri)
: SecurityUtil.buildTokenService(getCanonicalUri());
initializeTokenAspect();
@@ -1095,8 +1098,8 @@ public class WebHdfsFileSystem extends F
/**
* Resolve an HDFS URL into real INetSocketAddress. It works like a DNS
* resolver when the URL points to an non-HA cluster. When the URL points to
- * an HA cluster, the resolver further resolves the logical name (i.e., the
- * authority in the URL) into real namenode addresses.
+ * an HA cluster with its logical name, the resolver further resolves the
+ * logical name(i.e., the authority in the URL) into real namenode addresses.
*/
private InetSocketAddress[] resolveNNAddr() throws IOException {
Configuration conf = getConf();
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java Tue May 13 16:21:20 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -41,12 +42,17 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
+import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
+import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.net.StandardSocketFactory;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.hamcrest.BaseMatcher;
@@ -172,12 +178,12 @@ public class TestDFSClientFailover {
*/
@Test
public void testLogicalUriShouldNotHavePorts() {
- Configuration conf = new HdfsConfiguration();
- conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + ".foo",
- ConfiguredFailoverProxyProvider.class.getName());
- Path p = new Path("hdfs://foo:12345/");
+ Configuration config = new HdfsConfiguration(conf);
+ String logicalName = HATestUtil.getLogicalHostname(cluster);
+ HATestUtil.setFailoverConfigurations(cluster, config, logicalName);
+ Path p = new Path("hdfs://" + logicalName + ":12345/");
try {
- p.getFileSystem(conf).exists(p);
+ p.getFileSystem(config).exists(p);
fail("Did not fail with fake FS");
} catch (IOException ioe) {
GenericTestUtils.assertExceptionContains(
@@ -278,4 +284,72 @@ public class TestDFSClientFailover {
// Ensure that the logical hostname was never resolved.
Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
}
+
+ /** Dummy implementation of plain FailoverProxyProvider */
+ public static class DummyLegacyFailoverProxyProvider<T>
+ implements FailoverProxyProvider<T> {
+ private Class<T> xface;
+ private T proxy;
+ public DummyLegacyFailoverProxyProvider(Configuration conf, URI uri,
+ Class<T> xface) {
+ try {
+ this.proxy = NameNodeProxies.createNonHAProxy(conf,
+ NameNode.getAddress(uri), xface,
+ UserGroupInformation.getCurrentUser(), false).getProxy();
+ this.xface = xface;
+ } catch (IOException ioe) {
+ }
+ }
+
+ @Override
+ public Class<T> getInterface() {
+ return xface;
+ }
+
+ @Override
+ public ProxyInfo<T> getProxy() {
+ return new ProxyInfo<T>(proxy, "dummy");
+ }
+
+ @Override
+ public void performFailover(T currentProxy) {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ }
+
+ /**
+ * Test to verify legacy proxy providers are correctly wrapped.
+ */
+ public void testWrappedFailoverProxyProvider() throws Exception {
+ // setup the config with the dummy provider class
+ Configuration config = new HdfsConfiguration(conf);
+ String logicalName = HATestUtil.getLogicalHostname(cluster);
+ HATestUtil.setFailoverConfigurations(cluster, config, logicalName);
+ config.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
+ DummyLegacyFailoverProxyProvider.class.getName());
+ Path p = new Path("hdfs://" + logicalName + "/");
+
+ // Logical URI should be used.
+ assertTrue("Legacy proxy providers should use logical URI.",
+ HAUtil.useLogicalUri(config, p.toUri()));
+ }
+
+ /**
+ * Test to verify IPFailoverProxyProvider is not requiring logical URI.
+ */
+ public void testIPFailoverProxyProviderLogicalUri() throws Exception {
+ // setup the config with the IP failover proxy provider class
+ Configuration config = new HdfsConfiguration(conf);
+ URI nnUri = cluster.getURI(0);
+ config.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." +
+ nnUri.getHost(),
+ IPFailoverProxyProvider.class.getName());
+
+ assertFalse("IPFailoverProxyProvider should not use logical URI.",
+ HAUtil.useLogicalUri(config, nnUri));
+ }
+
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1594264&r1=1594263&r2=1594264&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java Tue May 13 16:21:20 2014
@@ -188,12 +188,9 @@ public class TestRetryCacheWithHA {
private DFSClient genClientWithDummyHandler() throws IOException {
URI nnUri = dfs.getUri();
- Class<FailoverProxyProvider<ClientProtocol>> failoverProxyProviderClass =
- NameNodeProxies.getFailoverProxyProviderClass(conf, nnUri,
- ClientProtocol.class);
FailoverProxyProvider<ClientProtocol> failoverProxyProvider =
NameNodeProxies.createFailoverProxyProvider(conf,
- failoverProxyProviderClass, ClientProtocol.class, nnUri);
+ nnUri, ClientProtocol.class, true);
InvocationHandler dummyHandler = new DummyRetryInvocationHandler(
failoverProxyProvider, RetryPolicies
.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,