You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/06/03 08:51:56 UTC
[1/3] hadoop git commit: HDFS-8270. create() always retried with
hardcoded timeout when file already exists with open lease (Contributed by
J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 84cae6c10 -> 066e45bcb
refs/heads/branch-2.7 a7058caa7 -> 8090a6ee6
refs/heads/trunk a6a8ee3c2 -> 54f83d9bd
HDFS-8270. create() always retried with hardcoded timeout when file already exists with open lease (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54f83d9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54f83d9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54f83d9b
Branch: refs/heads/trunk
Commit: 54f83d9bd917e8641e902c5f0695e65ded472f9a
Parents: a6a8ee3
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jun 3 12:11:46 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jun 3 12:11:46 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ----
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 43 +-------------------
.../apache/hadoop/hdfs/TestFileCreation.java | 6 ---
5 files changed, 5 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f83d9b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index abf6452..402a547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -936,6 +936,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8486. DN startup may cause severe data loss (Daryn Sharp via Colin P.
McCabe)
+ HDFS-8270. create() always retried with hardcoded timeout when file already
+ exists with open lease (J.Andreina via vinayakumarb)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f83d9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index fc1cd26..f4ceab3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -342,13 +342,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.namenode = rpcNamenode;
dtService = null;
} else {
- boolean noRetries = conf.getBoolean(
- DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES,
- DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT);
Preconditions.checkArgument(nameNodeUri != null,
"null URI");
proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
- ClientProtocol.class, nnFallbackToSimpleAuth, !noRetries);
+ ClientProtocol.class, nnFallbackToSimpleAuth);
this.dtService = proxyInfo.getDelegationTokenService();
this.namenode = proxyInfo.getProxy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f83d9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9c19f91..5bb6e53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs;
import java.util.concurrent.TimeUnit;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -999,13 +998,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number";
public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0;
- // Create a NN proxy without retries for testing.
- @VisibleForTesting
- public static final String DFS_CLIENT_TEST_NO_PROXY_RETRIES =
- "dfs.client.test.no.proxy.retries";
- @VisibleForTesting
- public static final boolean DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT = false;
-
public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.client.slow.io.warning.threshold.ms";
public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f83d9b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
index bafc76f..d873526 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
@@ -43,7 +42,6 @@ import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
@@ -161,31 +159,6 @@ public class NameNodeProxies {
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
throws IOException {
- return createProxy(conf, nameNodeUri, xface, fallbackToSimpleAuth, true);
- }
-
- /**
- * Creates the namenode proxy with the passed protocol. This will handle
- * creation of either HA- or non-HA-enabled proxy objects, depending upon
- * if the provided URI is a configured logical URI.
- *
- * @param conf the configuration containing the required IPC
- * properties, client failover configurations, etc.
- * @param nameNodeUri the URI pointing either to a specific NameNode
- * or to a logical nameservice.
- * @param xface the IPC interface which should be created
- * @param fallbackToSimpleAuth set to true or false during calls to
- * indicate if a secure client falls back to simple auth
- * @param withRetries certain interfaces have a non-standard retry policy
- * @return an object containing both the proxy and the associated
- * delegation token service it corresponds to
- * @throws IOException if there is an error creating the proxy
- **/
- @SuppressWarnings("unchecked")
- public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
- URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth,
- boolean withRetries)
- throws IOException {
AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
createFailoverProxyProvider(conf, nameNodeUri, xface, true,
fallbackToSimpleAuth);
@@ -193,7 +166,7 @@ public class NameNodeProxies {
if (failoverProxyProvider == null) {
// Non-HA case
return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
- UserGroupInformation.getCurrentUser(), withRetries,
+ UserGroupInformation.getCurrentUser(), true,
fallbackToSimpleAuth);
} else {
// HA case
@@ -442,22 +415,8 @@ public class NameNodeProxies {
if (withRetries) { // create the proxy with retries
- RetryPolicy createPolicy = RetryPolicies
- .retryUpToMaximumCountWithFixedSleep(5,
- HdfsServerConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
-
- Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap
- = new HashMap<Class<? extends Exception>, RetryPolicy>();
- remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
- createPolicy);
-
- RetryPolicy methodPolicy = RetryPolicies.retryByRemoteException(
- defaultPolicy, remoteExceptionToPolicyMap);
Map<String, RetryPolicy> methodNameToPolicyMap
= new HashMap<String, RetryPolicy>();
-
- methodNameToPolicyMap.put("create", methodPolicy);
-
ClientProtocol translatorProxy =
new ClientNamenodeProtocolTranslatorPB(proxy);
return (ClientProtocol) RetryProxy.create(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f83d9b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index 4977015..525d84e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -74,7 +73,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -380,10 +378,6 @@ public class TestFileCreation {
SimulatedFSDataset.setFactory(conf);
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
- // Force NameNodeProxies' createNNProxyWithClientProtocol to give
- // up file creation after one failure.
- conf.setBoolean(DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES, true);
-
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
[3/3] hadoop git commit: HDFS-8270. create() always retried with
hardcoded timeout when file already exists with open lease (Contributed by
J.Andreina)
Posted by vi...@apache.org.
HDFS-8270. create() always retried with hardcoded timeout when file already exists with open lease (Contributed by J.Andreina)
(cherry picked from commit 54f83d9bd917e8641e902c5f0695e65ded472f9a)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8090a6ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8090a6ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8090a6ee
Branch: refs/heads/branch-2.7
Commit: 8090a6ee63c414ac5b76c21df1f6b5a188e873d6
Parents: a7058ca
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jun 3 12:11:46 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jun 3 12:20:54 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/NameNodeProxies.java | 15 ---------------
.../org/apache/hadoop/hdfs/TestFileCreation.java | 3 +--
3 files changed, 4 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8090a6ee/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index dbe42dd..f5f9e8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -99,6 +99,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8486. DN startup may cause severe data loss (Daryn Sharp via Colin P.
McCabe)
+ HDFS-8270. create() always retried with hardcoded timeout when file already
+ exists with open lease (J.Andreina via vinayakumarb)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8090a6ee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
index b261220..c7e2cf2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
@@ -42,7 +42,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient.Conf;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
@@ -68,7 +67,6 @@ import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.io.retry.RetryUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.SecurityUtil;
@@ -425,22 +423,9 @@ public class NameNodeProxies {
if (withRetries) { // create the proxy with retries
- RetryPolicy createPolicy = RetryPolicies
- .retryUpToMaximumCountWithFixedSleep(5,
- HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
-
- Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap
- = new HashMap<Class<? extends Exception>, RetryPolicy>();
- remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
- createPolicy);
-
- RetryPolicy methodPolicy = RetryPolicies.retryByRemoteException(
- defaultPolicy, remoteExceptionToPolicyMap);
Map<String, RetryPolicy> methodNameToPolicyMap
= new HashMap<String, RetryPolicy>();
- methodNameToPolicyMap.put("create", methodPolicy);
-
ClientProtocol translatorProxy =
new ClientNamenodeProtocolTranslatorPB(proxy);
return (ClientProtocol) RetryProxy.create(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8090a6ee/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index a0129da..10c810c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -405,9 +405,8 @@ public class TestFileCreation {
} catch (IOException abce) {
GenericTestUtils.assertExceptionContains("Failed to CREATE_FILE", abce);
}
- // NameNodeProxies' createNNProxyWithClientProtocol has 5 retries.
assertCounter("AlreadyBeingCreatedExceptionNumOps",
- 6L, getMetrics(metricsName));
+ 1L, getMetrics(metricsName));
FSDataOutputStream stm2 = fs2.create(p, true);
stm2.write(2);
stm2.close();
[2/3] hadoop git commit: HDFS-8270. create() always retried with
hardcoded timeout when file already exists with open lease (Contributed by
J.Andreina)
Posted by vi...@apache.org.
HDFS-8270. create() always retried with hardcoded timeout when file already exists with open lease (Contributed by J.Andreina)
(cherry picked from commit 54f83d9bd917e8641e902c5f0695e65ded472f9a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/066e45bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/066e45bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/066e45bc
Branch: refs/heads/branch-2
Commit: 066e45bcb667bb0c37ef70fd297b24e4f26383eb
Parents: 84cae6c
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jun 3 12:11:46 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jun 3 12:12:22 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 ----
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 43 +-------------------
.../apache/hadoop/hdfs/TestFileCreation.java | 6 ---
5 files changed, 5 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066e45bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 813de66..4b89e1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -599,6 +599,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8486. DN startup may cause severe data loss (Daryn Sharp via Colin P.
McCabe)
+ HDFS-8270. create() always retried with hardcoded timeout when file already
+ exists with open lease (J.Andreina via vinayakumarb)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066e45bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index b228b11..46a1235 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -344,13 +344,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.namenode = rpcNamenode;
dtService = null;
} else {
- boolean noRetries = conf.getBoolean(
- DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES,
- DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT);
Preconditions.checkArgument(nameNodeUri != null,
"null URI");
proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
- ClientProtocol.class, nnFallbackToSimpleAuth, !noRetries);
+ ClientProtocol.class, nnFallbackToSimpleAuth);
this.dtService = proxyInfo.getDelegationTokenService();
this.namenode = proxyInfo.getProxy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066e45bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f78c870..dc0ecca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs;
import java.util.concurrent.TimeUnit;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -1000,13 +999,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number";
public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0;
- // Create a NN proxy without retries for testing.
- @VisibleForTesting
- public static final String DFS_CLIENT_TEST_NO_PROXY_RETRIES =
- "dfs.client.test.no.proxy.retries";
- @VisibleForTesting
- public static final boolean DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT = false;
-
public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.client.slow.io.warning.threshold.ms";
public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066e45bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
index a7dda7b..8706f01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
@@ -43,7 +42,6 @@ import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
@@ -161,31 +159,6 @@ public class NameNodeProxies {
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
throws IOException {
- return createProxy(conf, nameNodeUri, xface, fallbackToSimpleAuth, true);
- }
-
- /**
- * Creates the namenode proxy with the passed protocol. This will handle
- * creation of either HA- or non-HA-enabled proxy objects, depending upon
- * if the provided URI is a configured logical URI.
- *
- * @param conf the configuration containing the required IPC
- * properties, client failover configurations, etc.
- * @param nameNodeUri the URI pointing either to a specific NameNode
- * or to a logical nameservice.
- * @param xface the IPC interface which should be created
- * @param fallbackToSimpleAuth set to true or false during calls to
- * indicate if a secure client falls back to simple auth
- * @param withRetries certain interfaces have a non-standard retry policy
- * @return an object containing both the proxy and the associated
- * delegation token service it corresponds to
- * @throws IOException if there is an error creating the proxy
- **/
- @SuppressWarnings("unchecked")
- public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
- URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth,
- boolean withRetries)
- throws IOException {
AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
createFailoverProxyProvider(conf, nameNodeUri, xface, true,
fallbackToSimpleAuth);
@@ -193,7 +166,7 @@ public class NameNodeProxies {
if (failoverProxyProvider == null) {
// Non-HA case
return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
- UserGroupInformation.getCurrentUser(), withRetries,
+ UserGroupInformation.getCurrentUser(), true,
fallbackToSimpleAuth);
} else {
// HA case
@@ -442,22 +415,8 @@ public class NameNodeProxies {
if (withRetries) { // create the proxy with retries
- RetryPolicy createPolicy = RetryPolicies
- .retryUpToMaximumCountWithFixedSleep(5,
- HdfsServerConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
-
- Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap
- = new HashMap<Class<? extends Exception>, RetryPolicy>();
- remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
- createPolicy);
-
- RetryPolicy methodPolicy = RetryPolicies.retryByRemoteException(
- defaultPolicy, remoteExceptionToPolicyMap);
Map<String, RetryPolicy> methodNameToPolicyMap
= new HashMap<String, RetryPolicy>();
-
- methodNameToPolicyMap.put("create", methodPolicy);
-
ClientProtocol translatorProxy =
new ClientNamenodeProtocolTranslatorPB(proxy);
return (ClientProtocol) RetryProxy.create(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066e45bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index 4977015..525d84e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -74,7 +73,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -380,10 +378,6 @@ public class TestFileCreation {
SimulatedFSDataset.setFactory(conf);
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
- // Force NameNodeProxies' createNNProxyWithClientProtocol to give
- // up file creation after one failure.
- conf.setBoolean(DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES, true);
-
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();