You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/23 23:04:30 UTC
svn commit: r1126751 - in /hadoop/hdfs/branches/branch-0.22: CHANGES.txt
src/java/org/apache/hadoop/hdfs/DFSClient.java
src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Author: todd
Date: Mon May 23 21:04:29 2011
New Revision: 1126751
URL: http://svn.apache.org/viewvc?rev=1126751&view=rev
Log:
HDFS-1965. IPCs done using block token-based tickets can't reuse connections. Contributed by Todd Lipcon.
Modified:
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Mon May 23 21:04:29 2011
@@ -534,6 +534,9 @@ Release 0.22.0 - Unreleased
HDFS-1978. All but first option in LIBHDFS_OPTS is ignored. (eli)
+ HDFS-1965. IPCs done using block token-based tickets can't reuse
+ connections (todd)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java Mon May 23 21:04:29 2011
@@ -48,6 +48,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -195,12 +196,24 @@ public class DFSClient implements FSCons
if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
ClientDatanodeProtocol.LOG.info("ClientDatanodeProtocol addr=" + addr);
}
+
+ // Since we're creating a new UserGroupInformation here, we know that no
+ // future RPC proxies will be able to re-use the same connection. And
+ // usages of this proxy tend to be one-off calls.
+ //
+ // This is a temporary fix: callers should really achieve this by using
+ // RPC.stopProxy() on the resulting object, but this is currently not
+ // working in trunk. See the discussion on HDFS-1965.
+ Configuration confWithNoIpcIdle = new Configuration(conf);
+ confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
+
UserGroupInformation ticket = UserGroupInformation
.createRemoteUser(locatedBlock.getBlock().toString());
ticket.addToken(locatedBlock.getBlockToken());
return (ClientDatanodeProtocol)RPC.getProxy(ClientDatanodeProtocol.class,
- ClientDatanodeProtocol.versionID, addr, ticket, conf, NetUtils
- .getDefaultSocketFactory(conf), socketTimeout);
+ ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle,
+ NetUtils.getDefaultSocketFactory(conf), socketTimeout);
}
/**
Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java Mon May 23 21:04:29 2011
@@ -46,7 +46,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -55,6 +57,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
@@ -472,6 +475,14 @@ public class DFSTestUtil {
FSDataOutputStream out) {
return ((DFSOutputStream) out.getWrappedStream()).getBlockToken();
}
+
+ public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
+ DatanodeID datanodeid, Configuration conf, int socketTimeout,
+ LocatedBlock locatedBlock)
+ throws IOException {
+ return DFSClient.createClientDatanodeProtocolProxy(
+ datanodeid, conf, socketTimeout, locatedBlock);
+ }
static void setLogLevel2All(org.apache.commons.logging.Log log) {
((org.apache.commons.logging.impl.Log4JLogger)log
Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Mon May 23 21:04:29 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.security.
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.EnumSet;
@@ -29,8 +30,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.io.TestWritable;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.RPC;
@@ -45,6 +51,7 @@ import org.apache.hadoop.security.token.
import org.apache.log4j.Level;
import org.junit.Test;
+import org.junit.Assume;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import static org.junit.Assert.*;
@@ -77,6 +84,9 @@ public class TestBlockToken {
((Log4JLogger) SaslRpcServer.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger) SaslInputStream.LOG).getLogger().setLevel(Level.ALL);
}
+
+ /** Directory where we can count our open file descriptors under Linux */
+ static File FD_DIR = new File("/proc/self/fd/");
long blockKeyUpdateInterval = 10 * 60 * 1000; // 10 mins
long blockTokenLifetime = 2 * 60 * 1000; // 2 mins
@@ -180,14 +190,9 @@ public class TestBlockToken {
slaveHandler.setKeys(keys);
tokenGenerationAndVerification(masterHandler, slaveHandler);
}
-
- @Test
- public void testBlockTokenRpc() throws Exception {
- BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
- blockKeyUpdateInterval, blockTokenLifetime);
- Token<BlockTokenIdentifier> token = sm.generateToken(block3,
- EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
-
+
+ private Server createMockDatanode(BlockTokenSecretManager sm,
+ Token<BlockTokenIdentifier> token) throws IOException {
ClientDatanodeProtocol mockDN = mock(ClientDatanodeProtocol.class);
when(mockDN.getProtocolVersion(anyString(), anyLong())).thenReturn(
ClientDatanodeProtocol.versionID);
@@ -197,8 +202,18 @@ public class TestBlockToken {
doAnswer(new getLengthAnswer(sm, id)).when(mockDN).getReplicaVisibleLength(
any(Block.class));
- final Server server = RPC.getServer(ClientDatanodeProtocol.class, mockDN,
+ return RPC.getServer(ClientDatanodeProtocol.class, mockDN,
ADDRESS, 0, 5, true, conf, sm);
+ }
+
+ @Test
+ public void testBlockTokenRpc() throws Exception {
+ BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
+ blockKeyUpdateInterval, blockTokenLifetime);
+ Token<BlockTokenIdentifier> token = sm.generateToken(block3,
+ EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
+
+ final Server server = createMockDatanode(sm, token);
server.start();
@@ -209,7 +224,7 @@ public class TestBlockToken {
ClientDatanodeProtocol proxy = null;
try {
- proxy = (ClientDatanodeProtocol) RPC.getProxy(
+ proxy = (ClientDatanodeProtocol)RPC.getProxy(
ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr,
ticket, conf, NetUtils.getDefaultSocketFactory(conf));
assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
@@ -221,4 +236,62 @@ public class TestBlockToken {
}
}
+ /**
+ * Test that fast repeated invocations of createClientDatanodeProtocolProxy
+ * will not end up using up thousands of sockets. This is a regression test for
+ * HDFS-1965.
+ */
+ @Test
+ public void testBlockTokenRpcLeak() throws Exception {
+ Assume.assumeTrue(FD_DIR.exists());
+ BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
+ blockKeyUpdateInterval, blockTokenLifetime);
+ Token<BlockTokenIdentifier> token = sm.generateToken(block3,
+ EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
+
+ final Server server = createMockDatanode(sm, token);
+ server.start();
+
+ // a different server just used to trigger the behavior
+ // where RPC.stopProxy doesn't work.
+ final Server serverTwo = createMockDatanode(sm, token);
+ serverTwo.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ DatanodeID fakeDnId = new DatanodeID(
+ "localhost:" + addr.getPort(), "fake-storage", 0, addr.getPort());
+
+ Block b = new Block(12345L);
+ LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
+ fakeBlock.setBlockToken(token);
+
+ ClientDatanodeProtocol proxy = null;
+
+ int fdsAtStart = countOpenFileDescriptors();
+ try {
+ long endTime = System.currentTimeMillis() + 3000;
+ while (System.currentTimeMillis() < endTime) {
+ proxy = DFSTestUtil.createClientDatanodeProtocolProxy(
+ fakeDnId, conf, 1000, fakeBlock);
+ assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
+ LOG.info("Num open fds:" + countOpenFileDescriptors());
+ }
+
+ int fdsAtEnd = countOpenFileDescriptors();
+
+ if (fdsAtEnd - fdsAtStart > 50) {
+ fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
+ }
+ } finally {
+ server.stop();
+ }
+ }
+
+ /**
+ * @return the current number of file descriptors open by this
+ * process.
+ */
+ private static int countOpenFileDescriptors() throws IOException {
+ return FD_DIR.list().length;
+ }
}