You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/23 23:04:30 UTC

svn commit: r1126751 - in /hadoop/hdfs/branches/branch-0.22: CHANGES.txt src/java/org/apache/hadoop/hdfs/DFSClient.java src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java

Author: todd
Date: Mon May 23 21:04:29 2011
New Revision: 1126751

URL: http://svn.apache.org/viewvc?rev=1126751&view=rev
Log:
HDFS-1965. IPCs done using block token-based tickets can't reuse connections. Contributed by Todd Lipcon.

Modified:
    hadoop/hdfs/branches/branch-0.22/CHANGES.txt
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java
    hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
    hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java

Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Mon May 23 21:04:29 2011
@@ -534,6 +534,9 @@ Release 0.22.0 - Unreleased
 
     HDFS-1978. All but first option in LIBHDFS_OPTS is ignored. (eli)
 
+    HDFS-1965. IPCs done using block token-based tickets can't reuse
+    connections (todd)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/DFSClient.java Mon May 23 21:04:29 2011
@@ -48,6 +48,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -195,12 +196,24 @@ public class DFSClient implements FSCons
     if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
       ClientDatanodeProtocol.LOG.info("ClientDatanodeProtocol addr=" + addr);
     }
+    
+    // Since we're creating a new UserGroupInformation here, we know that no
+    // future RPC proxies will be able to re-use the same connection. And
+    // usages of this proxy tend to be one-off calls.
+    //
+    // This is a temporary fix: callers should really achieve this by using
+    // RPC.stopProxy() on the resulting object, but this is currently not
+    // working in trunk. See the discussion on HDFS-1965.
+    Configuration confWithNoIpcIdle = new Configuration(conf);
+    confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
+        .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
+
     UserGroupInformation ticket = UserGroupInformation
         .createRemoteUser(locatedBlock.getBlock().toString());
     ticket.addToken(locatedBlock.getBlockToken());
     return (ClientDatanodeProtocol)RPC.getProxy(ClientDatanodeProtocol.class,
-        ClientDatanodeProtocol.versionID, addr, ticket, conf, NetUtils
-        .getDefaultSocketFactory(conf), socketTimeout);
+        ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle,
+        NetUtils.getDefaultSocketFactory(conf), socketTimeout);
   }
         
   /**

Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java Mon May 23 21:04:29 2011
@@ -46,7 +46,9 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -55,6 +57,7 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -472,6 +475,14 @@ public class DFSTestUtil {
       FSDataOutputStream out) {
     return ((DFSOutputStream) out.getWrappedStream()).getBlockToken();
   }
+  
+  public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
+      DatanodeID datanodeid, Configuration conf, int socketTimeout,
+      LocatedBlock locatedBlock)
+      throws IOException {
+    return DFSClient.createClientDatanodeProtocolProxy(
+        datanodeid, conf, socketTimeout, locatedBlock);
+  }
 
   static void setLogLevel2All(org.apache.commons.logging.Log log) {
     ((org.apache.commons.logging.impl.Log4JLogger)log

Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1126751&r1=1126750&r2=1126751&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Mon May 23 21:04:29 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.security.
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.EnumSet;
@@ -29,8 +30,13 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.io.TestWritable;
 import org.apache.hadoop.ipc.Client;
 import org.apache.hadoop.ipc.RPC;
@@ -45,6 +51,7 @@ import org.apache.hadoop.security.token.
 import org.apache.log4j.Level;
 
 import org.junit.Test;
+import org.junit.Assume;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
 import static org.junit.Assert.*;
@@ -77,6 +84,9 @@ public class TestBlockToken {
     ((Log4JLogger) SaslRpcServer.LOG).getLogger().setLevel(Level.ALL);
     ((Log4JLogger) SaslInputStream.LOG).getLogger().setLevel(Level.ALL);
   }
+  
+  /** Directory where we can count our open file descriptors under Linux */
+  static File FD_DIR = new File("/proc/self/fd/");  
 
   long blockKeyUpdateInterval = 10 * 60 * 1000; // 10 mins
   long blockTokenLifetime = 2 * 60 * 1000; // 2 mins
@@ -180,14 +190,9 @@ public class TestBlockToken {
     slaveHandler.setKeys(keys);
     tokenGenerationAndVerification(masterHandler, slaveHandler);
   }
-
-  @Test
-  public void testBlockTokenRpc() throws Exception {
-    BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
-        blockKeyUpdateInterval, blockTokenLifetime);
-    Token<BlockTokenIdentifier> token = sm.generateToken(block3,
-        EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
-
+  
+  private Server createMockDatanode(BlockTokenSecretManager sm,
+      Token<BlockTokenIdentifier> token) throws IOException {
     ClientDatanodeProtocol mockDN = mock(ClientDatanodeProtocol.class);
     when(mockDN.getProtocolVersion(anyString(), anyLong())).thenReturn(
         ClientDatanodeProtocol.versionID);
@@ -197,8 +202,18 @@ public class TestBlockToken {
     doAnswer(new getLengthAnswer(sm, id)).when(mockDN).getReplicaVisibleLength(
         any(Block.class));
 
-    final Server server = RPC.getServer(ClientDatanodeProtocol.class, mockDN,
+    return RPC.getServer(ClientDatanodeProtocol.class, mockDN,
         ADDRESS, 0, 5, true, conf, sm);
+  }
+
+  @Test
+  public void testBlockTokenRpc() throws Exception {
+    BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
+        blockKeyUpdateInterval, blockTokenLifetime);
+    Token<BlockTokenIdentifier> token = sm.generateToken(block3,
+        EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
+
+    final Server server = createMockDatanode(sm, token);
 
     server.start();
 
@@ -209,7 +224,7 @@ public class TestBlockToken {
 
     ClientDatanodeProtocol proxy = null;
     try {
-      proxy = (ClientDatanodeProtocol) RPC.getProxy(
+      proxy = (ClientDatanodeProtocol)RPC.getProxy(
           ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr,
           ticket, conf, NetUtils.getDefaultSocketFactory(conf));
       assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
@@ -221,4 +236,62 @@ public class TestBlockToken {
     }
   }
 
+  /**
+   * Test that fast repeated invocations of createClientDatanodeProtocolProxy
+   * will not end up using up thousands of sockets. This is a regression test for
+   * HDFS-1965.
+   */
+  @Test
+  public void testBlockTokenRpcLeak() throws Exception {
+    Assume.assumeTrue(FD_DIR.exists());
+    BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
+        blockKeyUpdateInterval, blockTokenLifetime);
+    Token<BlockTokenIdentifier> token = sm.generateToken(block3,
+        EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
+    
+    final Server server = createMockDatanode(sm, token);
+    server.start();
+
+    // a different server just used to trigger the behavior
+    // where RPC.stopProxy doesn't work.
+    final Server serverTwo = createMockDatanode(sm, token);
+    serverTwo.start();
+
+    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+    DatanodeID fakeDnId = new DatanodeID(
+        "localhost:" + addr.getPort(), "fake-storage", 0, addr.getPort());
+    
+    Block b = new Block(12345L);
+    LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
+    fakeBlock.setBlockToken(token);
+
+    ClientDatanodeProtocol proxy = null;
+
+    int fdsAtStart = countOpenFileDescriptors();
+    try {
+      long endTime = System.currentTimeMillis() + 3000;
+      while (System.currentTimeMillis() < endTime) {
+        proxy = DFSTestUtil.createClientDatanodeProtocolProxy(
+            fakeDnId, conf, 1000, fakeBlock);
+        assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
+        LOG.info("Num open fds:" + countOpenFileDescriptors());
+      }
+
+      int fdsAtEnd = countOpenFileDescriptors();
+      
+      if (fdsAtEnd - fdsAtStart > 50) {
+        fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
+      }
+    } finally {
+      server.stop();
+    }
+  }
+
+  /**
+   * @return the current number of file descriptors open by this
+   * process.
+   */
+  private static int countOpenFileDescriptors() throws IOException {
+    return FD_DIR.list().length;
+  }
 }