You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2012/01/10 08:14:16 UTC
svn commit: r1229479 - in
/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/ src/...
Author: jitendra
Date: Tue Jan 10 07:14:15 2012
New Revision: 1229479
URL: http://svn.apache.org/viewvc?rev=1229479&view=rev
Log:
Merged r1179877 from trunk for HDFS-2181.
Added:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/
- copied from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/
- copied from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java
- copied, changed from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java
- copied, changed from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java
- copied, changed from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeWireProtocol.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeWireProtocol.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ContentSummaryWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ContentSummaryWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/CorruptFileBlocksWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/CorruptFileBlocksWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeIDWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeIDWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeInfoWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeInfoWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DirectoryListingWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DirectoryListingWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ExtendedBlockWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ExtendedBlockWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsPermissionWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsPermissionWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsServerDefaultsWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsServerDefaultsWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsFileStatusWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsFileStatusWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsLocatedFileStatusWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsLocatedFileStatusWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlockWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlockWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlocksWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlocksWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ProtocolSignatureWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ProtocolSignatureWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/TokenWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/TokenWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/UpgradeStatusReportWritable.java
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/UpgradeStatusReportWritable.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/overview.html
- copied unchanged from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/overview.html
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jan 10 07:14:15 2012
@@ -12,6 +12,8 @@ Release 0.23-PB - Unreleased
HDFS-2351 Change Namenode and Datanode to register each of their protocols seperately (Sanjay Radia)
HDFS-2337. DFSClient shouldn't keep multiple RPC proxy references (atm)
+
+ HDFS-2181. Separate HDFS Client wire protocol data types (sanjay)
Release 0.23.1 - UNRELEASED
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Tue Jan 10 07:14:15 2012
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs;
import java.io.BufferedOutputStream;
+import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
@@ -290,7 +291,7 @@ public class DFSClient implements java.i
this.clientName = leaserenewer.getClientName(dfsClientConf.taskId);
this.socketCache = new SocketCache(dfsClientConf.socketCacheCapacity);
if (nameNodeAddr != null && rpcNamenode == null) {
- this.namenode = DFSUtil.createNamenode(nameNodeAddr, conf);
+ this.namenode = DFSUtil.createNamenode(nameNodeAddr, conf, ugi);
} else if (nameNodeAddr == null && rpcNamenode != null) {
//This case is used for testing.
this.namenode = rpcNamenode;
@@ -378,12 +379,31 @@ public class DFSClient implements java.i
namenode.renewLease(clientName);
}
}
-
+
+ /**
+ * Close connections the Namenode.
+ * The namenode variable is either a rpcProxy passed by a test or
+ * created using the protocolTranslator which is closeable.
+ * If closeable then call close, else close using RPC.stopProxy().
+ */
+ void closeConnectionToNamenode() {
+ if (namenode instanceof Closeable) {
+ try {
+ ((Closeable) namenode).close();
+ return;
+ } catch (IOException e) {
+ // fall through - lets try the stopProxy
+ LOG.warn("Exception closing namenode, stopping the proxy");
+ }
+ }
+ RPC.stopProxy(namenode);
+ }
+
/** Abort and release resources held. Ignore all errors. */
void abort() {
clientRunning = false;
closeAllFilesBeingWritten(true);
- RPC.stopProxy(namenode); // close connections to the namenode
+ closeConnectionToNamenode();
}
/** Close/abort all files being written. */
@@ -423,7 +443,7 @@ public class DFSClient implements java.i
clientRunning = false;
leaserenewer.closeClient(this);
// close connections to the namenode
- RPC.stopProxy(namenode);
+ closeConnectionToNamenode();
}
}
@@ -604,7 +624,7 @@ public class DFSClient implements java.i
LOG.info("Renewing " +
DelegationTokenIdentifier.stringifyToken(delToken));
ClientProtocol nn =
- DFSUtil.createRPCNamenode
+ DFSUtil.createNamenode
(NameNode.getAddress(token.getService().toString()),
conf, UserGroupInformation.getCurrentUser());
try {
@@ -622,7 +642,7 @@ public class DFSClient implements java.i
(Token<DelegationTokenIdentifier>) token;
LOG.info("Cancelling " +
DelegationTokenIdentifier.stringifyToken(delToken));
- ClientProtocol nn = DFSUtil.createRPCNamenode(
+ ClientProtocol nn = DFSUtil.createNamenode(
NameNode.getAddress(token.getService().toString()), conf,
UserGroupInformation.getCurrentUser());
try {
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Tue Jan 10 07:14:15 2012
@@ -41,11 +41,12 @@ import java.util.Random;
import java.util.StringTokenizer;
import java.util.concurrent.TimeUnit;
+import javax.net.SocketFactory;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
@@ -626,25 +627,29 @@ public class DFSUtil {
/** Create a {@link NameNode} proxy */
- public static ClientProtocol createNamenode(Configuration conf) throws IOException {
+ public static ClientProtocol createNamenode(Configuration conf)
+ throws IOException {
return createNamenode(NameNode.getAddress(conf), conf);
}
/** Create a {@link NameNode} proxy */
public static ClientProtocol createNamenode( InetSocketAddress nameNodeAddr,
- Configuration conf) throws IOException {
- return createNamenode(createRPCNamenode(nameNodeAddr, conf,
- UserGroupInformation.getCurrentUser()));
-
+ Configuration conf) throws IOException {
+ return createNamenode(nameNodeAddr, conf,
+ UserGroupInformation.getCurrentUser());
}
-
+
/** Create a {@link NameNode} proxy */
- static ClientProtocol createRPCNamenode(InetSocketAddress nameNodeAddr,
- Configuration conf, UserGroupInformation ugi)
- throws IOException {
- return (ClientProtocol)RPC.getProxy(ClientProtocol.class,
- ClientProtocol.versionID, nameNodeAddr, ugi, conf,
- NetUtils.getSocketFactory(conf, ClientProtocol.class));
+ public static ClientProtocol createNamenode( InetSocketAddress nameNodeAddr,
+ Configuration conf, UserGroupInformation ugi) throws IOException {
+ /**
+ * Currently we have simply burnt-in support for a SINGLE
+ * protocol - protocolR23Compatible. This will be replaced
+ * by a way to pick the right protocol based on the
+ * version of the target server.
+ */
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientNamenodeProtocolTranslatorR23(nameNodeAddr, conf, ugi);
}
/** Create a {@link NameNode} proxy */
@@ -690,31 +695,18 @@ public class DFSUtil {
/** Create a {@link ClientDatanodeProtocol} proxy */
public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
DatanodeID datanodeid, Configuration conf, int socketTimeout,
- LocatedBlock locatedBlock)
- throws IOException {
- InetSocketAddress addr = NetUtils.createSocketAddr(
- datanodeid.getHost() + ":" + datanodeid.getIpcPort());
- if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
- ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr);
- }
-
- // Since we're creating a new UserGroupInformation here, we know that no
- // future RPC proxies will be able to re-use the same connection. And
- // usages of this proxy tend to be one-off calls.
- //
- // This is a temporary fix: callers should really achieve this by using
- // RPC.stopProxy() on the resulting object, but this is currently not
- // working in trunk. See the discussion on HDFS-1965.
- Configuration confWithNoIpcIdle = new Configuration(conf);
- confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
- .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
-
- UserGroupInformation ticket = UserGroupInformation
- .createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
- ticket.addToken(locatedBlock.getBlockToken());
- return (ClientDatanodeProtocol)RPC.getProxy(ClientDatanodeProtocol.class,
- ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle,
- NetUtils.getDefaultSocketFactory(conf), socketTimeout);
+ LocatedBlock locatedBlock) throws IOException {
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientDatanodeProtocolTranslatorR23(datanodeid, conf, socketTimeout,
+ locatedBlock);
+ }
+
+ /** Create a {@link ClientDatanodeProtocol} proxy */
+ public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory) throws IOException {
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientDatanodeProtocolTranslatorR23(addr, ticket, conf, factory);
}
/**
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java Tue Jan 10 07:14:15 2012
@@ -41,8 +41,30 @@ import org.apache.hadoop.security.token.
public interface ClientDatanodeProtocol extends VersionedProtocol {
public static final Log LOG = LogFactory.getLog(ClientDatanodeProtocol.class);
+
/**
+ * Until version 9, this class ClientDatanodeProtocol served as both
+ * the client interface to the DN AND the RPC protocol used to
+ * communicate with the NN.
+ *
+ * Post version 10 (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/ClientDatanodeWireProtocol}
+ *
+ * This class is used by both the DFSClient and the
+ * DN server side to insulate from the protocol serialization.
+ *
+ * If you are adding/changing DN's interface then you need to
+ * change both this class and ALSO
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol}.
+ * These changes need to be done in a compatible fashion as described in
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}
+ *
+ * The log of historical changes can be retrieved from the svn).
* 9: Added deleteBlockPool method
+ *
+ * 9 is the last version id when this class was used for protocols
+ * serialization. DO not update this version any further.
+ * Changes are recorded in R23 classes.
*/
public static final long versionID = 9L;
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Tue Jan 10 07:14:15 2012
@@ -64,10 +64,28 @@ import org.apache.hadoop.hdfs.security.t
public interface ClientProtocol extends VersionedProtocol {
/**
- * Compared to the previous version the following changes have been introduced:
- * (Only the latest change is reflected.
+ * Until version 69, this class ClientProtocol served as both
+ * the client interface to the NN AND the RPC protocol used to
+ * communicate with the NN.
+ *
+ * Post version 70 (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
+ *
+ * This class is used by both the DFSClient and the
+ * NN server side to insulate from the protocol serialization.
+ *
+ * If you are adding/changing NN's interface then you need to
+ * change both this class and ALSO
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}.
+ * These changes need to be done in a compatible fashion as described in
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}
+ *
* The log of historical changes can be retrieved from the svn).
* 69: Eliminate overloaded method names.
+ *
+ * 69L is the last version id when this class was used for protocols
+ * serialization. DO not update this version any further.
+ * Changes are recorded in R23 classes.
*/
public static final long versionID = 69L;
@@ -365,11 +383,8 @@ public interface ClientProtocol extends
* @return true if successful, or false if the old name does not exist
* or if the new name already belongs to the namespace.
*
- * @throws IOException an I/O error occurred
- *
- * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
+ * @throws IOException an I/O error occurred
*/
- @Deprecated
public boolean rename(String src, String dst)
throws UnresolvedLinkException, IOException;
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Tue Jan 10 07:14:15 2012
@@ -75,6 +75,13 @@ public class DatanodeInfo extends Datano
public String toString() {
return value;
}
+
+ public static AdminStates fromValue(final String value) {
+ for (AdminStates as : AdminStates.values()) {
+ if (as.value.equals(value)) return as;
+ }
+ return NORMAL;
+ }
}
@Nullable
@@ -110,11 +117,20 @@ public class DatanodeInfo extends Datano
this.adminState = null;
}
- protected DatanodeInfo(DatanodeID nodeID, String location, String hostName) {
+ public DatanodeInfo(DatanodeID nodeID, String location, String hostName) {
this(nodeID);
this.location = location;
this.hostName = hostName;
}
+
+ public DatanodeInfo(DatanodeID nodeID, String location, String hostName,
+ final long capacity, final long dfsUsed, final long remaining,
+ final long blockPoolUsed, final long lastUpdate, final int xceiverCount,
+ final AdminStates adminState) {
+ this(nodeID.getName(), nodeID.getStorageID(), nodeID.getInfoPort(), nodeID
+ .getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed, lastUpdate,
+ xceiverCount, location, hostName, adminState);
+ }
/** Constructor */
public DatanodeInfo(final String name, final String storageID,
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Tue Jan 10 07:14:15 2012
@@ -30,7 +30,16 @@ public class HdfsConstants {
/* Hidden constructor */
protected HdfsConstants() {
}
-
+
+ /**
+ * HDFS Protocol Names:
+ */
+ public static final String CLIENT_NAMENODE_PROTOCOL_NAME =
+ "org.apache.hadoop.hdfs.protocol.ClientProtocol";
+ public static final String CLIENT_DATANODE_PROTOCOL_NAME =
+ "org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
+
+
public static int MIN_BLOCKS_FOR_WRITE = 5;
// Long that indicates "leave current quota unchanged"
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java Tue Jan 10 07:14:15 2012
@@ -241,6 +241,10 @@ public class HdfsFileStatus implements W
final public String getSymlink() {
return DFSUtil.bytes2String(symlink);
}
+
+ final public byte[] getSymlinkInBytes() {
+ return symlink;
+ }
//////////////////////////////////////////////////
// Writable
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java Tue Jan 10 07:14:15 2012
@@ -54,6 +54,11 @@ public class LocatedBlock implements Wri
public LocatedBlock() {
this(new ExtendedBlock(), new DatanodeInfo[0], 0L, false);
}
+
+
+ public LocatedBlock(ExtendedBlock eb) {
+ this(eb, new DatanodeInfo[0], 0L, false);
+ }
public LocatedBlock(String bpid, Block b, DatanodeInfo[] locs) {
this(new ExtendedBlock(bpid, b), locs, -1, false); // startOffset is unknown
Copied: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java (from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java?p2=hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java&r1=1179877&r2=1229479&rev=1229479&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java Tue Jan 10 07:14:15 2012
@@ -21,9 +21,13 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.token.Token;
/**
* This class is used on the server side.
@@ -116,4 +120,10 @@ public class ClientDatanodeProtocolServe
public void deleteBlockPool(String bpid, boolean force) throws IOException {
server.deleteBlockPool(bpid, force);
}
+
+ @Override
+ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block,
+ Token<BlockTokenIdentifier> token) throws IOException {
+ return server.getBlockLocalPathInfo(block, token);
+ }
}
\ No newline at end of file
Copied: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java (from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java?p2=hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java&r1=1179877&r2=1229479&rev=1229479&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java Tue Jan 10 07:14:15 2012
@@ -26,14 +26,17 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
/**
@@ -134,4 +137,9 @@ public class ClientDatanodeProtocolTrans
}
+ @Override
+ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block,
+ Token<BlockTokenIdentifier> token) throws IOException {
+ return rpcProxy.getBlockLocalPathInfo(block, token);
+ }
}
Copied: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java (from r1179877, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java?p2=hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java&p1=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java&r1=1179877&r2=1229479&rev=1229479&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java Tue Jan 10 07:14:15 2012
@@ -24,11 +24,15 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
/**
@@ -75,7 +79,15 @@ public interface ClientDatanodeWireProto
* The specification of this method matches that of
* {@link org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol#deleteBlockPool(String, boolean)}
*/
- void deleteBlockPool(String bpid, boolean force) throws IOException;
+ void deleteBlockPool(String bpid, boolean force) throws IOException;
+
+ /**
+ * The specification of this method matches that of
+ * {@link org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol
+ * #getBlockLocalPathInfo(ExtendedBlock, Token)}
+ */
+ BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block,
+ Token<BlockTokenIdentifier> token) throws IOException;
/**
* This method is defined to get the protocol signature using
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Tue Jan 10 07:14:15 2012
@@ -105,6 +105,7 @@ import org.apache.hadoop.hdfs.protocol.d
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DNTransferAckProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
+import org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeProtocolServerSideTranslatorR23;
import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@@ -516,8 +517,13 @@ public class DataNode extends Configured
InetSocketAddress ipcAddr = NetUtils.createSocketAddr(
conf.get("dfs.datanode.ipc.address"));
- // Add all the RPC protocols that the Datanode implements
- ipcServer = RPC.getServer(ClientDatanodeProtocol.class, this, ipcAddr.getHostName(),
+ // Add all the RPC protocols that the Datanode implements
+ ClientDatanodeProtocolServerSideTranslatorR23
+ clientDatanodeProtocolServerTranslator =
+ new ClientDatanodeProtocolServerSideTranslatorR23(this);
+ ipcServer = RPC.getServer(
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.class,
+ clientDatanodeProtocolServerTranslator, ipcAddr.getHostName(),
ipcAddr.getPort(),
conf.getInt(DFS_DATANODE_HANDLER_COUNT_KEY,
DFS_DATANODE_HANDLER_COUNT_DEFAULT),
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Tue Jan 10 07:14:15 2012
@@ -57,6 +57,8 @@ import org.apache.hadoop.hdfs.protocol.U
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
+import org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol;
+import org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeProtocolServerSideTranslatorR23;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
@@ -143,10 +145,13 @@ class NameNodeRpcServer implements Namen
serviceRPCAddress = null;
}
// Add all the RPC protocols that the namenode implements
- this.server = RPC.getServer(ClientProtocol.class, this,
- socAddr.getHostName(), socAddr.getPort(),
- handlerCount, false, conf,
- namesystem.getDelegationTokenSecretManager());
+ ClientNamenodeProtocolServerSideTranslatorR23 clientProtocolServerTranslator =
+ new ClientNamenodeProtocolServerSideTranslatorR23(this);
+ this.server = RPC.getServer(
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol.class,
+ clientProtocolServerTranslator, socAddr.getHostName(),
+ socAddr.getPort(), handlerCount, false, conf,
+ namesystem.getDelegationTokenSecretManager());
this.server.addProtocol(DatanodeProtocol.class, this);
this.server.addProtocol(NamenodeProtocol.class, this);
this.server.addProtocol(RefreshAuthorizationPolicyProtocol.class, this);
@@ -210,7 +215,8 @@ class NameNodeRpcServer implements Namen
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
if (protocol.equals(ClientProtocol.class.getName())) {
- return ClientProtocol.versionID;
+ throw new IOException("Old Namenode Client protocol is not supported:" +
+ protocol + "Switch your clientside to " + ClientNamenodeWireProtocol.class);
} else if (protocol.equals(DatanodeProtocol.class.getName())){
return DatanodeProtocol.versionID;
} else if (protocol.equals(NamenodeProtocol.class.getName())){
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Tue Jan 10 07:14:15 2012
@@ -96,7 +96,7 @@ public class TestDFSClientRetries extend
}
@Override
- public Writable call(Class<?> protocol, Writable param, long receiveTime)
+ public Writable call(String protocol, Writable param, long receiveTime)
throws IOException {
if (sleep) {
// sleep a bit
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Tue Jan 10 07:14:15 2012
@@ -51,12 +51,12 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.io.TestWritable;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtocolSignature;
@@ -96,9 +96,9 @@ public class TestBlockToken {
((Log4JLogger) SaslRpcServer.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger) SaslInputStream.LOG).getLogger().setLevel(Level.ALL);
}
-
+
/** Directory where we can count our open file descriptors under Linux */
- static File FD_DIR = new File("/proc/self/fd/");
+ static File FD_DIR = new File("/proc/self/fd/");
long blockKeyUpdateInterval = 10 * 60 * 1000; // 10 mins
long blockTokenLifetime = 2 * 60 * 1000; // 2 mins
@@ -120,7 +120,8 @@ public class TestBlockToken {
public Long answer(InvocationOnMock invocation) throws IOException {
Object args[] = invocation.getArguments();
assertEquals(1, args.length);
- ExtendedBlock block = (ExtendedBlock) args[0];
+ org.apache.hadoop.hdfs.protocolR23Compatible.ExtendedBlockWritable block =
+ (org.apache.hadoop.hdfs.protocolR23Compatible.ExtendedBlockWritable) args[0];
Set<TokenIdentifier> tokenIds = UserGroupInformation.getCurrentUser()
.getTokenIdentifiers();
assertEquals("Only one BlockTokenIdentifier expected", 1, tokenIds.size());
@@ -129,7 +130,9 @@ public class TestBlockToken {
BlockTokenIdentifier id = (BlockTokenIdentifier) tokenId;
LOG.info("Got: " + id.toString());
assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
- sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
+ sm.checkAccess(id, null, org.apache.hadoop.hdfs.protocolR23Compatible.
+ ExtendedBlockWritable.convertExtendedBlock(block),
+ BlockTokenSecretManager.AccessMode.WRITE);
result = id.getBlockId();
}
return result;
@@ -137,7 +140,8 @@ public class TestBlockToken {
}
private BlockTokenIdentifier generateTokenId(BlockTokenSecretManager sm,
- ExtendedBlock block, EnumSet<BlockTokenSecretManager.AccessMode> accessModes)
+ ExtendedBlock block,
+ EnumSet<BlockTokenSecretManager.AccessMode> accessModes)
throws IOException {
Token<BlockTokenIdentifier> token = sm.generateToken(block, accessModes);
BlockTokenIdentifier id = sm.createIdentifier();
@@ -151,12 +155,12 @@ public class TestBlockToken {
TestWritable.testWritable(new BlockTokenIdentifier());
BlockTokenSecretManager sm = new BlockTokenSecretManager(true,
blockKeyUpdateInterval, blockTokenLifetime);
- TestWritable.testWritable(generateTokenId(sm, block1, EnumSet
- .allOf(BlockTokenSecretManager.AccessMode.class)));
- TestWritable.testWritable(generateTokenId(sm, block2, EnumSet
- .of(BlockTokenSecretManager.AccessMode.WRITE)));
- TestWritable.testWritable(generateTokenId(sm, block3, EnumSet
- .noneOf(BlockTokenSecretManager.AccessMode.class)));
+ TestWritable.testWritable(generateTokenId(sm, block1,
+ EnumSet.allOf(BlockTokenSecretManager.AccessMode.class)));
+ TestWritable.testWritable(generateTokenId(sm, block2,
+ EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE)));
+ TestWritable.testWritable(generateTokenId(sm, block3,
+ EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class)));
}
private void tokenGenerationAndVerification(BlockTokenSecretManager master,
@@ -176,8 +180,8 @@ public class TestBlockToken {
slave.checkAccess(token2, null, block2, mode);
}
// multi-mode tokens
- Token<BlockTokenIdentifier> mtoken = master.generateToken(block3, EnumSet
- .allOf(BlockTokenSecretManager.AccessMode.class));
+ Token<BlockTokenIdentifier> mtoken = master.generateToken(block3,
+ EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
for (BlockTokenSecretManager.AccessMode mode : BlockTokenSecretManager.AccessMode
.values()) {
master.checkAccess(mtoken, null, block3, mode);
@@ -202,25 +206,28 @@ public class TestBlockToken {
slaveHandler.setKeys(keys);
tokenGenerationAndVerification(masterHandler, slaveHandler);
}
-
+
private Server createMockDatanode(BlockTokenSecretManager sm,
Token<BlockTokenIdentifier> token) throws IOException {
- ClientDatanodeProtocol mockDN = mock(ClientDatanodeProtocol.class);
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol mockDN =
+ mock(org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.class);
when(mockDN.getProtocolVersion(anyString(), anyLong())).thenReturn(
- ClientDatanodeProtocol.versionID);
- doReturn(ProtocolSignature.getProtocolSignature(
- mockDN, ClientDatanodeProtocol.class.getName(),
- ClientDatanodeProtocol.versionID, 0))
- .when(mockDN).getProtocolSignature(anyString(), anyLong(), anyInt());
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.versionID);
+ doReturn(
+ ProtocolSignature.getProtocolSignature(mockDN,
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.class.getName(),
+ org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.versionID, 0)).when(mockDN)
+ .getProtocolSignature(anyString(), anyLong(), anyInt());
BlockTokenIdentifier id = sm.createIdentifier();
id.readFields(new DataInputStream(new ByteArrayInputStream(token
.getIdentifier())));
doAnswer(new getLengthAnswer(sm, id)).when(mockDN).getReplicaVisibleLength(
- any(ExtendedBlock.class));
+ any(org.apache.hadoop.hdfs.protocolR23Compatible.ExtendedBlockWritable.class));
- return RPC.getServer(ClientDatanodeProtocol.class, mockDN,
- ADDRESS, 0, 5, true, conf, sm);
+ return RPC.getServer(org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol.class,
+ mockDN, ADDRESS, 0, 5,
+ true, conf, sm);
}
@Test
@@ -241,9 +248,8 @@ public class TestBlockToken {
ClientDatanodeProtocol proxy = null;
try {
- proxy = RPC.getProxy(
- ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID, addr,
- ticket, conf, NetUtils.getDefaultSocketFactory(conf));
+ proxy = DFSUtil.createClientDatanodeProtocolProxy(addr, ticket, conf,
+ NetUtils.getDefaultSocketFactory(conf));
assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
} finally {
server.stop();
@@ -255,8 +261,8 @@ public class TestBlockToken {
/**
* Test that fast repeated invocations of createClientDatanodeProtocolProxy
- * will not end up using up thousands of sockets. This is a regression test for
- * HDFS-1965.
+ * will not end up using up thousands of sockets. This is a regression test
+ * for HDFS-1965.
*/
@Test
public void testBlockTokenRpcLeak() throws Exception {
@@ -270,9 +276,9 @@ public class TestBlockToken {
server.start();
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- DatanodeID fakeDnId = new DatanodeID(
- "localhost:" + addr.getPort(), "fake-storage", 0, addr.getPort());
-
+ DatanodeID fakeDnId = new DatanodeID("localhost:" + addr.getPort(),
+ "fake-storage", 0, addr.getPort());
+
ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.setBlockToken(token);
@@ -282,19 +288,19 @@ public class TestBlockToken {
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
- ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
+ ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
new InetSocketAddress("1.1.1.1", 1),
- UserGroupInformation.createRemoteUser("junk"),
- conf, NetUtils.getDefaultSocketFactory(conf));
-
+ UserGroupInformation.createRemoteUser("junk"), conf,
+ NetUtils.getDefaultSocketFactory(conf));
+
ClientDatanodeProtocol proxy = null;
int fdsAtStart = countOpenFileDescriptors();
try {
long endTime = System.currentTimeMillis() + 3000;
while (System.currentTimeMillis() < endTime) {
- proxy = DFSUtil.createClientDatanodeProtocolProxy(
- fakeDnId, conf, 1000, fakeBlock);
+ proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
+ fakeBlock);
assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
if (proxy != null) {
RPC.stopProxy(proxy);
@@ -303,32 +309,31 @@ public class TestBlockToken {
}
int fdsAtEnd = countOpenFileDescriptors();
-
+
if (fdsAtEnd - fdsAtStart > 50) {
fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
} finally {
server.stop();
}
-
+
RPC.stopProxy(proxyToNoWhere);
}
/**
- * @return the current number of file descriptors open by this
- * process.
+ * @return the current number of file descriptors open by this process.
*/
private static int countOpenFileDescriptors() throws IOException {
return FD_DIR.list().length;
}
- /**
+ /**
* Test {@link BlockPoolTokenSecretManager}
*/
@Test
public void testBlockPoolTokenSecretManager() throws Exception {
BlockPoolTokenSecretManager bpMgr = new BlockPoolTokenSecretManager();
-
+
// Test BlockPoolSecretManager with upto 10 block pools
for (int i = 0; i < 10; i++) {
String bpid = Integer.toString(i);
@@ -337,12 +342,11 @@ public class TestBlockToken {
BlockTokenSecretManager slaveHandler = new BlockTokenSecretManager(false,
blockKeyUpdateInterval, blockTokenLifetime);
bpMgr.addBlockPool(bpid, slaveHandler);
-
-
+
ExportedBlockKeys keys = masterHandler.exportKeys();
bpMgr.setKeys(bpid, keys);
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid));
-
+
// Test key updating
masterHandler.updateKeys();
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid));
@@ -351,11 +355,12 @@ public class TestBlockToken {
tokenGenerationAndVerification(masterHandler, bpMgr.get(bpid));
}
}
-
+
/**
- * This test writes a file and gets the block locations without closing
- * the file, and tests the block token in the last block. Block token is
- * verified by ensuring it is of correct kind.
+ * This test writes a file and gets the block locations without closing the
+ * file, and tests the block token in the last block. Block token is verified
+ * by ensuring it is of correct kind.
+ *
* @throws IOException
* @throws InterruptedException
*/
@@ -389,5 +394,5 @@ public class TestBlockToken {
} finally {
cluster.shutdown();
}
- }
+ }
}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java?rev=1229479&r1=1229478&r2=1229479&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestInterDatanodeProtocol.java Tue Jan 10 07:14:15 2012
@@ -81,7 +81,7 @@ public class TestInterDatanodeProtocol {
}
@Override
- public Writable call(Class<?> protocol, Writable param, long receiveTime)
+ public Writable call(String protocol, Writable param, long receiveTime)
throws IOException {
if (sleep) {
// sleep a bit