You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sr...@apache.org on 2011/10/06 23:58:23 UTC
svn commit: r1179877 [1/3] - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/
src/main/java/or...
Author: sradia
Date: Thu Oct 6 21:58:22 2011
New Revision: 1179877
URL: http://svn.apache.org/viewvc?rev=1179877&view=rev
Log:
HDFS-2181 Separate HDFS Client wire protocol data types (sanjay)
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeWireProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ContentSummaryWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/CorruptFileBlocksWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeIDWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DatanodeInfoWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/DirectoryListingWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ExtendedBlockWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsPermissionWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/FsServerDefaultsWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsFileStatusWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/HdfsLocatedFileStatusWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlockWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/LocatedBlocksWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ProtocolSignatureWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/TokenWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/UpgradeStatusReportWritable.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/overview.html
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Oct 6 21:58:22 2011
@@ -61,6 +61,8 @@ Trunk (unreleased changes)
HDFS-2395. Add a root element in the JSON responses of webhdfs.
(szetszwo)
+ HDFS-2181 Separate HDFS Client wire protocol data types (sanjay)
+
BUG FIXES
HDFS-2287. TestParallelRead has a small off-by-one bug. (todd)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Oct 6 21:58:22 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs;
import java.io.BufferedOutputStream;
+import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
@@ -252,7 +253,7 @@ public class DFSClient implements java.i
this.clientName = leaserenewer.getClientName(dfsClientConf.taskId);
this.socketCache = new SocketCache(dfsClientConf.socketCacheCapacity);
if (nameNodeAddr != null && rpcNamenode == null) {
- this.namenode = DFSUtil.createNamenode(nameNodeAddr, conf);
+ this.namenode = DFSUtil.createNamenode(nameNodeAddr, conf, ugi);
} else if (nameNodeAddr == null && rpcNamenode != null) {
//This case is used for testing.
this.namenode = rpcNamenode;
@@ -333,12 +334,31 @@ public class DFSClient implements java.i
namenode.renewLease(clientName);
}
}
-
+
+ /**
+ * Close connections the Namenode.
+ * The namenode variable is either a rpcProxy passed by a test or
+ * created using the protocolTranslator which is closeable.
+ * If closeable then call close, else close using RPC.stopProxy().
+ */
+ void closeConnectionToNamenode() {
+ if (namenode instanceof Closeable) {
+ try {
+ ((Closeable) namenode).close();
+ return;
+ } catch (IOException e) {
+ // fall through - lets try the stopProxy
+ LOG.warn("Exception closing namenode, stopping the proxy");
+ }
+ }
+ RPC.stopProxy(namenode);
+ }
+
/** Abort and release resources held. Ignore all errors. */
void abort() {
clientRunning = false;
closeAllFilesBeingWritten(true);
- RPC.stopProxy(namenode); // close connections to the namenode
+ closeConnectionToNamenode();
}
/** Close/abort all files being written. */
@@ -378,7 +398,7 @@ public class DFSClient implements java.i
clientRunning = false;
leaserenewer.closeClient(this);
// close connections to the namenode
- RPC.stopProxy(namenode);
+ closeConnectionToNamenode();
}
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Oct 6 21:58:22 2011
@@ -31,33 +31,23 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import java.util.StringTokenizer;
-import java.util.concurrent.TimeUnit;
+import javax.net.SocketFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
@@ -611,80 +601,46 @@ public class DFSUtil {
/** Create a {@link NameNode} proxy */
- public static ClientProtocol createNamenode(Configuration conf) throws IOException {
+ public static ClientProtocol createNamenode(Configuration conf)
+ throws IOException {
return createNamenode(NameNode.getAddress(conf), conf);
}
/** Create a {@link NameNode} proxy */
public static ClientProtocol createNamenode( InetSocketAddress nameNodeAddr,
- Configuration conf) throws IOException {
- return createNamenode(createRPCNamenode(nameNodeAddr, conf,
- UserGroupInformation.getCurrentUser()));
-
- }
-
- /** Create a {@link NameNode} proxy */
- static ClientProtocol createRPCNamenode(InetSocketAddress nameNodeAddr,
- Configuration conf, UserGroupInformation ugi)
- throws IOException {
- return (ClientProtocol)RPC.getProxy(ClientProtocol.class,
- ClientProtocol.versionID, nameNodeAddr, ugi, conf,
- NetUtils.getSocketFactory(conf, ClientProtocol.class));
+ Configuration conf) throws IOException {
+ return createNamenode(nameNodeAddr, conf,
+ UserGroupInformation.getCurrentUser());
}
-
- /** Create a {@link NameNode} proxy */
- static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
- throws IOException {
- RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
- 5, HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
- Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);
-
- Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- exceptionToPolicyMap.put(RemoteException.class,
- RetryPolicies.retryByRemoteException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
- RetryPolicy methodPolicy = RetryPolicies.retryByException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
- Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
-
- methodNameToPolicyMap.put("create", methodPolicy);
-
- return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
- rpcNamenode, methodNameToPolicyMap);
+ /** Create a {@link NameNode} proxy */
+ public static ClientProtocol createNamenode( InetSocketAddress nameNodeAddr,
+ Configuration conf, UserGroupInformation ugi) throws IOException {
+ /**
+ * Currently we have simply burnt-in support for a SINGLE
+ * protocol - protocolR23Compatible. This will be replaced
+ * by a way to pick the right protocol based on the
+ * version of the target server.
+ */
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientNamenodeProtocolTranslatorR23(nameNodeAddr, conf, ugi);
}
/** Create a {@link ClientDatanodeProtocol} proxy */
public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
DatanodeID datanodeid, Configuration conf, int socketTimeout,
- LocatedBlock locatedBlock)
- throws IOException {
- InetSocketAddress addr = NetUtils.createSocketAddr(
- datanodeid.getHost() + ":" + datanodeid.getIpcPort());
- if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
- ClientDatanodeProtocol.LOG.debug("ClientDatanodeProtocol addr=" + addr);
- }
-
- // Since we're creating a new UserGroupInformation here, we know that no
- // future RPC proxies will be able to re-use the same connection. And
- // usages of this proxy tend to be one-off calls.
- //
- // This is a temporary fix: callers should really achieve this by using
- // RPC.stopProxy() on the resulting object, but this is currently not
- // working in trunk. See the discussion on HDFS-1965.
- Configuration confWithNoIpcIdle = new Configuration(conf);
- confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
- .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
-
- UserGroupInformation ticket = UserGroupInformation
- .createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
- ticket.addToken(locatedBlock.getBlockToken());
- return (ClientDatanodeProtocol)RPC.getProxy(ClientDatanodeProtocol.class,
- ClientDatanodeProtocol.versionID, addr, ticket, confWithNoIpcIdle,
- NetUtils.getDefaultSocketFactory(conf), socketTimeout);
+ LocatedBlock locatedBlock) throws IOException {
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientDatanodeProtocolTranslatorR23(datanodeid, conf, socketTimeout,
+ locatedBlock);
+ }
+
+ /** Create a {@link ClientDatanodeProtocol} proxy */
+ public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory) throws IOException {
+ return new org.apache.hadoop.hdfs.protocolR23Compatible.
+ ClientDatanodeProtocolTranslatorR23(addr, ticket, conf, factory);
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java Thu Oct 6 21:58:22 2011
@@ -39,8 +39,30 @@ import org.apache.hadoop.security.token.
public interface ClientDatanodeProtocol extends VersionedProtocol {
public static final Log LOG = LogFactory.getLog(ClientDatanodeProtocol.class);
+
/**
+ * Until version 9, this class ClientDatanodeProtocol served as both
+ * the client interface to the DN AND the RPC protocol used to
+ * communicate with the NN.
+ *
+ * Post version 10 (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/ClientDatanodeWireProtocol}
+ *
+ * This class is used by both the DFSClient and the
+ * DN server side to insulate from the protocol serialization.
+ *
+ * If you are adding/changing DN's interface then you need to
+ * change both this class and ALSO
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientDatanodeWireProtocol}.
+ * These changes need to be done in a compatible fashion as described in
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}
+ *
+ * The log of historical changes can be retrieved from the svn).
* 9: Added deleteBlockPool method
+ *
+ * 9 is the last version id when this class was used for protocols
+ * serialization. DO not update this version any further.
+ * Changes are recorded in R23 classes.
*/
public static final long versionID = 9L;
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Thu Oct 6 21:58:22 2011
@@ -64,10 +64,28 @@ import org.apache.hadoop.hdfs.security.t
public interface ClientProtocol extends VersionedProtocol {
/**
- * Compared to the previous version the following changes have been introduced:
- * (Only the latest change is reflected.
+ * Until version 69, this class ClientProtocol served as both
+ * the client interface to the NN AND the RPC protocol used to
+ * communicate with the NN.
+ *
+ * Post version 70 (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
+ *
+ * This class is used by both the DFSClient and the
+ * NN server side to insulate from the protocol serialization.
+ *
+ * If you are adding/changing NN's interface then you need to
+ * change both this class and ALSO
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}.
+ * These changes need to be done in a compatible fashion as described in
+ * {@link org.apache.hadoop.hdfs.protocolR23Compatible.ClientNamenodeWireProtocol}
+ *
* The log of historical changes can be retrieved from the svn).
* 69: Eliminate overloaded method names.
+ *
+ * 69L is the last version id when this class was used for protocols
+ * serialization. DO not update this version any further.
+ * Changes are recorded in R23 classes.
*/
public static final long versionID = 69L;
@@ -365,11 +383,8 @@ public interface ClientProtocol extends
* @return true if successful, or false if the old name does not exist
* or if the new name already belongs to the namespace.
*
- * @throws IOException an I/O error occurred
- *
- * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
+ * @throws IOException an I/O error occurred
*/
- @Deprecated
public boolean rename(String src, String dst)
throws UnresolvedLinkException, IOException;
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Thu Oct 6 21:58:22 2011
@@ -24,7 +24,6 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.io.WritableComparable;
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Thu Oct 6 21:58:22 2011
@@ -75,6 +75,13 @@ public class DatanodeInfo extends Datano
public String toString() {
return value;
}
+
+ public static AdminStates fromValue(final String value) {
+ for (AdminStates as : AdminStates.values()) {
+ if (as.value.equals(value)) return as;
+ }
+ return NORMAL;
+ }
}
@Nullable
@@ -110,11 +117,20 @@ public class DatanodeInfo extends Datano
this.adminState = null;
}
- protected DatanodeInfo(DatanodeID nodeID, String location, String hostName) {
+ public DatanodeInfo(DatanodeID nodeID, String location, String hostName) {
this(nodeID);
this.location = location;
this.hostName = hostName;
}
+
+ public DatanodeInfo(DatanodeID nodeID, String location, String hostName,
+ final long capacity, final long dfsUsed, final long remaining,
+ final long blockPoolUsed, final long lastUpdate, final int xceiverCount,
+ final AdminStates adminState) {
+ this(nodeID.getName(), nodeID.getStorageID(), nodeID.getInfoPort(), nodeID
+ .getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed, lastUpdate,
+ xceiverCount, location, hostName, adminState);
+ }
/** Constructor */
public DatanodeInfo(final String name, final String storageID,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Thu Oct 6 21:58:22 2011
@@ -30,7 +30,16 @@ public final class HdfsConstants {
/* Hidden constructor */
private HdfsConstants() {
}
-
+
+ /**
+ * HDFS Protocol Names:
+ */
+ public static final String CLIENT_NAMENODE_PROTOCOL_NAME =
+ "org.apache.hadoop.hdfs.protocol.ClientProtocol";
+ public static final String CLIENT_DATANODE_PROTOCOL_NAME =
+ "org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol";
+
+
public static int MIN_BLOCKS_FOR_WRITE = 5;
// Long that indicates "leave current quota unchanged"
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java Thu Oct 6 21:58:22 2011
@@ -241,6 +241,10 @@ public class HdfsFileStatus implements W
final public String getSymlink() {
return DFSUtil.bytes2String(symlink);
}
+
+ final public byte[] getSymlinkInBytes() {
+ return symlink;
+ }
//////////////////////////////////////////////////
// Writable
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java?rev=1179877&r1=1179876&r2=1179877&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java Thu Oct 6 21:58:22 2011
@@ -54,6 +54,11 @@ public class LocatedBlock implements Wri
public LocatedBlock() {
this(new ExtendedBlock(), new DatanodeInfo[0], 0L, false);
}
+
+
+ public LocatedBlock(ExtendedBlock eb) {
+ this(eb, new DatanodeInfo[0], 0L, false);
+ }
public LocatedBlock(String bpid, Block b, DatanodeInfo[] locs) {
this(new ExtendedBlock(bpid, b), locs, -1, false); // startOffset is unknown
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html Thu Oct 6 21:58:22 2011
@@ -0,0 +1,29 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <title>Protocol Buffers based data types for NN protocols</title>
+</head>
+<body>
+<p>
+The Protocol Buffers data types for NN protocols that use
+PB go in this package.
+</p>
+
+
+
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolServerSideTranslatorR23.java Thu Oct 6 21:58:22 2011
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolR23Compatible;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * This class is used on the server side.
+ * Calls come across the wire for the protocol family of Release 23 onwards.
+ * This class translates the R23 data types to the internal data types used
+ * inside the DN as specified in the generic ClientDatanodeProtocol.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class ClientDatanodeProtocolServerSideTranslatorR23 implements
+ ClientDatanodeWireProtocol {
+
+ final private ClientDatanodeProtocol server;
+
+ /**
+ *
+ * @param server - the NN server
+ * @throws IOException
+ */
+ public ClientDatanodeProtocolServerSideTranslatorR23(
+ ClientDatanodeProtocol server) throws IOException {
+ this.server = server;
+ }
+
+ /**
+ * the client side will redirect getProtocolSignature to
+ * getProtocolSignature2.
+ *
+ * However the RPC layer below on the Server side will call
+ * getProtocolVersion and possibly in the future getProtocolSignature.
+ * Hence we still implement it even though the end client's call will
+ * never reach here.
+ */
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ /**
+ * Don't forward this to the server. The protocol version and
+ * signature is that of {@link ClientDatanodeProtocol}
+ */
+ if (!protocol.equals(RPC.getProtocolName(
+ ClientDatanodeWireProtocol.class))) {
+ throw new IOException("Datanode Serverside implements " +
+ ClientDatanodeWireProtocol.class +
+ ". The following requested protocol is unknown: " + protocol);
+ }
+
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ ClientDatanodeWireProtocol.versionID,
+ ClientDatanodeWireProtocol.class);
+ }
+
+ @Override
+ public ProtocolSignatureWritable
+ getProtocolSignature2(
+ String protocol, long clientVersion, int clientMethodsHash)
+ throws IOException {
+ /**
+ * Don't forward this to the server. The protocol version and
+ * signature is that of {@link ClientNamenodeProtocol}
+ */
+ return ProtocolSignatureWritable.convert(
+ this.getProtocolSignature(protocol, clientVersion, clientMethodsHash));
+
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ if (protocol.equals(RPC.getProtocolName(
+ ClientDatanodeWireProtocol.class))) {
+ return ClientDatanodeWireProtocol.versionID;
+ }
+ throw new IOException("Datanode Serverside implements " +
+ ClientDatanodeWireProtocol.class +
+ ". The following requested protocol is unknown: " + protocol);
+ }
+
+ @Override
+ public long getReplicaVisibleLength(ExtendedBlockWritable b) throws IOException {
+ return
+ server.getReplicaVisibleLength(ExtendedBlockWritable.convertExtendedBlock(b));
+ }
+
+ @Override
+ public void refreshNamenodes() throws IOException {
+ server.refreshNamenodes();
+ }
+
+ @Override
+ public void deleteBlockPool(String bpid, boolean force) throws IOException {
+ server.deleteBlockPool(bpid, force);
+ }
+}
\ No newline at end of file
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeProtocolTranslatorR23.java Thu Oct 6 21:58:22 2011
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolR23Compatible;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import javax.net.SocketFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+
+/**
+ * This class forwards ClientDatanodeProtocol calls as RPC to the DN server
+ * while translating from the parameter types used in ClientDatanodeProtocol to
+ * those used in protocolR23Compatile.*.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class ClientDatanodeProtocolTranslatorR23 implements
+ ClientDatanodeProtocol {
+
+ final private ClientDatanodeWireProtocol rpcProxy;
+
+ public ClientDatanodeProtocolTranslatorR23(DatanodeID datanodeid,
+ Configuration conf, int socketTimeout, LocatedBlock locatedBlock)
+ throws IOException {
+ rpcProxy = createClientDatanodeProtocolProxy( datanodeid, conf,
+ socketTimeout, locatedBlock);
+ }
+
+ /** used for testing */
+ public ClientDatanodeProtocolTranslatorR23(InetSocketAddress addr,
+ UserGroupInformation ticket,
+ Configuration conf,
+ SocketFactory factory) throws IOException {
+ rpcProxy = createClientDatanodeProtocolProxy(addr, ticket, conf, factory);
+ }
+
+ static ClientDatanodeWireProtocol createClientDatanodeProtocolProxy(
+ DatanodeID datanodeid, Configuration conf, int socketTimeout,
+ LocatedBlock locatedBlock)
+ throws IOException {
+ InetSocketAddress addr = NetUtils.createSocketAddr(
+ datanodeid.getHost() + ":" + datanodeid.getIpcPort());
+ if (ClientDatanodeWireProtocol.LOG.isDebugEnabled()) {
+ ClientDatanodeWireProtocol.LOG.debug(
+ "ClientDatanodeProtocol addr=" + addr);
+ }
+
+ // Since we're creating a new UserGroupInformation here, we know that no
+ // future RPC proxies will be able to re-use the same connection. And
+ // usages of this proxy tend to be one-off calls.
+ //
+ // This is a temporary fix: callers should really achieve this by using
+ // RPC.stopProxy() on the resulting object, but this is currently not
+ // working in trunk. See the discussion on HDFS-1965.
+ Configuration confWithNoIpcIdle = new Configuration(conf);
+ confWithNoIpcIdle.setInt(CommonConfigurationKeysPublic
+ .IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
+
+ UserGroupInformation ticket = UserGroupInformation
+ .createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
+ ticket.addToken(locatedBlock.getBlockToken());
+ return RPC.getProxy(ClientDatanodeWireProtocol.class,
+ ClientDatanodeWireProtocol.versionID, addr, ticket, confWithNoIpcIdle,
+ NetUtils.getDefaultSocketFactory(conf), socketTimeout);
+ }
+
+ static ClientDatanodeWireProtocol createClientDatanodeProtocolProxy(
+ InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
+ SocketFactory factory) throws IOException {
+ return RPC.getProxy(ClientDatanodeWireProtocol.class,
+ ClientDatanodeWireProtocol.versionID, addr, ticket, conf,
+ factory);
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(
+ String protocolName, long clientVersion, int clientMethodHash)
+ throws IOException {
+ return ProtocolSignatureWritable.convert(
+ rpcProxy.getProtocolSignature2(
+ protocolName, clientVersion, clientMethodHash));
+ }
+
+ @Override
+ public long getProtocolVersion(String protocolName, long clientVersion)
+ throws IOException {
+ return rpcProxy.getProtocolVersion(protocolName, clientVersion);
+ }
+
+ @Override
+ public long getReplicaVisibleLength(ExtendedBlock b) throws IOException {
+ return rpcProxy.getReplicaVisibleLength(
+ ExtendedBlockWritable.convertExtendedBlock(b));
+ }
+
+ @Override
+ public void refreshNamenodes() throws IOException {
+ rpcProxy.refreshNamenodes();
+
+ }
+
+ @Override
+ public void deleteBlockPool(String bpid, boolean force) throws IOException {
+ rpcProxy.deleteBlockPool(bpid, force);
+
+ }
+
+}
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientDatanodeWireProtocol.java Thu Oct 6 21:58:22 2011
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolR23Compatible;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+
+/**
+ * This class defines the actual protocol used to communicate with the
+ * DN via RPC using writable types.
+ * The parameters in the methods which are specified in the
+ * package are separate from those used internally in the DN and DFSClient
+ * and hence need to be converted using {@link ClientDatanodeProtocolTranslatorR23}
+ * and {@link ClientDatanodeProtocolServerSideTranslatorR23}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+@KerberosInfo(
+ serverPrincipal = DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY)
+@TokenInfo(BlockTokenSelector.class)
+@ProtocolInfo(protocolName = HdfsConstants.CLIENT_DATANODE_PROTOCOL_NAME)
+public interface ClientDatanodeWireProtocol extends VersionedProtocol {
+ public static final Log LOG =
+ LogFactory.getLog(ClientDatanodeWireProtocol.class);
+
+ /**
+ * The rules for changing this protocol are the same as that for
+ * {@link ClientNamenodeWireProtocol} - see that java file for details.
+ * 9: Added deleteBlockPool method
+ * 10 Moved the R23 protocol
+ */
+ public static final long versionID = 10L;
+
+ /**
+ * The specification of this method matches that of
+ *
+ * {@link org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol
+ * #getReplicaVisibleLength(org.apache.hadoop.hdfs.protocol.ExtendedBlock)}
+ */
+ long getReplicaVisibleLength(ExtendedBlockWritable b) throws IOException;
+
+ /**
+ * The specification of this method matches that of
+ * {@link org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol#refreshNamenodes()}
+ */
+ void refreshNamenodes() throws IOException;
+
+ /**
+ * The specification of this method matches that of
+ * {@link org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol#deleteBlockPool(String, boolean)}
+ */
+ void deleteBlockPool(String bpid, boolean force) throws IOException;
+
+ /**
+ * This method is defined to get the protocol signature using
+ * the R23 protocol - hence we have added the suffix of 2 to the method name
+ * to avoid conflict.
+ */
+ public org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable
+ getProtocolSignature2(String protocol,
+ long clientVersion,
+ int clientMethodsHash) throws IOException;
+}
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolServerSideTranslatorR23.java Thu Oct 6 21:58:22 2011
@@ -0,0 +1,462 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolR23Compatible;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * This class is used on the server side. Calls come across the wire for the
+ * protocol family of Release 23 onwards. This class translates the R23 data
+ * types to the native data types used inside the NN as specified in the generic
+ * ClientProtocol.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class ClientNamenodeProtocolServerSideTranslatorR23 implements
+ ClientNamenodeWireProtocol {
+ final private ClientProtocol server;
+
+ /**
+ * Constructor
+ *
+ * @param server - the NN server
+ * @throws IOException
+ */
+ public ClientNamenodeProtocolServerSideTranslatorR23(ClientProtocol server)
+ throws IOException {
+ this.server = server;
+ }
+
+ /**
+ * The client side will redirect getProtocolSignature to
+ * getProtocolSignature2.
+ *
+ * However the RPC layer below on the Server side will call getProtocolVersion
+ * and possibly in the future getProtocolSignature. Hence we still implement
+ * it even though the end client's call will never reach here.
+ */
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ /**
+ * Don't forward this to the server. The protocol version and signature is
+ * that of {@link ClientNamenodeProtocol}
+ *
+ */
+ if (!protocol.equals(RPC.getProtocolName(
+ ClientNamenodeWireProtocol.class))) {
+ throw new IOException("Namenode Serverside implements " +
+ RPC.getProtocolName(ClientNamenodeWireProtocol.class) +
+ ". The following requested protocol is unknown: " + protocol);
+ }
+
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ ClientNamenodeWireProtocol.versionID,
+ ClientNamenodeWireProtocol.class);
+ }
+
+ @Override
+ public ProtocolSignatureWritable
+ getProtocolSignature2(
+ String protocol, long clientVersion, int clientMethodsHash)
+ throws IOException {
+ /**
+ * Don't forward this to the server. The protocol version and signature is
+ * that of {@link ClientNamenodeProtocol}
+ *
+ */
+
+ return ProtocolSignatureWritable.convert(
+ this.getProtocolSignature(protocol, clientVersion, clientMethodsHash));
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ if (protocol.equals(RPC.getProtocolName(
+ ClientNamenodeWireProtocol.class))) {
+ return ClientNamenodeWireProtocol.versionID;
+ }
+ throw new IOException("Namenode Serverside implements " +
+ RPC.getProtocolName(ClientNamenodeWireProtocol.class) +
+ ". The following requested protocol is unknown: " + protocol);
+ }
+
+ @Override
+ public LocatedBlocksWritable getBlockLocations(
+ String src, long offset, long length)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ return LocatedBlocksWritable.convertLocatedBlocks(
+ server.getBlockLocations(src, offset, length));
+ }
+
+ @Override
+ public FsServerDefaultsWritable getServerDefaults() throws IOException {
+ return FsServerDefaultsWritable.convert(server.getServerDefaults());
+ }
+
+ @Override
+ public void create(String src, FsPermissionWritable masked, String clientName,
+ EnumSetWritable<CreateFlag> flag, boolean createParent,
+ short replication, long blockSize) throws AccessControlException,
+ AlreadyBeingCreatedException, DSQuotaExceededException,
+ FileAlreadyExistsException, FileNotFoundException,
+ NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ server.create(src, FsPermissionWritable.convertPermission(masked),
+ clientName, flag, createParent, replication, blockSize);
+
+ }
+
+ @Override
+ public LocatedBlockWritable append(String src, String clientName)
+ throws AccessControlException, DSQuotaExceededException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable.convertLocatedBlock(
+ server.append(src, clientName));
+ }
+
+ @Override
+ public boolean setReplication(String src, short replication)
+ throws AccessControlException, DSQuotaExceededException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return server.setReplication(src, replication);
+ }
+
+ @Override
+ public void setPermission(String src, FsPermissionWritable permission)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ server.setPermission(src,
+ FsPermissionWritable.convertPermission(permission));
+
+ }
+
+ @Override
+ public void setOwner(String src, String username, String groupname)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ server.setOwner(src, username, groupname);
+
+ }
+
+ @Override
+ public void abandonBlock(ExtendedBlockWritable b, String src, String holder)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ server.abandonBlock(
+ ExtendedBlockWritable.convertExtendedBlock(b), src, holder);
+
+ }
+
+ @Override
+ public LocatedBlockWritable addBlock(String src, String clientName,
+ ExtendedBlockWritable previous, DatanodeInfoWritable[] excludeNodes)
+ throws AccessControlException, FileNotFoundException,
+ NotReplicatedYetException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable.convertLocatedBlock(
+ server.addBlock(src, clientName,
+ ExtendedBlockWritable.convertExtendedBlock(previous),
+ DatanodeInfoWritable.convertDatanodeInfo(excludeNodes)));
+ }
+
+ @Override
+ public LocatedBlockWritable getAdditionalDatanode(String src, ExtendedBlockWritable blk,
+ DatanodeInfoWritable[] existings, DatanodeInfoWritable[] excludes,
+ int numAdditionalNodes, String clientName) throws AccessControlException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable.convertLocatedBlock(
+ server.getAdditionalDatanode(src,
+ ExtendedBlockWritable.convertExtendedBlock(blk),
+ DatanodeInfoWritable.convertDatanodeInfo(existings),
+ DatanodeInfoWritable.convertDatanodeInfo(excludes),
+ numAdditionalNodes, clientName));
+ }
+
+ @Override
+ public boolean complete(String src, String clientName, ExtendedBlockWritable last)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ return server.complete(src, clientName,
+ ExtendedBlockWritable.convertExtendedBlock(last));
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlockWritable[] blocks) throws IOException {
+ server.reportBadBlocks(LocatedBlockWritable.convertLocatedBlock(blocks));
+
+ }
+
+ @Override
+ public boolean rename(String src, String dst) throws UnresolvedLinkException,
+ IOException {
+ return server.rename(src, dst);
+ }
+
+ @Override
+ public void concat(String trg, String[] srcs) throws IOException,
+ UnresolvedLinkException {
+ server.concat(trg, srcs);
+
+ }
+
+ @Override
+ public void rename2(String src, String dst, Rename... options)
+ throws AccessControlException, DSQuotaExceededException,
+ FileAlreadyExistsException, FileNotFoundException,
+ NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ server.rename2(src, dst, options);
+ }
+
+ @Override
+ public boolean delete(String src, boolean recursive)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ return server.delete(src, recursive);
+ }
+
+ @Override
+ public boolean mkdirs(String src, FsPermissionWritable masked, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, NSQuotaExceededException,
+ ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
+ IOException {
+
+ return server.mkdirs(src, FsPermissionWritable.convertPermission(masked),
+ createParent);
+ }
+
+ @Override
+ public DirectoryListingWritable getListing(String src, byte[] startAfter,
+ boolean needLocation) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ return DirectoryListingWritable.convertDirectoryListing(
+ server.getListing(src, startAfter, needLocation));
+ }
+
+ @Override
+ public void renewLease(String clientName) throws AccessControlException,
+ IOException {
+ server.renewLease(clientName);
+
+ }
+
+ @Override
+ public boolean recoverLease(String src, String clientName) throws IOException {
+ return server.recoverLease(src, clientName);
+ }
+
+ @Override
+ public long[] getStats() throws IOException {
+ return server.getStats();
+ }
+
+ @Override
+ public DatanodeInfoWritable[] getDatanodeReport(DatanodeReportType type)
+ throws IOException {
+ return DatanodeInfoWritable
+ .convertDatanodeInfo(server.getDatanodeReport(type));
+ }
+
+ @Override
+ public long getPreferredBlockSize(String filename) throws IOException,
+ UnresolvedLinkException {
+ return server.getPreferredBlockSize(filename);
+ }
+
+ @Override
+ public boolean setSafeMode(SafeModeAction action) throws IOException {
+ return server.setSafeMode(action);
+ }
+
+ @Override
+ public void saveNamespace() throws AccessControlException, IOException {
+ server.saveNamespace();
+
+ }
+
+ @Override
+ public boolean restoreFailedStorage(String arg) throws AccessControlException {
+ return server.restoreFailedStorage(arg);
+ }
+
+ @Override
+ public void refreshNodes() throws IOException {
+ server.refreshNodes();
+
+ }
+
+ @Override
+ public void finalizeUpgrade() throws IOException {
+ server.finalizeUpgrade();
+
+ }
+
+ @Override
+ public UpgradeStatusReportWritable distributedUpgradeProgress(UpgradeAction action)
+ throws IOException {
+ return UpgradeStatusReportWritable.convert(
+ server.distributedUpgradeProgress(action));
+ }
+
+ @Override
+ public CorruptFileBlocksWritable listCorruptFileBlocks(String path, String cookie)
+ throws IOException {
+ return CorruptFileBlocksWritable.convertCorruptFilesBlocks(
+ server.listCorruptFileBlocks(path, cookie));
+ }
+
+ @Override
+ public void metaSave(String filename) throws IOException {
+ server.metaSave(filename);
+
+ }
+
+ @Override
+ public HdfsFileStatusWritable getFileInfo(String src) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ return HdfsFileStatusWritable.convertHdfsFileStatus(
+ server.getFileInfo(src));
+ }
+
+ @Override
+ public HdfsFileStatusWritable getFileLinkInfo(String src)
+ throws AccessControlException, UnresolvedLinkException, IOException {
+ return HdfsFileStatusWritable.convertHdfsFileStatus(
+ server.getFileLinkInfo(src));
+ }
+
+ @Override
+ public ContentSummaryWritable getContentSummary(String path)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ return ContentSummaryWritable.convert(server.getContentSummary(path));
+ }
+
+ @Override
+ public void setQuota(String path, long namespaceQuota, long diskspaceQuota)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ server.setQuota(path, namespaceQuota, diskspaceQuota);
+
+ }
+
+ @Override
+ public void fsync(String src, String client) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ server.fsync(src, client);
+
+ }
+
+ @Override
+ public void setTimes(String src, long mtime, long atime)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ server.setTimes(src, mtime, atime);
+
+ }
+
+ @Override
+ public void createSymlink(String target, String link, FsPermissionWritable dirPerm,
+ boolean createParent) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ server.createSymlink(target, link, FsPermissionWritable.convertPermission(dirPerm),
+ createParent);
+
+ }
+
+ @Override
+ public String getLinkTarget(String path) throws AccessControlException,
+ FileNotFoundException, IOException {
+ return server.getLinkTarget(path);
+ }
+
+ @Override
+ public LocatedBlockWritable updateBlockForPipeline(ExtendedBlockWritable block,
+ String clientName) throws IOException {
+ return LocatedBlockWritable.convertLocatedBlock(
+ server.updateBlockForPipeline(
+ ExtendedBlockWritable.convertExtendedBlock(block), clientName));
+ }
+
+ @Override
+ public void updatePipeline(String clientName, ExtendedBlockWritable oldBlock,
+ ExtendedBlockWritable newBlock, DatanodeIDWritable[] newNodes)
+ throws IOException {
+ server.updatePipeline(clientName,
+ ExtendedBlockWritable.convertExtendedBlock(oldBlock),
+ ExtendedBlockWritable.convertExtendedBlock(newBlock),
+ DatanodeIDWritable.convertDatanodeID(newNodes));
+ }
+
+ @Override
+ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ return server.getDelegationToken(renewer);
+ }
+
+ @Override
+ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ return server.renewDelegationToken(token);
+ }
+
+ @Override
+ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ server.cancelDelegationToken(token);
+ }
+
+ @Override
+ public void setBalancerBandwidth(long bandwidth) throws IOException {
+ server.setBalancerBandwidth(bandwidth);
+ }
+}
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java?rev=1179877&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ClientNamenodeProtocolTranslatorR23.java Thu Oct 6 21:58:22 2011
@@ -0,0 +1,485 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolR23Compatible;
+
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+
+/**
+ * This class forwards NN's ClientProtocol calls as RPC calls to the NN server
+ * while translating from the parameter types used in ClientProtocol to those
+ * used in protocolR23Compatile.*.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class ClientNamenodeProtocolTranslatorR23 implements
+ ClientProtocol, Closeable {
+ final private ClientNamenodeWireProtocol rpcProxyWithoutRetry;
+ final private ClientNamenodeWireProtocol rpcProxy;
+
+ private static ClientNamenodeWireProtocol createNamenode(
+ InetSocketAddress nameNodeAddr, Configuration conf,
+ UserGroupInformation ugi) throws IOException {
+ return RPC.getProxy(ClientNamenodeWireProtocol.class,
+ ClientNamenodeWireProtocol.versionID, nameNodeAddr, ugi, conf,
+ NetUtils.getSocketFactory(conf, ClientNamenodeWireProtocol.class));
+ }
+
+ /** Create a {@link NameNode} proxy */
+ static ClientNamenodeWireProtocol createNamenodeWithRetry(
+ ClientNamenodeWireProtocol rpcNamenode) {
+ RetryPolicy createPolicy = RetryPolicies
+ .retryUpToMaximumCountWithFixedSleep(5,
+ HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
+
+ Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
+ remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
+ createPolicy);
+
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
+ new HashMap<Class<? extends Exception>, RetryPolicy>();
+ exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
+ .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
+ remoteExceptionToPolicyMap));
+ RetryPolicy methodPolicy = RetryPolicies.retryByException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
+ Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
+
+ methodNameToPolicyMap.put("create", methodPolicy);
+
+ return (ClientNamenodeWireProtocol) RetryProxy.create(
+ ClientNamenodeWireProtocol.class, rpcNamenode, methodNameToPolicyMap);
+ }
+
+ public ClientNamenodeProtocolTranslatorR23(InetSocketAddress nameNodeAddr,
+ Configuration conf, UserGroupInformation ugi) throws IOException {
+ rpcProxyWithoutRetry = createNamenode(nameNodeAddr, conf, ugi);
+ rpcProxy = createNamenodeWithRetry(rpcProxyWithoutRetry);
+ }
+
+ public Object getProxyWithoutRetry() {
+ return rpcProxyWithoutRetry;
+ }
+
+ public void close() {
+ RPC.stopProxy(rpcProxyWithoutRetry);
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocolName,
+ long clientVersion, int clientMethodHash)
+ throws IOException {
+ return ProtocolSignatureWritable.convert(rpcProxy.getProtocolSignature2(
+ protocolName, clientVersion, clientMethodHash));
+ }
+
+ @Override
+ public long getProtocolVersion(String protocolName, long clientVersion) throws IOException {
+ return rpcProxy.getProtocolVersion(protocolName, clientVersion);
+ }
+
+ @Override
+ public LocatedBlocks getBlockLocations(String src, long offset, long length)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ return LocatedBlocksWritable
+ .convertLocatedBlocks(rpcProxy.getBlockLocations(src, offset, length));
+ }
+
+ @Override
+ public FsServerDefaults getServerDefaults() throws IOException {
+ return FsServerDefaultsWritable
+ .convert(rpcProxy.getServerDefaults());
+ }
+
+ @Override
+ public void create(String src, FsPermission masked, String clientName,
+ EnumSetWritable<CreateFlag> flag, boolean createParent,
+ short replication, long blockSize) throws AccessControlException,
+ AlreadyBeingCreatedException, DSQuotaExceededException,
+ FileAlreadyExistsException, FileNotFoundException,
+ NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.create(src, FsPermissionWritable.convertPermission(masked),
+ clientName, flag, createParent, replication, blockSize);
+
+ }
+
+ @Override
+ public LocatedBlock append(String src, String clientName)
+ throws AccessControlException, DSQuotaExceededException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable
+ .convertLocatedBlock(rpcProxy.append(src, clientName));
+ }
+
+ @Override
+ public boolean setReplication(String src, short replication)
+ throws AccessControlException, DSQuotaExceededException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return rpcProxy.setReplication(src, replication);
+ }
+
+ @Override
+ public void setPermission(String src, FsPermission permission)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.setPermission(src,
+ FsPermissionWritable.convertPermission(permission));
+
+ }
+
+ @Override
+ public void setOwner(String src, String username, String groupname)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.setOwner(src, username, groupname);
+
+ }
+
+ @Override
+ public void abandonBlock(ExtendedBlock b, String src, String holder)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.abandonBlock(
+ ExtendedBlockWritable.convertExtendedBlock(b), src, holder);
+
+ }
+
+ @Override
+ public LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludeNodes)
+ throws AccessControlException, FileNotFoundException,
+ NotReplicatedYetException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable
+ .convertLocatedBlock(rpcProxy.addBlock(src, clientName,
+ ExtendedBlockWritable.convertExtendedBlock(previous),
+ DatanodeInfoWritable.convertDatanodeInfo(excludeNodes)));
+ }
+
+ @Override
+ public LocatedBlock getAdditionalDatanode(String src, ExtendedBlock blk,
+ DatanodeInfo[] existings, DatanodeInfo[] excludes,
+ int numAdditionalNodes, String clientName) throws AccessControlException,
+ FileNotFoundException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ return LocatedBlockWritable
+ .convertLocatedBlock(rpcProxy.getAdditionalDatanode(src,
+ ExtendedBlockWritable.convertExtendedBlock(blk),
+ DatanodeInfoWritable.convertDatanodeInfo(existings),
+ DatanodeInfoWritable.convertDatanodeInfo(excludes),
+ numAdditionalNodes, clientName));
+ }
+
+ @Override
+ public boolean complete(String src, String clientName, ExtendedBlock last)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ return rpcProxy.complete(src, clientName,
+ ExtendedBlockWritable.convertExtendedBlock(last));
+ }
+
+ @Override
+ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+ rpcProxy.reportBadBlocks(LocatedBlockWritable.convertLocatedBlock(blocks));
+
+ }
+
+ @Override
+ public boolean rename(String src, String dst) throws UnresolvedLinkException,
+ IOException {
+ return rpcProxy.rename(src, dst);
+ }
+
+ @Override
+ public void concat(String trg, String[] srcs) throws IOException,
+ UnresolvedLinkException {
+ rpcProxy.concat(trg, srcs);
+
+ }
+
+ @Override
+ public void rename2(String src, String dst, Rename... options)
+ throws AccessControlException, DSQuotaExceededException,
+ FileAlreadyExistsException, FileNotFoundException,
+ NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.rename2(src, dst, options);
+
+ }
+
+ @Override
+ public boolean delete(String src, boolean recursive)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ return rpcProxy.delete(src, recursive);
+ }
+
+ @Override
+ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
+ throws AccessControlException, FileAlreadyExistsException,
+ FileNotFoundException, NSQuotaExceededException,
+ ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
+ IOException {
+
+ return rpcProxy.mkdirs(src,
+ FsPermissionWritable.convertPermission(masked), createParent);
+ }
+
+ @Override
+ public DirectoryListing getListing(String src, byte[] startAfter,
+ boolean needLocation) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ return DirectoryListingWritable.convertDirectoryListing(
+ rpcProxy.getListing(src, startAfter, needLocation));
+ }
+
+ @Override
+ public void renewLease(String clientName) throws AccessControlException,
+ IOException {
+ rpcProxy.renewLease(clientName);
+
+ }
+
+ @Override
+ public boolean recoverLease(String src, String clientName) throws IOException {
+ return rpcProxy.recoverLease(src, clientName);
+ }
+
+ @Override
+ public long[] getStats() throws IOException {
+ return rpcProxy.getStats();
+ }
+
+ @Override
+ public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
+ throws IOException {
+ return DatanodeInfoWritable.convertDatanodeInfo(
+ rpcProxy.getDatanodeReport(type));
+ }
+
+ @Override
+ public long getPreferredBlockSize(String filename) throws IOException,
+ UnresolvedLinkException {
+ return rpcProxy.getPreferredBlockSize(filename);
+ }
+
+ @Override
+ public boolean setSafeMode(SafeModeAction action) throws IOException {
+ return rpcProxy.setSafeMode(action);
+ }
+
+ @Override
+ public void saveNamespace() throws AccessControlException, IOException {
+ rpcProxy.saveNamespace();
+
+ }
+
+ @Override
+ public boolean restoreFailedStorage(String arg) throws AccessControlException {
+ return rpcProxy.restoreFailedStorage(arg);
+ }
+
+ @Override
+ public void refreshNodes() throws IOException {
+ rpcProxy.refreshNodes();
+
+ }
+
+ @Override
+ public void finalizeUpgrade() throws IOException {
+ rpcProxy.finalizeUpgrade();
+
+ }
+
+ @Override
+ public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action)
+ throws IOException {
+ return UpgradeStatusReportWritable.convert(
+ rpcProxy.distributedUpgradeProgress(action));
+ }
+
+ @Override
+ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
+ throws IOException {
+ return CorruptFileBlocksWritable.convertCorruptFileBlocks(
+ rpcProxy.listCorruptFileBlocks(path, cookie));
+ }
+
+ @Override
+ public void metaSave(String filename) throws IOException {
+ rpcProxy.metaSave(filename);
+
+ }
+
+ @Override
+ public HdfsFileStatus getFileInfo(String src) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ return HdfsFileStatusWritable.convertHdfsFileStatus(
+ rpcProxy.getFileInfo(src));
+ }
+
+ @Override
+ public HdfsFileStatus getFileLinkInfo(String src)
+ throws AccessControlException, UnresolvedLinkException, IOException {
+ return HdfsFileStatusWritable
+ .convertHdfsFileStatus(rpcProxy.getFileLinkInfo(src));
+ }
+
+ @Override
+ public ContentSummary getContentSummary(String path)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ return ContentSummaryWritable
+ .convert(rpcProxy.getContentSummary(path));
+ }
+
+ @Override
+ public void setQuota(String path, long namespaceQuota, long diskspaceQuota)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.setQuota(path, namespaceQuota, diskspaceQuota);
+
+ }
+
+ @Override
+ public void fsync(String src, String client) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ rpcProxy.fsync(src, client);
+
+ }
+
+ @Override
+ public void setTimes(String src, long mtime, long atime)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ rpcProxy.setTimes(src, mtime, atime);
+
+ }
+
+ @Override
+ public void createSymlink(String target, String link, FsPermission dirPerm,
+ boolean createParent) throws AccessControlException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
+ IOException {
+ rpcProxy.createSymlink(target, link,
+ FsPermissionWritable.convertPermission(dirPerm), createParent);
+
+ }
+
+ @Override
+ public String getLinkTarget(String path) throws AccessControlException,
+ FileNotFoundException, IOException {
+ return rpcProxy.getLinkTarget(path);
+ }
+
+ @Override
+ public LocatedBlock updateBlockForPipeline(ExtendedBlock block,
+ String clientName) throws IOException {
+ return LocatedBlockWritable.convertLocatedBlock(
+ rpcProxy.updateBlockForPipeline(
+ ExtendedBlockWritable.convertExtendedBlock(block), clientName));
+ }
+
+ @Override
+ public void updatePipeline(String clientName, ExtendedBlock oldBlock,
+ ExtendedBlock newBlock, DatanodeID[] newNodes) throws IOException {
+ rpcProxy.updatePipeline(clientName,
+ ExtendedBlockWritable.convertExtendedBlock(oldBlock),
+ ExtendedBlockWritable.convertExtendedBlock(newBlock),
+ DatanodeIDWritable.convertDatanodeID(newNodes));
+
+ }
+
+ @Override
+ public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ return rpcProxy.getDelegationToken(renewer);
+ }
+
+ @Override
+ public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ return rpcProxy.renewDelegationToken(token);
+ }
+
+ @Override
+ public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
+ throws IOException {
+ rpcProxy.cancelDelegationToken(token);
+ }
+
+ @Override
+ public void setBalancerBandwidth(long bandwidth) throws IOException {
+ rpcProxy.setBalancerBandwidth(bandwidth);
+ }
+}