You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/02/27 00:32:14 UTC
svn commit: r1293964 [8/11] - in
/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs:
./ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolPB/ src/...
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java Sun Feb 26 23:32:06 2012
@@ -39,6 +39,25 @@ interface JournalManager {
*/
void finalizeLogSegment(long firstTxId, long lastTxId) throws IOException;
+ /**
+ * Get the input stream starting with fromTxnId from this journal manager
+ * @param fromTxnId the first transaction id we want to read
+ * @return the stream starting with transaction fromTxnId
+ * @throws IOException if a stream cannot be found.
+ */
+ EditLogInputStream getInputStream(long fromTxnId) throws IOException;
+
+ /**
+ * Get the number of transaction contiguously available from fromTxnId.
+ *
+ * @param fromTxnId Transaction id to count from
+ * @return The number of transactions available from fromTxnId
+ * @throws IOException if the journal cannot be read.
+ * @throws CorruptionException if there is a gap in the journal at fromTxnId.
+ */
+ long getNumberOfTransactions(long fromTxnId)
+ throws IOException, CorruptionException;
+
/**
* Set the amount of memory that this stream should use to buffer edits
*/
@@ -57,10 +76,21 @@ interface JournalManager {
throws IOException;
/**
- * @return an EditLogInputStream that reads from the same log that
- * the edit log is currently writing. May return null if this journal
- * manager does not support this operation.
- */
- EditLogInputStream getInProgressInputStream(long segmentStartsAtTxId)
- throws IOException;
+ * Recover segments which have not been finalized.
+ */
+ void recoverUnfinalizedSegments() throws IOException;
+
+ /**
+ * Indicate that a journal is cannot be used to load a certain range of
+ * edits.
+ * This exception occurs in the case of a gap in the transactions, or a
+ * corrupt edit file.
+ */
+ public static class CorruptionException extends IOException {
+ static final long serialVersionUID = -4687802717006172702L;
+
+ public CorruptionException(String reason) {
+ super(reason);
+ }
+ }
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Sun Feb 26 23:32:06 2012
@@ -245,13 +245,13 @@ public class LeaseManager {
return paths.remove(src);
}
- /** {@inheritDoc} */
+ @Override
public String toString() {
return "[Lease. Holder: " + holder
+ ", pendingcreates: " + paths.size() + "]";
}
- /** {@inheritDoc} */
+ @Override
public int compareTo(Lease o) {
Lease l1 = this;
Lease l2 = o;
@@ -266,7 +266,7 @@ public class LeaseManager {
}
}
- /** {@inheritDoc} */
+ @Override
public boolean equals(Object o) {
if (!(o instanceof Lease)) {
return false;
@@ -279,7 +279,7 @@ public class LeaseManager {
return false;
}
- /** {@inheritDoc} */
+ @Override
public int hashCode() {
return holder.hashCode();
}
@@ -429,7 +429,7 @@ public class LeaseManager {
}
}
- /** {@inheritDoc} */
+ @Override
public synchronized String toString() {
return getClass().getSimpleName() + "= {"
+ "\n leases=" + leases
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sun Feb 26 23:32:06 2012
@@ -296,7 +296,7 @@ public class NameNode {
}
protected void loadNamesystem(Configuration conf) throws IOException {
- this.namesystem = new FSNamesystem(conf);
+ this.namesystem = FSNamesystem.loadFromDisk(conf);
}
NamenodeRegistration getRegistration() {
@@ -602,16 +602,16 @@ public class NameNode {
}
System.out.println("Formatting using clusterid: " + clusterId);
- FSImage fsImage = new FSImage(conf, null, dirsToFormat, editDirsToFormat);
- FSNamesystem nsys = new FSNamesystem(fsImage, conf);
- nsys.dir.fsImage.format(clusterId);
+ FSImage fsImage = new FSImage(conf, dirsToFormat, editDirsToFormat);
+ FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+ fsImage.format(fsn, clusterId);
return false;
}
private static boolean finalize(Configuration conf,
boolean isConfirmationNeeded
) throws IOException {
- FSNamesystem nsys = new FSNamesystem(new FSImage(conf), conf);
+ FSNamesystem nsys = new FSNamesystem(conf, new FSImage(conf));
System.err.print(
"\"finalize\" will remove the previous state of the files system.\n"
+ "Recent upgrade will become permanent.\n"
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Sun Feb 26 23:32:06 2012
@@ -195,7 +195,9 @@ public class NameNodeHttpServer {
}
public void stop() throws Exception {
- httpServer.stop();
+ if (httpServer != null) {
+ httpServer.stop();
+ }
}
public InetSocketAddress getHttpAddress() {
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java Sun Feb 26 23:32:06 2012
@@ -64,7 +64,7 @@ public class NameNodeResourceChecker {
duReserved = conf.getLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY,
DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_DEFAULT);
-
+
Collection<URI> extraCheckedVolumes = Util.stringCollectionAsURIs(conf
.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_KEY));
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java Sun Feb 26 23:32:06 2012
@@ -38,6 +38,8 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -57,6 +59,24 @@ import org.apache.hadoop.hdfs.protocol.U
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.NamenodeProtocolService;
+import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeProtocolService;
+import org.apache.hadoop.hdfs.protocol.proto.GetUserMappingsProtocolProtos.GetUserMappingsProtocolService;
+import org.apache.hadoop.hdfs.protocol.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshAuthorizationPolicyProtocolService;
+import org.apache.hadoop.hdfs.protocol.proto.RefreshUserMappingsProtocolProtos.RefreshUserMappingsProtocolService;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.GetUserMappingsProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.GetUserMappingsProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.RefreshAuthorizationPolicyProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RefreshAuthorizationPolicyProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolPB;
+import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
@@ -68,6 +88,8 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -75,12 +97,17 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.NodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -93,6 +120,8 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
+import com.google.protobuf.BlockingService;
+
/**
* This class is responsible for handling all of the RPC calls to the NameNode.
* It is created, started, and stopped by {@link NameNode}.
@@ -114,8 +143,8 @@ class NameNodeRpcServer implements Namen
private final InetSocketAddress serviceRPCAddress;
/** The RPC server that listens to requests from clients */
- protected final RPC.Server server;
- protected final InetSocketAddress rpcAddress;
+ protected final RPC.Server clientRpcServer;
+ protected final InetSocketAddress clientRpcAddress;
public NameNodeRpcServer(Configuration conf, NameNode nn)
throws IOException {
@@ -127,46 +156,107 @@ class NameNodeRpcServer implements Namen
conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY,
DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
InetSocketAddress socAddr = nn.getRpcServerAddress(conf);
-
+ RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
+ ProtobufRpcEngine.class);
+ ClientNamenodeProtocolServerSideTranslatorPB
+ clientProtocolServerTranslator =
+ new ClientNamenodeProtocolServerSideTranslatorPB(this);
+ BlockingService clientNNPbService = ClientNamenodeProtocol.
+ newReflectiveBlockingService(clientProtocolServerTranslator);
+
+ DatanodeProtocolServerSideTranslatorPB dnProtoPbTranslator =
+ new DatanodeProtocolServerSideTranslatorPB(this);
+ BlockingService dnProtoPbService = DatanodeProtocolService
+ .newReflectiveBlockingService(dnProtoPbTranslator);
+
+ NamenodeProtocolServerSideTranslatorPB namenodeProtocolXlator =
+ new NamenodeProtocolServerSideTranslatorPB(this);
+ BlockingService NNPbService = NamenodeProtocolService
+ .newReflectiveBlockingService(namenodeProtocolXlator);
+
+ RefreshAuthorizationPolicyProtocolServerSideTranslatorPB refreshAuthPolicyXlator =
+ new RefreshAuthorizationPolicyProtocolServerSideTranslatorPB(this);
+ BlockingService refreshAuthService = RefreshAuthorizationPolicyProtocolService
+ .newReflectiveBlockingService(refreshAuthPolicyXlator);
+
+ RefreshUserMappingsProtocolServerSideTranslatorPB refreshUserMappingXlator =
+ new RefreshUserMappingsProtocolServerSideTranslatorPB(this);
+ BlockingService refreshUserMappingService = RefreshUserMappingsProtocolService
+ .newReflectiveBlockingService(refreshUserMappingXlator);
+
+ GetUserMappingsProtocolServerSideTranslatorPB getUserMappingXlator =
+ new GetUserMappingsProtocolServerSideTranslatorPB(this);
+ BlockingService getUserMappingService = GetUserMappingsProtocolService
+ .newReflectiveBlockingService(getUserMappingXlator);
+
+ WritableRpcEngine.ensureInitialized();
+
InetSocketAddress dnSocketAddr = nn.getServiceRpcServerAddress(conf);
if (dnSocketAddr != null) {
int serviceHandlerCount =
conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
- this.serviceRpcServer = RPC.getServer(NamenodeProtocols.class, this,
- dnSocketAddr.getHostName(), dnSocketAddr.getPort(), serviceHandlerCount,
+ // Add all the RPC protocols that the namenode implements
+ this.serviceRpcServer =
+ RPC.getServer(org.apache.hadoop.hdfs.protocolPB.
+ ClientNamenodeProtocolPB.class, clientNNPbService,
+ dnSocketAddr.getHostName(), dnSocketAddr.getPort(),
+ serviceHandlerCount,
false, conf, namesystem.getDelegationTokenSecretManager());
+ DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
+ serviceRpcServer);
+ DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService,
+ serviceRpcServer);
+ DFSUtil.addPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class,
+ refreshAuthService, serviceRpcServer);
+ DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class,
+ refreshUserMappingService, serviceRpcServer);
+ DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
+ getUserMappingService, serviceRpcServer);
+
this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress();
nn.setRpcServiceServerAddress(conf, serviceRPCAddress);
} else {
serviceRpcServer = null;
serviceRPCAddress = null;
}
- this.server = RPC.getServer(NamenodeProtocols.class, this,
- socAddr.getHostName(), socAddr.getPort(),
- handlerCount, false, conf,
- namesystem.getDelegationTokenSecretManager());
+ // Add all the RPC protocols that the namenode implements
+ this.clientRpcServer = RPC.getServer(
+ org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB.class,
+ clientNNPbService, socAddr.getHostName(),
+ socAddr.getPort(), handlerCount, false, conf,
+ namesystem.getDelegationTokenSecretManager());
+ DFSUtil.addPBProtocol(conf, NamenodeProtocolPB.class, NNPbService,
+ clientRpcServer);
+ DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService,
+ clientRpcServer);
+ DFSUtil.addPBProtocol(conf, RefreshAuthorizationPolicyProtocolPB.class,
+ refreshAuthService, clientRpcServer);
+ DFSUtil.addPBProtocol(conf, RefreshUserMappingsProtocolPB.class,
+ refreshUserMappingService, clientRpcServer);
+ DFSUtil.addPBProtocol(conf, GetUserMappingsProtocolPB.class,
+ getUserMappingService, clientRpcServer);
// set service-level authorization security policy
if (serviceAuthEnabled =
conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
- this.server.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ this.clientRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
if (this.serviceRpcServer != null) {
this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
}
}
// The rpc-server port can be ephemeral... ensure we have the correct info
- this.rpcAddress = this.server.getListenerAddress();
- nn.setRpcServerAddress(conf, rpcAddress);
+ this.clientRpcAddress = this.clientRpcServer.getListenerAddress();
+ nn.setRpcServerAddress(conf, clientRpcAddress);
}
/**
* Actually start serving requests.
*/
void start() {
- server.start(); //start RPC server
+ clientRpcServer.start(); //start RPC server
if (serviceRpcServer != null) {
serviceRpcServer.start();
}
@@ -176,11 +266,11 @@ class NameNodeRpcServer implements Namen
* Wait until the RPC server has shut down.
*/
void join() throws InterruptedException {
- this.server.join();
+ this.clientRpcServer.join();
}
void stop() {
- if(server != null) server.stop();
+ if(clientRpcServer != null) clientRpcServer.stop();
if(serviceRpcServer != null) serviceRpcServer.stop();
}
@@ -189,7 +279,7 @@ class NameNodeRpcServer implements Namen
}
InetSocketAddress getRpcAddress() {
- return rpcAddress;
+ return clientRpcAddress;
}
@Override // VersionedProtocol
@@ -203,7 +293,8 @@ class NameNodeRpcServer implements Namen
public long getProtocolVersion(String protocol,
long clientVersion) throws IOException {
if (protocol.equals(ClientProtocol.class.getName())) {
- return ClientProtocol.versionID;
+ throw new IOException("Old Namenode Client protocol is not supported:" +
+ protocol + "Switch your clientside to " + ClientNamenodeProtocol.class);
} else if (protocol.equals(DatanodeProtocol.class.getName())){
return DatanodeProtocol.versionID;
} else if (protocol.equals(NamenodeProtocol.class.getName())){
@@ -756,8 +847,8 @@ class NameNodeRpcServer implements Namen
@Override // DatanodeProtocol
- public DatanodeRegistration registerDatanode(DatanodeRegistration nodeReg)
- throws IOException {
+ public DatanodeRegistration registerDatanode(DatanodeRegistration nodeReg,
+ DatanodeStorage[] storages) throws IOException {
verifyVersion(nodeReg.getVersion());
namesystem.registerDatanode(nodeReg);
@@ -766,19 +857,20 @@ class NameNodeRpcServer implements Namen
@Override // DatanodeProtocol
public DatanodeCommand[] sendHeartbeat(DatanodeRegistration nodeReg,
- long capacity, long dfsUsed, long remaining, long blockPoolUsed,
- int xmitsInProgress, int xceiverCount, int failedVolumes)
- throws IOException {
+ StorageReport[] report, int xmitsInProgress, int xceiverCount,
+ int failedVolumes) throws IOException {
verifyRequest(nodeReg);
- return namesystem.handleHeartbeat(nodeReg, capacity, dfsUsed, remaining,
- blockPoolUsed, xceiverCount, xmitsInProgress, failedVolumes);
+ return namesystem.handleHeartbeat(nodeReg, report[0].getCapacity(),
+ report[0].getDfsUsed(), report[0].getRemaining(),
+ report[0].getBlockPoolUsed(), xceiverCount, xmitsInProgress,
+ failedVolumes);
}
@Override // DatanodeProtocol
public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
- String poolId, long[] blocks) throws IOException {
+ String poolId, StorageBlockReport[] reports) throws IOException {
verifyRequest(nodeReg);
- BlockListAsLongs blist = new BlockListAsLongs(blocks);
+ BlockListAsLongs blist = new BlockListAsLongs(reports[0].getBlocks());
if(stateChangeLog.isDebugEnabled()) {
stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
+ "from " + nodeReg.getName() + " " + blist.getNumberOfBlocks()
@@ -787,22 +879,21 @@ class NameNodeRpcServer implements Namen
namesystem.getBlockManager().processReport(nodeReg, poolId, blist);
if (nn.getFSImage().isUpgradeFinalized())
- return new DatanodeCommand.Finalize(poolId);
+ return new FinalizeCommand(poolId);
return null;
}
@Override // DatanodeProtocol
- public void blockReceived(DatanodeRegistration nodeReg, String poolId,
- Block blocks[], String delHints[]) throws IOException {
+ public void blockReceivedAndDeleted(DatanodeRegistration nodeReg, String poolId,
+ StorageReceivedDeletedBlocks[] receivedAndDeletedBlocks) throws IOException {
verifyRequest(nodeReg);
if(stateChangeLog.isDebugEnabled()) {
- stateChangeLog.debug("*BLOCK* NameNode.blockReceived: "
- +"from "+nodeReg.getName()+" "+blocks.length+" blocks.");
- }
- for (int i = 0; i < blocks.length; i++) {
- namesystem.getBlockManager().blockReceived(
- nodeReg, poolId, blocks[i], delHints[i]);
+ stateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: "
+ +"from "+nodeReg.getName()+" "+receivedAndDeletedBlocks.length
+ +" blocks.");
}
+ namesystem.getBlockManager().blockReceivedAndDeleted(
+ nodeReg, poolId, receivedAndDeletedBlocks[0].getBlocks());
}
@Override // DatanodeProtocol
@@ -862,7 +953,7 @@ class NameNodeRpcServer implements Namen
throw new AuthorizationException("Service Level Authorization not enabled!");
}
- this.server.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
+ this.clientRpcServer.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
if (this.serviceRpcServer != null) {
this.serviceRpcServer.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Sun Feb 26 23:32:06 2012
@@ -646,7 +646,7 @@ public class NamenodeFsck {
return (float) (totalReplicas) / (float) totalBlocks;
}
- /** {@inheritDoc} */
+ @Override
public String toString() {
StringBuilder res = new StringBuilder();
res.append("Status: ").append((isHealthy() ? "HEALTHY" : "CORRUPT"))
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Sun Feb 26 23:32:06 2012
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.JspHelper;
@@ -58,7 +59,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.io.MD5Hash;
-import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
@@ -123,8 +123,10 @@ public class SecondaryNameNode implement
/** checkpoint once every this many transactions, regardless of time */
private long checkpointTxnCount;
+ private FSNamesystem namesystem;
- /** {@inheritDoc} */
+
+ @Override
public String toString() {
return getClass().getSimpleName() + " Status"
+ "\nName Node Address : " + nameNodeAddr
@@ -214,9 +216,8 @@ public class SecondaryNameNode implement
nameNodeAddr = NameNode.getServiceAddress(conf, true);
this.conf = conf;
- this.namenode =
- (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
- NamenodeProtocol.versionID, nameNodeAddr, conf);
+ this.namenode = new NamenodeProtocolTranslatorPB(nameNodeAddr, conf,
+ UserGroupInformation.getCurrentUser());
// initialize checkpoint directories
fsName = getInfoServer();
@@ -226,6 +227,8 @@ public class SecondaryNameNode implement
"/tmp/hadoop/dfs/namesecondary");
checkpointImage = new CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs);
checkpointImage.recoverCreate(commandLineOpts.shouldFormat());
+
+ namesystem = new FSNamesystem(conf, checkpointImage);
// Initialize other scheduling parameters from the configuration
checkpointCheckPeriod = conf.getLong(
@@ -527,7 +530,7 @@ public class SecondaryNameNode implement
boolean loadImage = downloadCheckpointFiles(
fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits
- doMerge(sig, manifest, loadImage, checkpointImage);
+ doMerge(sig, manifest, loadImage, checkpointImage, namesystem);
//
// Upload the new image into the NameNode. Then tell the Namenode
@@ -757,8 +760,7 @@ public class SecondaryNameNode implement
CheckpointStorage(Configuration conf,
Collection<URI> imageDirs,
Collection<URI> editsDirs) throws IOException {
- super(conf, (FSNamesystem)null, imageDirs, editsDirs);
- setFSNamesystem(new FSNamesystem(this, conf));
+ super(conf, imageDirs, editsDirs);
// the 2NN never writes edits -- it only downloads them. So
// we shouldn't have any editLog instance. Setting to null
@@ -844,7 +846,8 @@ public class SecondaryNameNode implement
static void doMerge(
CheckpointSignature sig, RemoteEditLogManifest manifest,
- boolean loadImage, FSImage dstImage) throws IOException {
+ boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
+ throws IOException {
NNStorage dstStorage = dstImage.getStorage();
dstStorage.setStorageInfo(sig);
@@ -855,11 +858,11 @@ public class SecondaryNameNode implement
sig.mostRecentCheckpointTxId + " even though it should have " +
"just been downloaded");
}
- dstImage.reloadFromImageFile(file);
+ dstImage.reloadFromImageFile(file, dstNamesystem);
}
- Checkpointer.rollForwardByApplyingLogs(manifest, dstImage);
- dstImage.saveFSImageInAllDirs(dstImage.getLastAppliedTxId());
+ Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
+ dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
dstStorage.writeAll();
}
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java Sun Feb 26 23:32:06 2012
@@ -64,7 +64,7 @@ class SerialNumberManager {
return i2t.get(i);
}
- /** {@inheritDoc} */
+ @Override
public String toString() {
return "max=" + max + ",\n t2i=" + t2i + ",\n i2t=" + i2t;
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java Sun Feb 26 23:32:06 2012
@@ -44,6 +44,16 @@ import org.apache.hadoop.io.WritableFact
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class BlockCommand extends DatanodeCommand {
+
+ /**
+ * This constant is used to indicate that the block deletion does not need
+ * explicit ACK from the datanode. When a block is put into the list of blocks
+ * to be deleted, it's size is set to this constant. We assume that no block
+ * would actually have this size. Otherwise, we would miss ACKs for blocks
+ * with such size. Positive number is used for compatibility reasons.
+ */
+ public static final long NO_ACK = Long.MAX_VALUE;
+
String poolId;
Block blocks[];
DatanodeInfo targets[][];
@@ -57,7 +67,6 @@ public class BlockCommand extends Datano
public BlockCommand(int action, String poolId,
List<BlockTargetPair> blocktargetlist) {
super(action);
-
this.poolId = poolId;
blocks = new Block[blocktargetlist.size()];
targets = new DatanodeInfo[blocks.length][];
@@ -75,12 +84,21 @@ public class BlockCommand extends Datano
* @param blocks blocks related to the action
*/
public BlockCommand(int action, String poolId, Block blocks[]) {
+ this(action, poolId, blocks, EMPTY_TARGET);
+ }
+
+ /**
+ * Create BlockCommand for the given action
+ * @param blocks blocks related to the action
+ */
+ public BlockCommand(int action, String poolId, Block[] blocks,
+ DatanodeInfo[][] targets) {
super(action);
this.poolId = poolId;
this.blocks = blocks;
- this.targets = EMPTY_TARGET;
+ this.targets = targets;
}
-
+
public String getBlockPoolId() {
return poolId;
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java Sun Feb 26 23:32:06 2012
@@ -32,6 +32,8 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
+import com.google.common.base.Joiner;
+
/**
* BlockRecoveryCommand is an instruction to a data-node to recover
* the specified blocks.
@@ -117,8 +119,12 @@ public class BlockRecoveryCommand extend
* the specified capacity for recovering blocks.
*/
public BlockRecoveryCommand(int capacity) {
+ this(new ArrayList<RecoveringBlock>(capacity));
+ }
+
+ public BlockRecoveryCommand(Collection<RecoveringBlock> blocks) {
super(DatanodeProtocol.DNA_RECOVERBLOCK);
- recoveringBlocks = new ArrayList<RecoveringBlock>(capacity);
+ recoveringBlocks = blocks;
}
/**
@@ -134,6 +140,15 @@ public class BlockRecoveryCommand extend
public void add(RecoveringBlock block) {
recoveringBlocks.add(block);
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("BlockRecoveryCommand(\n ");
+ Joiner.on("\n ").appendTo(sb, recoveringBlocks);
+ sb.append("\n)");
+ return sb.toString();
+ }
///////////////////////////////////////////
// Writable
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java Sun Feb 26 23:32:06 2012
@@ -17,74 +17,16 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.avro.reflect.Union;
/**
* Base class for data-node command.
* Issued by the name-node to notify data-nodes what should be done.
*/
-
-// Declare subclasses for Avro's denormalized representation
-@Union({Void.class,
- DatanodeCommand.Register.class, DatanodeCommand.Finalize.class,
- BlockCommand.class, UpgradeCommand.class,
- BlockRecoveryCommand.class, KeyUpdateCommand.class})
-
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class DatanodeCommand extends ServerCommand {
- static class Register extends DatanodeCommand {
- private Register() {super(DatanodeProtocol.DNA_REGISTER);}
- public void readFields(DataInput in) {}
- public void write(DataOutput out) {}
- }
-
- public static class Finalize extends DatanodeCommand {
- String blockPoolId;
- private Finalize() {
- super(DatanodeProtocol.DNA_FINALIZE);
- }
-
- public Finalize(String bpid) {
- super(DatanodeProtocol.DNA_FINALIZE);
- blockPoolId = bpid;
- }
-
- public String getBlockPoolId() {
- return blockPoolId;
- }
-
- public void readFields(DataInput in) throws IOException {
- blockPoolId = WritableUtils.readString(in);
- }
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeString(out, blockPoolId);
- }
- }
-
- static { // register a ctor
- WritableFactories.setFactory(Register.class,
- new WritableFactory() {
- public Writable newInstance() {return new Register();}
- });
- WritableFactories.setFactory(Finalize.class,
- new WritableFactory() {
- public Writable newInstance() {return new Finalize();}
- });
- }
-
- public static final DatanodeCommand REGISTER = new Register();
-
public DatanodeCommand() {
super();
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Sun Feb 26 23:32:06 2012
@@ -22,15 +22,12 @@ import java.io.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
-import org.apache.avro.reflect.Nullable;
-
/**********************************************************************
* Protocol that a DFS datanode uses to communicate with the NameNode.
* It's used to upload current load information and block reports.
@@ -45,7 +42,15 @@ import org.apache.avro.reflect.Nullable;
@InterfaceAudience.Private
public interface DatanodeProtocol extends VersionedProtocol {
/**
- * 28: Add Balancer Bandwidth Command protocol.
+ * This class is used by both the Namenode (client) and BackupNode (server)
+ * to insulate from the protocol serialization.
+ *
+ * If you are adding/changing DN's interface then you need to
+ * change both this class and ALSO related protocol buffer
+ * wire protocol definition in DatanodeProtocol.proto.
+ *
+ * For more details on protocol buffer wire protocol, please see
+ * .../org/apache/hadoop/hdfs/protocolPB/overview.html
*/
public static final long versionID = 28L;
@@ -68,18 +73,23 @@ public interface DatanodeProtocol extend
final static int DNA_RECOVERBLOCK = 6; // request a block recovery
final static int DNA_ACCESSKEYUPDATE = 7; // update access key
final static int DNA_BALANCERBANDWIDTHUPDATE = 8; // update balancer bandwidth
+ final static int DNA_UC_ACTION_REPORT_STATUS = 100; // Report upgrade status
+ final static int DNA_UC_ACTION_START_UPGRADE = 101; // start upgrade
/**
* Register Datanode.
*
* @see org.apache.hadoop.hdfs.server.namenode.FSNamesystem#registerDatanode(DatanodeRegistration)
- *
+ * @param registration datanode registration information
+ * @param storages list of storages on the datanode``
* @return updated {@link org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration}, which contains
* new storageID if the datanode did not have one and
* registration ID for further communication.
*/
- public DatanodeRegistration registerDatanode(DatanodeRegistration registration
- ) throws IOException;
+ public DatanodeRegistration registerDatanode(
+ DatanodeRegistration registration, DatanodeStorage[] storages)
+ throws IOException;
+
/**
* sendHeartbeat() tells the NameNode that the DataNode is still
* alive and well. Includes some status info, too.
@@ -88,20 +98,14 @@ public interface DatanodeProtocol extend
* A DatanodeCommand tells the DataNode to invalidate local block(s),
* or to copy them to other DataNodes, etc.
* @param registration datanode registration information
- * @param capacity total storage capacity available at the datanode
- * @param dfsUsed storage used by HDFS
- * @param remaining remaining storage available for HDFS
- * @param blockPoolUsed storage used by the block pool
+ * @param reports utilization report per storage
* @param xmitsInProgress number of transfers from this datanode to others
* @param xceiverCount number of active transceiver threads
* @param failedVolumes number of failed volumes
* @throws IOException on error
*/
- @Nullable
public DatanodeCommand[] sendHeartbeat(DatanodeRegistration registration,
- long capacity,
- long dfsUsed, long remaining,
- long blockPoolUsed,
+ StorageReport[] reports,
int xmitsInProgress,
int xceiverCount,
int failedVolumes) throws IOException;
@@ -114,7 +118,7 @@ public interface DatanodeProtocol extend
* infrequently afterwards.
* @param registration
* @param poolId - the block pool ID for the blocks
- * @param blocks - the block list as an array of longs.
+ * @param reports - report of blocks per storage
* Each block is represented as 2 longs.
* This is done instead of Block[] to reduce memory used by block reports.
*
@@ -122,21 +126,22 @@ public interface DatanodeProtocol extend
* @throws IOException
*/
public DatanodeCommand blockReport(DatanodeRegistration registration,
- String poolId,
- long[] blocks) throws IOException;
+ String poolId, StorageBlockReport[] reports) throws IOException;
/**
- * blockReceived() allows the DataNode to tell the NameNode about
- * recently-received block data, with a hint for pereferred replica
- * to be deleted when there is any excessive blocks.
+ * blockReceivedAndDeleted() allows the DataNode to tell the NameNode about
+ * recently-received and -deleted block data.
+ *
+ * For the case of received blocks, a hint for preferred replica to be
+ * deleted when there is any excessive blocks is provided.
* For example, whenever client code
* writes a new Block here, or another DataNode copies a Block to
* this DataNode, it will call blockReceived().
*/
- public void blockReceived(DatanodeRegistration registration,
+ public void blockReceivedAndDeleted(DatanodeRegistration registration,
String poolId,
- Block blocks[],
- String[] delHints) throws IOException;
+ StorageReceivedDeletedBlocks[] rcvdAndDeletedBlocks)
+ throws IOException;
/**
* errorReport() tells the NameNode about something that has gone
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java Sun Feb 26 23:32:06 2012
@@ -63,9 +63,21 @@ implements Writable, NodeRegistration {
* Create DatanodeRegistration
*/
public DatanodeRegistration(String nodeName) {
+ this(nodeName, new StorageInfo(), new ExportedBlockKeys());
+ }
+
+ public DatanodeRegistration(DatanodeID dn, StorageInfo info,
+ ExportedBlockKeys keys) {
+ super(dn);
+ this.storageInfo = info;
+ this.exportedKeys = keys;
+ }
+
+ public DatanodeRegistration(String nodeName, StorageInfo info,
+ ExportedBlockKeys keys) {
super(nodeName);
- this.storageInfo = new StorageInfo();
- this.exportedKeys = new ExportedBlockKeys();
+ this.storageInfo = info;
+ this.exportedKeys = keys;
}
public void setStorageInfo(StorageInfo storage) {
@@ -101,7 +113,7 @@ implements Writable, NodeRegistration {
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
- /** {@inheritDoc} */
+ @Override
public void write(DataOutput out) throws IOException {
super.write(out);
@@ -112,7 +124,7 @@ implements Writable, NodeRegistration {
exportedKeys.write(out);
}
- /** {@inheritDoc} */
+ @Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
Copied: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java (from r1293950, hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java?p2=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java&p1=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java&r1=1293950&r2=1293964&rev=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java Sun Feb 26 23:32:06 2012
@@ -15,25 +15,30 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.hadoop.hdfs.security.token.block;
-
-import javax.crypto.SecretKey;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
+package org.apache.hadoop.hdfs.server.protocol;
/**
- * Key used for generating and verifying block tokens
+ * Class capatures information about a storage in Datanode
*/
-@InterfaceAudience.Private
-public class BlockKey extends DelegationKey {
+public class DatanodeStorage {
+ public enum State {
+ NORMAL,
+ READ_ONLY
+ }
+
+ private final String storageID;
+ private final State state;
+
+ public DatanodeStorage(String sid, State s) {
+ storageID = sid;
+ state = s;
+ }
- public BlockKey() {
- super();
+ public String getStorageID() {
+ return storageID;
}
- public BlockKey(int keyId, long expiryDate, SecretKey key) {
- super(keyId, expiryDate, key);
+ public State getState() {
+ return state;
}
}
Added: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java?rev=1293964&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java (added)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java Sun Feb 26 23:32:06 2012
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * A BlockCommand is an instruction to a datanode to register with the namenode.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class FinalizeCommand extends DatanodeCommand {
+ // /////////////////////////////////////////
+ // Writable
+ // /////////////////////////////////////////
+ static { // register a ctor
+ WritableFactories.setFactory(FinalizeCommand.class, new WritableFactory() {
+ public Writable newInstance() {
+ return new FinalizeCommand();
+ }
+ });
+ }
+
+ String blockPoolId;
+ private FinalizeCommand() {
+ super(DatanodeProtocol.DNA_FINALIZE);
+ }
+
+ public FinalizeCommand(String bpid) {
+ super(DatanodeProtocol.DNA_FINALIZE);
+ blockPoolId = bpid;
+ }
+
+ public String getBlockPoolId() {
+ return blockPoolId;
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ blockPoolId = WritableUtils.readString(in);
+ }
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeString(out, blockPoolId);
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java Sun Feb 26 23:32:06 2012
@@ -39,6 +39,24 @@ public interface InterDatanodeProtocol e
public static final Log LOG = LogFactory.getLog(InterDatanodeProtocol.class);
/**
+ * Until version 9, this class InterDatanodeProtocol served as both
+ * the interface to the DN AND the RPC protocol used to communicate with the
+ * DN.
+ *
+ * Post version 6L (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/InterDatanodeWireProtocol}
+ *
+ * This class is used by both the DN to insulate from the protocol
+ * serialization.
+ *
+ * If you are adding/changing DN's interface then you need to
+ * change both this class and ALSO related protocol buffer
+ * wire protocol definition in InterDatanodeProtocol.proto.
+ *
+ * For more details on protocol buffer wire protocol, please see
+ * .../org/apache/hadoop/hdfs/protocolPB/overview.html
+ *
+ * The log of historical changes can be retrieved from the svn).
* 6: Add block pool ID to Block
*/
public static final long versionID = 6L;
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalProtocol.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/JournalProtocol.java Sun Feb 26 23:32:06 2012
@@ -33,6 +33,18 @@ import org.apache.hadoop.security.Kerber
clientPrincipal = DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)
@InterfaceAudience.Private
public interface JournalProtocol extends VersionedProtocol {
+ /**
+ *
+ * This class is used by both the Namenode (client) and BackupNode (server)
+ * to insulate from the protocol serialization.
+ *
+ * If you are adding/changing DN's interface then you need to
+ * change both this class and ALSO related protocol buffer
+ * wire protocol definition in JournalProtocol.proto.
+ *
+ * For more details on protocol buffer wire protocol, please see
+ * .../org/apache/hadoop/hdfs/protocolPB/overview.html
+ */
public static final long versionID = 1L;
/**
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Sun Feb 26 23:32:06 2012
@@ -38,9 +38,22 @@ import org.apache.hadoop.security.Kerber
@InterfaceAudience.Private
public interface NamenodeProtocol extends VersionedProtocol {
/**
- * Compared to the previous version the following changes have been introduced:
- * (Only the latest change is reflected.
- * The log of historical changes can be retrieved from the svn).
+ * Until version 6L, this class served as both
+ * the client interface to the NN AND the RPC protocol used to
+ * communicate with the NN.
+ *
+ * Post version 70 (release 23 of Hadoop), the protocol is implemented in
+ * {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
+ *
+ * This class is used by both the DFSClient and the
+ * NN server side to insulate from the protocol serialization.
+ *
+ * If you are adding/changing NN's interface then you need to
+ * change both this class and ALSO related protocol buffer
+ * wire protocol definition in NamenodeProtocol.proto.
+ *
+ * For more details on protocol buffer wire protocol, please see
+ * .../org/apache/hadoop/hdfs/protocolPB/overview.html
*
* 6: Switch to txid-based file naming for image and edits
*/
@@ -62,7 +75,7 @@ public interface NamenodeProtocol extend
* @param datanode a data node
* @param size requested size
* @return a list of blocks & their locations
- * @throws RemoteException if size is less than or equal to 0 or
+ * @throws IOException if size is less than or equal to 0 or
datanode does not exist
*/
public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java Sun Feb 26 23:32:06 2012
@@ -52,10 +52,9 @@ implements NodeRegistration {
String httpAddress,
StorageInfo storageInfo,
NamenodeRole role) {
- super();
+ super(storageInfo);
this.rpcAddress = address;
this.httpAddress = httpAddress;
- this.setStorageInfo(storageInfo);
this.role = role;
}
@@ -64,6 +63,10 @@ implements NodeRegistration {
return rpcAddress;
}
+ public String getHttpAddress() {
+ return httpAddress;
+ }
+
@Override // NodeRegistration
public String getRegistrationID() {
return Storage.getRegistrationID(this);
Added: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java?rev=1293964&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java (added)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java Sun Feb 26 23:32:06 2012
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.protocol;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A data structure to store Block and delHints together, used to send
+ * received/deleted ACKs.
+ */
+public class ReceivedDeletedBlockInfo implements Writable {
+ Block block;
+ String delHints;
+
+ public final static String TODELETE_HINT = "-";
+
+ public ReceivedDeletedBlockInfo() {
+ }
+
+ public ReceivedDeletedBlockInfo(Block blk, String delHints) {
+ this.block = blk;
+ this.delHints = delHints;
+ }
+
+ public Block getBlock() {
+ return this.block;
+ }
+
+ public void setBlock(Block blk) {
+ this.block = blk;
+ }
+
+ public String getDelHints() {
+ return this.delHints;
+ }
+
+ public void setDelHints(String hints) {
+ this.delHints = hints;
+ }
+
+ public boolean equals(Object o) {
+ if (!(o instanceof ReceivedDeletedBlockInfo)) {
+ return false;
+ }
+ ReceivedDeletedBlockInfo other = (ReceivedDeletedBlockInfo) o;
+ return this.block.equals(other.getBlock())
+ && this.delHints.equals(other.delHints);
+ }
+
+ public int hashCode() {
+ assert false : "hashCode not designed";
+ return 0;
+ }
+
+ public boolean blockEquals(Block b) {
+ return this.block.equals(b);
+ }
+
+ public boolean isDeletedBlock() {
+ return delHints.equals(TODELETE_HINT);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ this.block.write(out);
+ Text.writeString(out, this.delHints);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.block = new Block();
+ this.block.readFields(in);
+ this.delHints = Text.readString(in);
+ }
+
+ public String toString() {
+ return block.toString() + ", delHint: " + delHints;
+ }
+}
Added: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java?rev=1293964&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java (added)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java Sun Feb 26 23:32:06 2012
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+
+/**
+ * A BlockCommand is an instruction to a datanode to register with the namenode.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class RegisterCommand extends DatanodeCommand {
+ // /////////////////////////////////////////
+ // Writable
+ // /////////////////////////////////////////
+ static { // register a ctor
+ WritableFactories.setFactory(RegisterCommand.class, new WritableFactory() {
+ public Writable newInstance() {
+ return new RegisterCommand();
+ }
+ });
+ }
+
+ public static final DatanodeCommand REGISTER = new RegisterCommand();
+
+ public RegisterCommand() {
+ super(DatanodeProtocol.DNA_REGISTER);
+ }
+
+ @Override
+ public void readFields(DataInput in) { }
+
+ @Override
+ public void write(DataOutput out) { }
+}
\ No newline at end of file
Copied: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageBlockReport.java (from r1293950, hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageBlockReport.java?p2=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageBlockReport.java&p1=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java&r1=1293950&r2=1293964&rev=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageBlockReport.java Sun Feb 26 23:32:06 2012
@@ -16,24 +16,25 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.security.token.block;
-
-import javax.crypto.SecretKey;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
+package org.apache.hadoop.hdfs.server.protocol;
/**
- * Key used for generating and verifying block tokens
+ * Block report for a Datanode storage
*/
-@InterfaceAudience.Private
-public class BlockKey extends DelegationKey {
+public class StorageBlockReport {
+ private final String storageID;
+ private final long[] blocks;
+
+ public StorageBlockReport(String sid, long[] blocks) {
+ this.storageID = sid;
+ this.blocks = blocks;
+ }
- public BlockKey() {
- super();
+ public String getStorageID() {
+ return storageID;
}
- public BlockKey(int keyId, long expiryDate, SecretKey key) {
- super(keyId, expiryDate, key);
+ public long[] getBlocks() {
+ return blocks;
}
}
Copied: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java (from r1293950, hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java?p2=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java&p1=hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java&r1=1293950&r2=1293964&rev=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockKey.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReceivedDeletedBlocks.java Sun Feb 26 23:32:06 2012
@@ -16,24 +16,27 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.security.token.block;
-
-import javax.crypto.SecretKey;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
+package org.apache.hadoop.hdfs.server.protocol;
/**
- * Key used for generating and verifying block tokens
+ * Report of block received and deleted per Datanode
+ * storage.
*/
-@InterfaceAudience.Private
-public class BlockKey extends DelegationKey {
+public class StorageReceivedDeletedBlocks {
+ private final String storageID;
+ private final ReceivedDeletedBlockInfo[] blocks;
+
+ public String getStorageID() {
+ return storageID;
+ }
- public BlockKey() {
- super();
+ public ReceivedDeletedBlockInfo[] getBlocks() {
+ return blocks;
}
- public BlockKey(int keyId, long expiryDate, SecretKey key) {
- super(keyId, expiryDate, key);
+ public StorageReceivedDeletedBlocks(final String storageID,
+ final ReceivedDeletedBlockInfo[] blocks) {
+ this.storageID = storageID;
+ this.blocks = blocks;
}
}
Added: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java?rev=1293964&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java (added)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java Sun Feb 26 23:32:06 2012
@@ -0,0 +1,47 @@
+package org.apache.hadoop.hdfs.server.protocol;
+
+/**
+ * Utilization report for a Datanode storage
+ */
+public class StorageReport {
+ private final String storageID;
+ private final boolean failed;
+ private final long capacity;
+ private final long dfsUsed;
+ private final long remaining;
+ private final long blockPoolUsed;
+
+ public StorageReport(String sid, boolean failed, long capacity, long dfsUsed,
+ long remaining, long bpUsed) {
+ this.storageID = sid;
+ this.failed = failed;
+ this.capacity = capacity;
+ this.dfsUsed = dfsUsed;
+ this.remaining = remaining;
+ this.blockPoolUsed = bpUsed;
+ }
+
+ public String getStorageID() {
+ return storageID;
+ }
+
+ public boolean isFailed() {
+ return failed;
+ }
+
+ public long getCapacity() {
+ return capacity;
+ }
+
+ public long getDfsUsed() {
+ return dfsUsed;
+ }
+
+ public long getRemaining() {
+ return remaining;
+ }
+
+ public long getBlockPoolUsed() {
+ return blockPoolUsed;
+ }
+}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java Sun Feb 26 23:32:06 2012
@@ -40,9 +40,11 @@ import org.apache.hadoop.io.WritableFact
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class UpgradeCommand extends DatanodeCommand {
- final static int UC_ACTION_UNKNOWN = DatanodeProtocol.DNA_UNKNOWN;
- public final static int UC_ACTION_REPORT_STATUS = 100; // report upgrade status
- public final static int UC_ACTION_START_UPGRADE = 101; // start upgrade
+ public final static int UC_ACTION_UNKNOWN = DatanodeProtocol.DNA_UNKNOWN;
+ public final static int UC_ACTION_REPORT_STATUS =
+ DatanodeProtocol.DNA_UC_ACTION_REPORT_STATUS;
+ public final static int UC_ACTION_START_UPGRADE =
+ DatanodeProtocol.DNA_UC_ACTION_START_UPGRADE;
private int version;
private short upgradeStatus;
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java Sun Feb 26 23:32:06 2012
@@ -43,14 +43,14 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
+import org.apache.hadoop.hdfs.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -791,13 +791,9 @@ public class DFSAdmin extends FsShell {
conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
- RefreshAuthorizationPolicyProtocol refreshProtocol =
- (RefreshAuthorizationPolicyProtocol)
- RPC.getProxy(RefreshAuthorizationPolicyProtocol.class,
- RefreshAuthorizationPolicyProtocol.versionID,
- NameNode.getAddress(conf), getUGI(), conf,
- NetUtils.getSocketFactory(conf,
- RefreshAuthorizationPolicyProtocol.class));
+ RefreshAuthorizationPolicyProtocolClientSideTranslatorPB refreshProtocol =
+ new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(
+ NameNode.getAddress(conf), getUGI(), conf);
// Refresh the authorization policy in-effect
refreshProtocol.refreshServiceAcl();
@@ -821,13 +817,9 @@ public class DFSAdmin extends FsShell {
conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
- RefreshUserMappingsProtocol refreshProtocol =
- (RefreshUserMappingsProtocol)
- RPC.getProxy(RefreshUserMappingsProtocol.class,
- RefreshUserMappingsProtocol.versionID,
- NameNode.getAddress(conf), getUGI(), conf,
- NetUtils.getSocketFactory(conf,
- RefreshUserMappingsProtocol.class));
+ RefreshUserMappingsProtocolClientSideTranslatorPB refreshProtocol =
+ new RefreshUserMappingsProtocolClientSideTranslatorPB(
+ NameNode.getAddress(conf), getUGI(), conf);
// Refresh the user-to-groups mappings
refreshProtocol.refreshUserToGroupsMappings();
@@ -852,13 +844,9 @@ public class DFSAdmin extends FsShell {
conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, ""));
// Create the client
- RefreshUserMappingsProtocol refreshProtocol =
- (RefreshUserMappingsProtocol)
- RPC.getProxy(RefreshUserMappingsProtocol.class,
- RefreshUserMappingsProtocol.versionID,
- NameNode.getAddress(conf), getUGI(), conf,
- NetUtils.getSocketFactory(conf,
- RefreshUserMappingsProtocol.class));
+ RefreshUserMappingsProtocolClientSideTranslatorPB refreshProtocol =
+ new RefreshUserMappingsProtocolClientSideTranslatorPB(
+ NameNode.getAddress(conf), getUGI(), conf);
// Refresh the user-to-groups mappings
refreshProtocol.refreshSuperUserGroupsConfiguration();
@@ -1148,10 +1136,9 @@ public class DFSAdmin extends FsShell {
conf.get(DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY, ""));
// Create the client
- ClientDatanodeProtocol dnProtocol = RPC.getProxy(
- ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
- datanodeAddr, getUGI(), conf, NetUtils.getSocketFactory(conf,
- ClientDatanodeProtocol.class));
+ ClientDatanodeProtocol dnProtocol =
+ DFSUtil.createClientDatanodeProtocolProxy(datanodeAddr, getUGI(), conf,
+ NetUtils.getSocketFactory(conf, ClientDatanodeProtocol.class));
return dnProtocol;
}
Modified: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java?rev=1293964&r1=1293963&r2=1293964&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java (original)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java Sun Feb 26 23:32:06 2012
@@ -23,8 +23,11 @@ import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.GetGroupsBase;
+import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.apache.hadoop.util.ToolRunner;
/**
@@ -51,6 +54,13 @@ public class GetGroups extends GetGroups
throws IOException {
return NameNode.getAddress(conf);
}
+
+ @Override
+ protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
+ return new GetUserMappingsProtocolClientSideTranslatorPB(
+ NameNode.getAddress(getConf()), UserGroupInformation.getCurrentUser(),
+ getConf());
+ }
public static void main(String[] argv) throws Exception {
int res = ToolRunner.run(new GetGroups(new HdfsConfiguration()), argv);
Added: hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto?rev=1293964&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto (added)
+++ hadoop/common/branches/branch-0.23-PB-merge/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto Sun Feb 26 23:32:06 2012
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used throughout HDFS -- i.e.
+// by the client, server, and data transfer protocols.
+
+option java_package = "org.apache.hadoop.hdfs.protocol.proto";
+option java_outer_classname = "ClientDatanodeProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "hdfs.proto";
+
+/**
+ * block - block for which visible length is requested
+ */
+message GetReplicaVisibleLengthRequestProto {
+ required ExtendedBlockProto block = 1;
+}
+
+/**
+ * length - visible length of the block
+ */
+message GetReplicaVisibleLengthResponseProto {
+ required uint64 length = 1;
+}
+
+/**
+ * void request
+ */
+message RefreshNamenodesRequestProto {
+}
+
+/**
+ * void response
+ */
+message RefreshNamenodesResponseProto {
+}
+
+/**
+ * blockPool - block pool to be deleted
+ * force - if false, delete the block pool only if it is empty.
+ * if true, delete the block pool even if it has blocks.
+ */
+message DeleteBlockPoolRequestProto {
+ required string blockPool = 1;
+ required bool force = 2;
+}
+
+/**
+ * void response
+ */
+message DeleteBlockPoolResponseProto {
+}
+
+/**
+ * Gets the file information where block and its metadata is stored
+ * block - block for which path information is being requested
+ * token - block token
+ */
+message GetBlockLocalPathInfoRequestProto {
+ required ExtendedBlockProto block = 1;
+ required BlockTokenIdentifierProto token = 2;
+}
+
+/**
+ * block - block for which file path information is being returned
+ * localPath - file path where the block data is stored
+ * localMetaPath - file path where the block meta data is stored
+ */
+message GetBlockLocalPathInfoResponseProto {
+ required ExtendedBlockProto block = 1;
+ required string localPath = 2;
+ required string localMetaPath = 3;
+}
+
+/**
+ * Protocol used from client to the Datanode.
+ * See the request and response for details of rpc call.
+ */
+service ClientDatanodeProtocolService {
+ /**
+ * Returns the visible length of the replica
+ */
+ rpc getReplicaVisibleLength(GetReplicaVisibleLengthRequestProto)
+ returns(GetReplicaVisibleLengthResponseProto);
+
+ /**
+ * Refresh the list of federated namenodes from updated configuration.
+ * Adds new namenodes and stops the deleted namenodes.
+ */
+ rpc refreshNamenodes(RefreshNamenodesRequestProto)
+ returns(RefreshNamenodesResponseProto);
+
+ /**
+ * Delete the block pool from the datanode.
+ */
+ rpc deleteBlockPool(DeleteBlockPoolRequestProto)
+ returns(DeleteBlockPoolResponseProto);
+
+ /**
+ * Retrieves the path names of the block file and metadata file stored on the
+ * local file system.
+ */
+ rpc getBlockLocalPathInfo(GetBlockLocalPathInfoRequestProto)
+ returns(GetBlockLocalPathInfoResponseProto);
+}