You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2009/09/27 04:23:42 UTC
svn commit: r819243 - in /hadoop/hdfs/branches/HDFS-265: ./
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/server/protocol/
Author: shv
Date: Sun Sep 27 02:23:41 2009
New Revision: 819243
URL: http://svn.apache.org/viewvc?rev=819243&view=rev
Log:
HDFS-657. Remove unused legacy data-node protocol methods. Contributed by Konstantin Shvachko.
Modified:
hadoop/hdfs/branches/HDFS-265/CHANGES.txt
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
Modified: hadoop/hdfs/branches/HDFS-265/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/CHANGES.txt?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-265/CHANGES.txt Sun Sep 27 02:23:41 2009
@@ -15,6 +15,8 @@
HDFS-570. Get last block length from a data-node when opening a file
being written to. (Tsz Wo (Nicholas), SZE via shv)
+ HDFS-657. Remove unused legacy data-node protocol methods. (shv)
+
NEW FEATURES
HDFS-536. Support hflush at DFSClient. (hairong)
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java Sun Sep 27 02:23:41 2009
@@ -29,22 +29,9 @@
public static final Log LOG = LogFactory.getLog(ClientDatanodeProtocol.class);
/**
- * 5: add getReplicaVisibleLength(..)
+ * 6: recoverBlock() removed.
*/
- public static final long versionID = 5L;
-
- /** Start generation-stamp recovery for specified block
- * @param block the specified block
- * @param keepLength keep the block length
- * @param targets the list of possible locations of specified block
- * @return either a new generation stamp, or the original generation stamp.
- * Regardless of whether a new generation stamp is returned, a newly
- * generated access token is returned as part of the return value.
- * @throws IOException
- */
- @Deprecated // not used anymore - should be removed
- LocatedBlock recoverBlock(Block block, boolean keepLength,
- DatanodeInfo[] targets) throws IOException;
+ public static final long versionID = 6L;
/** Return the visible length of a replica. */
long getReplicaVisibleLength(Block b) throws IOException;
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Sun Sep 27 02:23:41 2009
@@ -36,10 +36,8 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
@@ -58,7 +56,6 @@
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.protocol.DataTransferProtocol.BlockConstructionStage;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -172,8 +169,6 @@
volatile boolean shouldRun = true;
private LinkedList<Block> receivedBlockList = new LinkedList<Block>();
- /** list of blocks being recovered */
- private final Map<Block, Block> ongoingRecovery = new HashMap<Block, Block>();
private LinkedList<String> delHints = new LinkedList<String>();
public final static String EMPTY_DEL_HINT = "";
AtomicInteger xmitsInProgress = new AtomicInteger();
@@ -1587,55 +1582,36 @@
Block block = rBlock.getBlock();
DatanodeInfo[] targets = rBlock.getLocations();
DatanodeID[] datanodeids = (DatanodeID[])targets;
- // If the block is already being recovered, then skip recovering it.
- // This can happen if the namenode and client start recovering the same
- // file at the same time.
- synchronized (ongoingRecovery) {
- if (ongoingRecovery.get(new Block(block.getBlockId(), block.getNumBytes(),
- GenerationStamp.WILDCARD_STAMP)) != null) {
- String msg = "Block " + block + " is already being recovered, " +
- " ignoring this request to recover it.";
- LOG.info(msg);
- throw new IOException(msg);
- }
- ongoingRecovery.put(block, block);
- }
- try {
- List<BlockRecord> syncList = new ArrayList<BlockRecord>();
- long minlength = Long.MAX_VALUE;
- int errorCount = 0;
+ List<BlockRecord> syncList = new ArrayList<BlockRecord>();
+ long minlength = Long.MAX_VALUE;
+ int errorCount = 0;
- //check generation stamps
- for(DatanodeID id : datanodeids) {
- try {
- InterDatanodeProtocol datanode = dnRegistration.equals(id)?
- this: DataNode.createInterDataNodeProtocolProxy(id, getConf());
- BlockMetaDataInfo info = datanode.getBlockMetaDataInfo(block);
- if (info != null && info.getGenerationStamp() >= block.getGenerationStamp()) {
- syncList.add(new BlockRecord(id, datanode, new Block(info)));
- if (info.getNumBytes() < minlength) {
- minlength = info.getNumBytes();
- }
+ //check generation stamps
+ for(DatanodeID id : datanodeids) {
+ try {
+ InterDatanodeProtocol datanode = dnRegistration.equals(id)?
+ this: DataNode.createInterDataNodeProtocolProxy(id, getConf());
+ BlockMetaDataInfo info = datanode.getBlockMetaDataInfo(block);
+ if (info != null && info.getGenerationStamp() >= block.getGenerationStamp()) {
+ syncList.add(new BlockRecord(id, datanode, new Block(info)));
+ if (info.getNumBytes() < minlength) {
+ minlength = info.getNumBytes();
}
- } catch (IOException e) {
- ++errorCount;
- InterDatanodeProtocol.LOG.warn(
- "Failed to getBlockMetaDataInfo for block (=" + block
- + ") from datanode (=" + id + ")", e);
}
+ } catch (IOException e) {
+ ++errorCount;
+ InterDatanodeProtocol.LOG.warn(
+ "Failed to getBlockMetaDataInfo for block (=" + block
+ + ") from datanode (=" + id + ")", e);
}
+ }
- if (syncList.isEmpty() && errorCount > 0) {
- throw new IOException("All datanodes failed: block=" + block
- + ", datanodeids=" + Arrays.asList(datanodeids));
- }
- block.setNumBytes(minlength);
- return syncBlock(rBlock, syncList);
- } finally {
- synchronized (ongoingRecovery) {
- ongoingRecovery.remove(block);
- }
+ if (syncList.isEmpty() && errorCount > 0) {
+ throw new IOException("All datanodes failed: block=" + block
+ + ", datanodeids=" + Arrays.asList(datanodeids));
}
+ block.setNumBytes(minlength);
+ return syncBlock(rBlock, syncList);
}
/** Block synchronization */
@@ -1706,16 +1682,6 @@
+ syncList.size() + " datanodes success {" + b + "\n}");
}
- // ClientDataNodeProtocol implementation
- /** {@inheritDoc} */
- @SuppressWarnings("deprecation")
- public LocatedBlock recoverBlock(Block block, boolean keepLength, DatanodeInfo[] targets
- ) throws IOException {
- logRecoverBlock("Client", block, targets);
- assert false : "ClientDatanodeProtocol.recoverBlock: should never be called.";
- return null;
- }
-
private static void logRecoverBlock(String who,
Block block, DatanodeID[] targets) {
StringBuilder msg = new StringBuilder(targets[0].getName());
@@ -1726,8 +1692,9 @@
+ ", targets=[" + msg + "])");
}
+ // ClientDataNodeProtocol implementation
/** {@inheritDoc} */
- @Override
+ @Override // ClientDataNodeProtocol
public long getReplicaVisibleLength(final Block block) throws IOException {
final Replica replica = data.getReplica(block.getBlockId());
if (replica == null) {
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java Sun Sep 27 02:23:41 2009
@@ -307,7 +307,6 @@
BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this;
ucBlock.setBlockUCState(s);
ucBlock.setExpectedLocations(targets);
- ucBlock.setLastRecoveryTime(0);
return ucBlock;
}
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfoUnderConstruction.java Sun Sep 27 02:23:41 2009
@@ -42,9 +42,6 @@
/** A data-node responsible for block recovery. */
private int primaryNodeIndex = -1;
- /** The last time the block was recovered. */
- private long lastRecoveryTime = 0;
-
/**
* The new generation stamp, which this block will have
* after the recovery succeeds. Also used as a recovery id to identify
@@ -234,18 +231,6 @@
}
}
- /**
- * Update lastRecoveryTime if expired.
- * @return true if lastRecoveryTimeis updated.
- */
- boolean setLastRecoveryTime(long now) {
- boolean expired = now - lastRecoveryTime > NameNode.LEASE_RECOVER_PERIOD;
- if (expired) {
- lastRecoveryTime = now;
- }
- return expired;
- }
-
void addReplicaIfNotPresent(DatanodeDescriptor dn,
Block block,
ReplicaState rState) {
@@ -275,7 +260,6 @@
.append("\n blockUCState=").append(blockUCState)
.append("\n replicas=").append(replicas)
.append("\n primaryNodeIndex=").append(primaryNodeIndex)
- .append("\n lastRecoveryTime=").append(lastRecoveryTime)
.append("}");
return b.toString();
}
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sun Sep 27 02:23:41 2009
@@ -3866,33 +3866,6 @@
return gs;
}
- /**
- * Verifies that the block is associated with a file that has a lease.
- * Increments, logs and then returns the stamp
- */
- synchronized long nextGenerationStampForBlock(Block block) throws IOException {
- BlockInfo storedBlock = blockManager.getStoredBlock(block);
- if (storedBlock == null) {
- String msg = block + " is already commited, storedBlock == null.";
- LOG.info(msg);
- throw new IOException(msg);
- }
- INodeFile fileINode = storedBlock.getINode();
- if(!fileINode.isUnderConstruction() || storedBlock.isComplete()) {
- String msg = block +
- " is already commited, file or block is not under construction().";
- LOG.info(msg);
- throw new IOException(msg);
- }
- if(!((BlockInfoUnderConstruction)storedBlock).setLastRecoveryTime(now())) {
- String msg = block + " is beening recovered, ignoring this request.";
- LOG.info(msg);
- throw new IOException(msg);
- }
- return nextGenerationStamp();
- }
-
-
private INodeFileUnderConstruction checkUCBlock(Block block, String clientName)
throws IOException {
// check safe mode
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sun Sep 27 02:23:41 2009
@@ -667,10 +667,6 @@
}
/** {@inheritDoc} */
- public long nextGenerationStamp(Block block) throws IOException{
- return namesystem.nextGenerationStampForBlock(block);
- }
-
@Override
public LocatedBlock updateBlockForPipeline(Block block, String clientName)
throws IOException {
Modified: hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java?rev=819243&r1=819242&r2=819243&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java (original)
+++ hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java Sun Sep 27 02:23:41 2009
@@ -35,9 +35,9 @@
**********************************************************************/
public interface DatanodeProtocol extends VersionedProtocol {
/**
- * 22: BlockRecoveryCommand introduced in reply to sendHeartbeat().
+ * 23: nextGenerationStamp() removed.
*/
- public static final long versionID = 22L;
+ public static final long versionID = 23L;
// error code
final static int NOTIFY = 0;
@@ -142,12 +142,6 @@
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException;
/**
- * @return the next GenerationStamp to be associated with the specified
- * block.
- */
- public long nextGenerationStamp(Block block) throws IOException;
-
- /**
* Commit block synchronization in lease recovery
*/
public void commitBlockSynchronization(Block block,