You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ha...@apache.org on 2009/10/17 02:41:25 UTC
svn commit: r826150 - in /hadoop/hdfs/branches/branch-0.21: ./
.eclipse.templates/.launches/ src/contrib/block_forensics/
src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/ja...
Author: hairong
Date: Sat Oct 17 00:41:25 2009
New Revision: 826150
URL: http://svn.apache.org/viewvc?rev=826150&view=rev
Log:
Merge -c 826149 to move the change of HDFS-668 from trunk to branch 0.21.
Modified:
hadoop/hdfs/branches/branch-0.21/ (props changed)
hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/ (props changed)
hadoop/hdfs/branches/branch-0.21/CHANGES.txt
hadoop/hdfs/branches/branch-0.21/build.xml (props changed)
hadoop/hdfs/branches/branch-0.21/src/contrib/block_forensics/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend3.java
hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.21/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
-/hadoop/hdfs/trunk:818294-818298,824552,824944
+/hadoop/hdfs/trunk:818294-818298,824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/.eclipse.templates/.launches:824552,824944
+/hadoop/hdfs/trunk/.eclipse.templates/.launches:824552,824944,826149
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Sat Oct 17 00:41:25 2009
@@ -394,6 +394,8 @@
HDFS-682. Fix bugs in TestBlockUnderConstruction. (szetszwo)
+ HDFS-668. TestFileAppend3#TC7 sometimes hangs. (hairong)
+
Release 0.20.2 - Unreleased
BUG FIXES
Propchange: hadoop/hdfs/branches/branch-0.21/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
-/hadoop/hdfs/trunk/build.xml:818294-818298,824552,824944,825229
+/hadoop/hdfs/trunk/build.xml:818294-818298,824552,824944,825229,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/src/contrib/block_forensics:824552,824944
+/hadoop/hdfs/trunk/src/contrib/block_forensics:824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:818294-818298,824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
-/hadoop/hdfs/trunk/src/java:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/java:818294-818298,824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,5 +1,5 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:713112
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:776175-785643,785929-786278
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:817353-818319,818321-818553
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:824552,824944
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:824552,824944,826149
/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:796829-800617,800619-803337,804756-805652,808672-809439,811495-813103,813105-813630,814223-815964,818294-818298
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Sat Oct 17 00:41:25 2009
@@ -1102,7 +1102,7 @@
// construct a RBW replica with the new GS
File blkfile = replicaInfo.getBlockFile();
FSVolume v = volumes.getNextVolume(estimateBlockLen);
- File newBlkFile = v.createRbwFile(replicaInfo);
+ File newBlkFile = new File(v.rbwDir, replicaInfo.getBlockName());
File oldmeta = replicaInfo.getMetaFile();
ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(
replicaInfo.getBlockId(), replicaInfo.getNumBytes(), newGS,
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:818294-818298,824552,824944,826149
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Sat Oct 17 00:41:25 2009
@@ -1000,7 +1000,7 @@
DatanodeDescriptor node,
DatanodeDescriptor delNodeHint)
throws IOException {
- BlockInfo storedBlock = blocksMap.getStoredBlock(block);
+ BlockInfo storedBlock = findStoredBlock(block.getBlockId());
if (storedBlock == null || storedBlock.getINode() == null) {
// If this block does not belong to anyfile, then we are done.
NameNode.stateChangeLog.info("BLOCK* NameSystem.addStoredBlock: "
@@ -1668,6 +1668,18 @@
blocksMap.removeBlock(block);
}
+ /**
+ * Update the block with the new generation stamp and new length.
+ *
+ * @param block block
+ * @param newGS new generation stamp
+ * @param newLen new block size
+ * @return the stored block in the blocks map
+ */
+ BlockInfo updateBlock(Block block, long newGS, long newLen) {
+ return blocksMap.updateBlock(block, newGS, newLen);
+ }
+
int getCapacity() {
synchronized(namesystem) {
return blocksMap.getCapacity();
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/BlocksMap.java Sat Oct 17 00:41:25 2009
@@ -86,6 +86,25 @@
}
/**
+ * Update the old block with the new generation stamp and new length.
+ *
+ * After update, the block has a newer generation stamp so it requires
+ * to remove the old entry first and reinsert the entry
+ *
+ * @param block block
+ * @param newGS new generation stamp
+ * @param newLen new block size
+ * @return the stored block in the map
+ */
+ BlockInfo updateBlock(Block block, long newGS, long newLen) {
+ BlockInfo blockInfo = map.remove(block);
+ blockInfo.setGenerationStamp(newGS);
+ blockInfo.setNumBytes(newLen);
+ map.put(blockInfo, blockInfo);
+ return blockInfo;
+ }
+
+ /**
* Remove the block from the block map;
* remove it from all data-node lists it belongs to;
* and remove all data-node locations associated with the block.
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Oct 17 00:41:25 2009
@@ -1399,6 +1399,9 @@
//
for (BlockInfo block: v.getBlocks()) {
if (!blockManager.checkMinReplication(block)) {
+ LOG.info("BLOCK* NameSystem.checkFileProgress: "
+ + "block " + block + " has not reached minimal replication "
+ + blockManager.minReplication);
return false;
}
}
@@ -1408,6 +1411,9 @@
//
BlockInfo b = v.getPenultimateBlock();
if (b != null && !blockManager.checkMinReplication(b)) {
+ LOG.info("BLOCK* NameSystem.checkFileProgress: "
+ + "block " + b + " has not reached minimal replication "
+ + blockManager.minReplication);
return false;
}
}
@@ -3942,6 +3948,8 @@
synchronized void updatePipeline(String clientName, Block oldBlock,
Block newBlock, DatanodeID[] newNodes)
throws IOException {
+ assert newBlock.getBlockId()==oldBlock.getBlockId() : newBlock + " and "
+ + oldBlock + " has different block identifier";
LOG.info("updatePipeline(block=" + oldBlock
+ ", newGenerationStamp=" + newBlock.getGenerationStamp()
+ ", newLength=" + newBlock.getNumBytes()
@@ -3952,27 +3960,24 @@
// check the vadility of the block and lease holder name
final INodeFileUnderConstruction pendingFile =
checkUCBlock(oldBlock, clientName);
- final BlockInfo oldblockinfo = pendingFile.getLastBlock();
+ final BlockInfoUnderConstruction blockinfo = pendingFile.getLastBlock();
// check new GS & length: this is not expected
- if (newBlock.getGenerationStamp() <= oldblockinfo.getGenerationStamp() ||
- newBlock.getNumBytes() < oldblockinfo.getNumBytes()) {
+ if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
+ newBlock.getNumBytes() < blockinfo.getNumBytes()) {
String msg = "Update " + oldBlock + " (len = " +
- oldblockinfo.getNumBytes() + ") to an older state: " + newBlock +
+ blockinfo.getNumBytes() + ") to an older state: " + newBlock +
" (len = " + newBlock.getNumBytes() +")";
LOG.warn(msg);
throw new IOException(msg);
}
- // Remove old block from blocks map. This always have to be done
+ // Remove old block from the raw map in blocks map.
+ // This does not change any other value of the oldblockinfo
+ // This always have to be done
// because the generation stamp of this block is changing.
- blockManager.removeBlockFromMap(oldblockinfo);
-
- // update last block, construct newblockinfo and add it to the blocks map
- BlockInfoUnderConstruction newblockinfo =
- new BlockInfoUnderConstruction(
- newBlock, pendingFile.getReplication());
- blockManager.addINode(newblockinfo, pendingFile);
+ blockManager.updateBlock(oldBlock,
+ newBlock.getGenerationStamp(), newBlock.getNumBytes());
// find the DatanodeDescriptor objects
DatanodeDescriptor[] descriptors = null;
@@ -3982,8 +3987,7 @@
descriptors[i] = getDatanode(newNodes[i]);
}
}
- // add locations into the INodeUnderConstruction
- pendingFile.setLastBlock(newblockinfo, descriptors);
+ blockinfo.setExpectedLocations(descriptors);
// persist blocks only if append is supported
String src = leaseManager.findPath(pendingFile);
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:824552,824944
+/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/test/hdfs:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/test/hdfs:818294-818298,824552,824944,826149
Modified: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend3.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend3.java?rev=826150&r1=826149&r2=826150&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend3.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend3.java Sat Oct 17 00:41:25 2009
@@ -24,6 +24,7 @@
import junit.framework.Test;
import junit.framework.TestSuite;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -33,9 +34,23 @@
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FSDataset;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
+import org.apache.log4j.Level;
/** This class implements some of tests posted in HADOOP-2658. */
public class TestFileAppend3 extends junit.framework.TestCase {
+ {
+ ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)InterDatanodeProtocol.LOG).getLogger().setLevel(Level.ALL);
+ }
+
static final long BLOCK_SIZE = 64 * 1024;
static final short REPLICATION = 3;
static final int DATANODE_NUM = 5;
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
-/hadoop/hdfs/trunk/src/webapps/datanode:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/webapps/datanode:818294-818298,824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/webapps/hdfs:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/webapps/hdfs:818294-818298,824552,824944,826149
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Oct 17 00:41:25 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
-/hadoop/hdfs/trunk/src/webapps/secondary:818294-818298,824552,824944
+/hadoop/hdfs/trunk/src/webapps/secondary:818294-818298,824552,824944,826149