You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by co...@apache.org on 2009/12/16 04:14:23 UTC
svn commit: r891107 - in /hadoop/hdfs/branches/branch-0.21: ./
.eclipse.templates/.launches/ src/contrib/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/ha...
Author: cos
Date: Wed Dec 16 03:14:22 2009
New Revision: 891107
URL: http://svn.apache.org/viewvc?rev=891107&view=rev
Log:
HDFS-812. svn merge -c 891106 from trunk to branch-0.21
Modified:
hadoop/hdfs/branches/branch-0.21/ (props changed)
hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/ (props changed)
hadoop/hdfs/branches/branch-0.21/CHANGES.txt (contents, props changed)
hadoop/hdfs/branches/branch-0.21/build.xml (props changed)
hadoop/hdfs/branches/branch-0.21/src/contrib/build.xml (props changed)
hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.21/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
-/hadoop/hdfs/trunk:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/.eclipse.templates/.launches/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/.eclipse.templates/.launches:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/.eclipse.templates/.launches:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=891107&r1=891106&r2=891107&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Wed Dec 16 03:14:22 2009
@@ -492,6 +492,9 @@
HDFS-456. Fix URI generation for windows file paths. (shv)
+ HDFS-812. FSNamesystem#internalReleaseLease throws NullPointerException on
+ a single-block file's lease recovery. (cos)
+
Release 0.20.2 - Unreleased
IMPROVEMENTS
Propchange: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/hdfs/CHANGES.txt:713112
/hadoop/hdfs/branches/HDFS-265/CHANGES.txt:796829-820463
-/hadoop/hdfs/trunk/CHANGES.txt:817853-817863,818294-818298,818801,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/CHANGES.txt:817853-817863,818294-818298,818801,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
-/hadoop/hdfs/trunk/build.xml:817853-817863,818294-818298,818801,824552,824944,825229,826149,828116,828926,829258,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/build.xml:817853-817863,818294-818298,818801,824552,824944,825229,826149,828116,828926,829258,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/build.xml:713112
/hadoop/hdfs/branches/HDFS-265/src/contrib/build.xml:796829-820463
-/hadoop/hdfs/trunk/src/contrib/build.xml:817853-817863,818294-818298,818801,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/contrib/build.xml:817853-817863,818294-818298,818801,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
-/hadoop/hdfs/trunk/src/java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,5 +1,5 @@
/hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:713112
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:776175-785643,785929-786278
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:817353-818319,818321-818553
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/RecoveryInProgressException.java:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/RecoveryInProgressException.java:796829-800617,800619-803337,804756-805652,808672-809439,811495-813103,813105-813630,814223-815964,818294-818298
Propchange: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -3,4 +3,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Modified: hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=891107&r1=891106&r2=891107&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Dec 16 03:14:22 2009
@@ -1757,8 +1757,17 @@
BlockInfoUnderConstruction lastBlock = pendingFile.getLastBlock();
BlockUCState lastBlockState = lastBlock.getBlockUCState();
BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
- BlockUCState penultimateBlockState = (penultimateBlock == null ?
- BlockUCState.COMPLETE : penultimateBlock.getBlockUCState());
+ boolean penultimateBlockMinReplication;
+ BlockUCState penultimateBlockState;
+ if (penultimateBlock == null) {
+ penultimateBlockState = BlockUCState.COMPLETE;
+ // If penultimate block doesn't exist then its minReplication is met
+ penultimateBlockMinReplication = true;
+ } else {
+ penultimateBlockState = BlockUCState.COMMITTED;
+ penultimateBlockMinReplication =
+ blockManager.checkMinReplication(penultimateBlock);
+ }
assert penultimateBlockState == BlockUCState.COMPLETE ||
penultimateBlockState == BlockUCState.COMMITTED :
"Unexpected state of penultimate block in " + src;
@@ -1769,7 +1778,7 @@
break;
case COMMITTED:
// Close file if committed blocks are minimally replicated
- if(blockManager.checkMinReplication(penultimateBlock) &&
+ if(penultimateBlockMinReplication &&
blockManager.checkMinReplication(lastBlock)) {
finalizeINodeFileUnderConstruction(src, pendingFile);
NameNode.stateChangeLog.warn("BLOCK*"
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/protocol/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1 +1 @@
-/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/protocol:817853-817863,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/test/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/test/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Modified: hadoop/hdfs/branches/branch-0.21/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java?rev=891107&r1=891106&r2=891107&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java Wed Dec 16 03:14:22 2009
@@ -157,6 +157,27 @@
assertTrue("FSNamesystem.internalReleaseLease suppose to throw " +
"AlreadyBeingCreatedException here", false);
}
+
+ /**
+ * Mocks FSNamesystem instance, adds an empty file with 0 blocks
+ * and invokes lease recovery method.
+ *
+ */
+ @Test
+ public void testInternalReleaseLease_0blocks () throws IOException {
+ LOG.debug("Running " + GenericTestUtils.getMethodName());
+ LeaseManager.Lease lm = mock(LeaseManager.Lease.class);
+ Path file =
+ spy(new Path("/" + GenericTestUtils.getMethodName() + "_test.dat"));
+ DatanodeDescriptor dnd = mock(DatanodeDescriptor.class);
+ PermissionStatus ps =
+ new PermissionStatus("test", "test", new FsPermission((short)0777));
+
+ mockFileBlocks(0, null, null, file, dnd, ps, false);
+
+ assertTrue("True has to be returned in this case",
+ fsn.internalReleaseLease(lm, file.toString(), null));
+ }
/**
* Mocks FSNamesystem instance, adds an empty file with 1 block
@@ -346,16 +367,6 @@
when(b.getBlockUCState()).thenReturn(penUltState);
when(b1.getBlockUCState()).thenReturn(lastState);
BlockInfo[] blocks;
- switch (fileBlocksNumber) {
- case 0:
- blocks = new BlockInfo[0];
- break;
- case 1:
- blocks = new BlockInfo[]{b1};
- break;
- default:
- blocks = new BlockInfo[]{b, b1};
- }
FSDirectory fsDir = mock(FSDirectory.class);
INodeFileUnderConstruction iNFmock = mock(INodeFileUnderConstruction.class);
@@ -368,14 +379,29 @@
when(fsn.getFSImage().getEditLog()).thenReturn(editLog);
fsn.getFSImage().setFSNamesystem(fsn);
+ switch (fileBlocksNumber) {
+ case 0:
+ blocks = new BlockInfo[0];
+ break;
+ case 1:
+ blocks = new BlockInfo[]{b1};
+ when(iNFmock.getLastBlock()).thenReturn(b1);
+ break;
+ default:
+ when(iNFmock.getPenultimateBlock()).thenReturn(b);
+ when(iNFmock.getLastBlock()).thenReturn(b1);
+ blocks = new BlockInfo[]{b, b1};
+ }
+
when(iNFmock.getBlocks()).thenReturn(blocks);
- when(iNFmock.numBlocks()).thenReturn(2);
- when(iNFmock.getPenultimateBlock()).thenReturn(b);
- when(iNFmock.getLastBlock()).thenReturn(b1);
+ when(iNFmock.numBlocks()).thenReturn(blocks.length);
when(iNFmock.isUnderConstruction()).thenReturn(true);
+ when(iNFmock.convertToInodeFile()).thenReturn(iNFmock);
fsDir.addFile(file.toString(), ps, (short)3, 1l, "test",
"test-machine", dnd, 1001l);
+ fsn.leaseManager = mock(LeaseManager.class);
+ fsn.leaseManager.addLease("mock-lease", file.toString());
if (setStoredBlock) {
when(b1.getINode()).thenReturn(iNFmock);
fsn.blockManager.blocksMap.addINode(b1, iNFmock);
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
-/hadoop/hdfs/trunk/src/webapps/datanode:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/webapps/datanode:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
-/hadoop/hdfs/trunk/src/webapps/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/webapps/hdfs:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106
Propchange: hadoop/hdfs/branches/branch-0.21/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 16 03:14:22 2009
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
-/hadoop/hdfs/trunk/src/webapps/secondary:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655
+/hadoop/hdfs/trunk/src/webapps/secondary:817853-817863,818294-818298,824552,824944,826149,828116,828926,829880,829894,830003,831436,831455-831490,832043,833499,835728,880971,881014,881017,884432,888084,888507,888519,889002,890655,891106