You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sh...@apache.org on 2013/08/30 01:08:47 UTC
svn commit: r1518852 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: shv
Date: Thu Aug 29 23:08:47 2013
New Revision: 1518852
URL: http://svn.apache.org/r1518852
Log:
HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization(). Contributed by Plamen Jeliazkov.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1518852&r1=1518851&r2=1518852&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 29 23:08:47 2013
@@ -175,6 +175,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and
DatanodeManager#handleHeartbeat. (kihwal)
+ HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization().
+ (Plamen Jeliazkov via shv)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1518852&r1=1518851&r2=1518852&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Thu Aug 29 23:08:47 2013
@@ -174,7 +174,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
@@ -3755,24 +3754,32 @@ public class FSNamesystem implements Nam
// find the DatanodeDescriptor objects
// There should be no locations in the blockManager till now because the
// file is underConstruction
- DatanodeDescriptor[] descriptors = null;
+ List<DatanodeDescriptor> targetList =
+ new ArrayList<DatanodeDescriptor>(newtargets.length);
if (newtargets.length > 0) {
- descriptors = new DatanodeDescriptor[newtargets.length];
- for(int i = 0; i < newtargets.length; i++) {
- descriptors[i] = blockManager.getDatanodeManager().getDatanode(
- newtargets[i]);
+ for (DatanodeID newtarget : newtargets) {
+ // try to get targetNode
+ DatanodeDescriptor targetNode =
+ blockManager.getDatanodeManager().getDatanode(newtarget);
+ if (targetNode != null)
+ targetList.add(targetNode);
+ else if (LOG.isDebugEnabled()) {
+ LOG.debug("DatanodeDescriptor (=" + newtarget + ") not found");
+ }
}
}
- if ((closeFile) && (descriptors != null)) {
+ if ((closeFile) && !targetList.isEmpty()) {
// the file is getting closed. Insert block locations into blockManager.
// Otherwise fsck will report these blocks as MISSING, especially if the
// blocksReceived from Datanodes take a long time to arrive.
- for (int i = 0; i < descriptors.length; i++) {
- descriptors[i].addBlock(storedBlock);
+ for (DatanodeDescriptor targetNode : targetList) {
+ targetNode.addBlock(storedBlock);
}
}
// add pipeline locations into the INodeUnderConstruction
- pendingFile.setLastBlock(storedBlock, descriptors);
+ DatanodeDescriptor[] targetArray =
+ new DatanodeDescriptor[targetList.size()];
+ pendingFile.setLastBlock(storedBlock, targetList.toArray(targetArray));
}
if (closeFile) {
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1518852&r1=1518851&r2=1518852&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Thu Aug 29 23:08:47 2013
@@ -169,4 +169,23 @@ public class TestCommitBlockSynchronizat
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
}
+
+ @Test
+ public void testCommitBlockSynchronizationWithCloseAndNonExistantTarget()
+ throws IOException {
+ INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ Block block = new Block(blockId, length, genStamp);
+ FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
+ DatanodeID[] newTargets = new DatanodeID[]{
+ new DatanodeID("0.0.0.0", "nonexistantHost", "1", 0, 0, 0)};
+
+ ExtendedBlock lastBlock = new ExtendedBlock();
+ namesystemSpy.commitBlockSynchronization(
+ lastBlock, genStamp, length, true,
+ false, newTargets, null);
+
+ // Repeat the call to make sure it returns true
+ namesystemSpy.commitBlockSynchronization(
+ lastBlock, genStamp, length, true, false, newTargets, null);
+ }
}