You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2013/10/16 20:38:31 UTC
svn commit: r1532857 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/
Author: szetszwo
Date: Wed Oct 16 18:38:31 2013
New Revision: 1532857
URL: http://svn.apache.org/r1532857
Log:
HDFS-5283. Under construction blocks only inside snapshots should not be counted in safemode threshhold. Contributed by Vinay
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1532857&r1=1532856&r2=1532857&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 16 18:38:31 2013
@@ -349,6 +349,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via
jing9)
+ HDFS-5283. Under construction blocks only inside snapshots should not be
+ counted in safemode threshhold. (Vinay via szetszwo)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1532857&r1=1532856&r2=1532857&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Oct 16 18:38:31 2013
@@ -1780,6 +1780,14 @@ public class BlockManager {
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent(
node, iblk, reportedState);
+ // OpenFileBlocks only inside snapshots also will be added to safemode
+ // threshold. So we need to update such blocks to safemode
+ // refer HDFS-5283
+ BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock;
+ if (namesystem.isInSnapshot(blockUC)) {
+ int numOfReplicas = blockUC.getNumExpectedLocations();
+ namesystem.incrementSafeBlockCount(numOfReplicas);
+ }
//and fall through to next clause
}
//add replica if appropriate
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1532857&r1=1532856&r2=1532857&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Oct 16 18:38:31 2013
@@ -165,6 +165,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -3717,6 +3718,39 @@ public class FSNamesystem implements Nam
BlockInfo getStoredBlock(Block block) {
return blockManager.getStoredBlock(block);
}
+
+ @Override
+ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
+ assert hasReadOrWriteLock();
+ final BlockCollection bc = blockUC.getBlockCollection();
+ if (bc == null || !(bc instanceof INodeFileUnderConstruction)) {
+ return false;
+ }
+
+ INodeFileUnderConstruction inodeUC = (INodeFileUnderConstruction) blockUC
+ .getBlockCollection();
+ String fullName = inodeUC.getName();
+ try {
+ if (fullName != null && fullName.startsWith(Path.SEPARATOR)
+ && dir.getINode(fullName) == inodeUC) {
+ // If file exists in normal path then no need to look in snapshot
+ return false;
+ }
+ } catch (UnresolvedLinkException e) {
+ LOG.error("Error while resolving the link : " + fullName, e);
+ return false;
+ }
+ /*
+ * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and
+ * bc is not in the current fsdirectory tree, bc must represent a snapshot
+ * file.
+ * 2. if fullName is not an absolute path, bc cannot be existent in the
+ * current fsdirectory tree.
+ * 3. if bc is not the current node associated with fullName, bc must be a
+ * snapshot inode.
+ */
+ return true;
+ }
void commitBlockSynchronization(ExtendedBlock lastblock,
long newgenerationstamp, long newlength,
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java?rev=1532857&r1=1532856&r2=1532857&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java Wed Oct 16 18:38:31 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.util.RwLock;
import org.apache.hadoop.ipc.StandbyException;
@@ -43,4 +44,6 @@ public interface Namesystem extends RwLo
public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal);
public void checkOperation(OperationCategory read) throws StandbyException;
+
+ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC);
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1532857&r1=1532856&r2=1532857&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Wed Oct 16 18:38:31 2013
@@ -994,4 +994,8 @@ public class DFSTestUtil {
cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length);
} while (locatedBlocks.isUnderConstruction());
}
+
+ public static void abortStream(DFSOutputStream out) throws IOException {
+ out.abort();
+ }
}