You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2014/11/05 03:11:55 UTC
git commit: HDFS-7357. FSNamesystem.checkFileProgress should log file
path. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 56ab33316 -> b96fefc03
HDFS-7357. FSNamesystem.checkFileProgress should log file path. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b96fefc0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b96fefc0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b96fefc0
Branch: refs/heads/branch-2
Commit: b96fefc032c31f9147cb79382ec698c367496317
Parents: 56ab333
Author: Haohui Mai <wh...@apache.org>
Authored: Tue Nov 4 18:03:39 2014 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Tue Nov 4 18:11:45 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../BlockInfoUnderConstruction.java | 4 +-
.../namenode/EditLogFileOutputStream.java | 6 +--
.../hdfs/server/namenode/FSEditLogLoader.java | 4 +-
.../hdfs/server/namenode/FSNamesystem.java | 54 +++++++++++---------
5 files changed, 39 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b96fefc0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index aae0e14..c558530 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -78,6 +78,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7356. Use DirectoryListing.hasMore() directly in nfs. (Li Lu via jing9)
+ HDFS-7357. FSNamesystem.checkFileProgress should log file path.
+ (Tsz Wo Nicholas Sze via wheat9)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b96fefc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
index dd3593f..0ea7c2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -141,7 +141,7 @@ public class BlockInfoUnderConstruction extends BlockInfo {
@Override
public void appendStringTo(StringBuilder sb) {
- sb.append("ReplicaUnderConstruction[")
+ sb.append("ReplicaUC[")
.append(expectedLocation)
.append("|")
.append(state)
@@ -370,7 +370,7 @@ public class BlockInfoUnderConstruction extends BlockInfo {
}
private void appendUCParts(StringBuilder sb) {
- sb.append("{blockUCState=").append(blockUCState)
+ sb.append("{UCState=").append(blockUCState)
.append(", primaryNodeIndex=").append(primaryNodeIndex)
.append(", replicas=[");
if (replicas != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b96fefc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
index e9f47b9..830814c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
@@ -161,7 +161,7 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
fp.close();
fp = null;
} finally {
- IOUtils.cleanup(FSNamesystem.LOG, fc, fp);
+ IOUtils.cleanup(LOG, fc, fp);
doubleBuf = null;
fc = null;
fp = null;
@@ -233,8 +233,8 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
size += fillCapacity;
total += fillCapacity;
}
- if(FSNamesystem.LOG.isDebugEnabled()) {
- FSNamesystem.LOG.debug("Preallocated " + total + " bytes at the end of " +
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Preallocated " + total + " bytes at the end of " +
"the edit log (offset " + oldSize + ")");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b96fefc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 7dfe688..a8086ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -330,8 +330,8 @@ public class FSEditLogLoader {
AddCloseOp addCloseOp = (AddCloseOp)op;
final String path =
renameReservedPathsOnUpgrade(addCloseOp.path, logVersion);
- if (FSNamesystem.LOG.isDebugEnabled()) {
- FSNamesystem.LOG.debug(op.opCode + ": " + path +
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(op.opCode + ": " + path +
" numblocks : " + addCloseOp.blocks.length +
" clientHolder " + addCloseOp.clientName +
" clientMachine " + addCloseOp.clientMachine);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b96fefc0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index a3bf793..54ad653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3158,7 +3158,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
String clientMachine = null;
if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("BLOCK* NameSystem.getAdditionalBlock: "
+ NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: "
+ src + " inodeId " + fileId + " for " + clientName);
}
@@ -3382,7 +3382,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
// Check if the penultimate block is minimally replicated
- if (!checkFileProgress(pendingFile, false)) {
+ if (!checkFileProgress(src, pendingFile, false)) {
throw new NotReplicatedYetException("Not replicated yet: " + src);
}
return new FileState(pendingFile, src);
@@ -3630,14 +3630,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
// Check the state of the penultimate block. It should be completed
// before attempting to complete the last one.
- if (!checkFileProgress(pendingFile, false)) {
+ if (!checkFileProgress(src, pendingFile, false)) {
return false;
}
// commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, last);
- if (!checkFileProgress(pendingFile, true)) {
+ if (!checkFileProgress(src, pendingFile, true)) {
return false;
}
@@ -3661,8 +3661,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
throws IOException {
assert hasWriteLock();
BlockInfo b = dir.addBlock(src, inodes, newBlock, targets);
- NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". "
- + getBlockPoolId() + " " + b);
+ NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
return b;
}
@@ -3683,30 +3682,21 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
* replicated. If not, return false. If checkall is true, then check
* all blocks, otherwise check only penultimate block.
*/
- boolean checkFileProgress(INodeFile v, boolean checkall) {
+ private boolean checkFileProgress(String src, INodeFile v, boolean checkall) {
readLock();
try {
if (checkall) {
- //
// check all blocks of the file.
- //
for (BlockInfo block: v.getBlocks()) {
- if (!block.isComplete()) {
- LOG.info("BLOCK* checkFileProgress: " + block
- + " has not reached minimal replication "
- + blockManager.minReplication);
+ if (!isCompleteBlock(src, block, blockManager.minReplication)) {
return false;
}
}
} else {
- //
// check the penultimate block of this file
- //
BlockInfo b = v.getPenultimateBlock();
- if (b != null && !b.isComplete()) {
- LOG.warn("BLOCK* checkFileProgress: " + b
- + " has not reached minimal replication "
- + blockManager.minReplication);
+ if (b != null
+ && !isCompleteBlock(src, b, blockManager.minReplication)) {
return false;
}
}
@@ -3716,6 +3706,19 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
}
+ private static boolean isCompleteBlock(String src, BlockInfo b, int minRepl) {
+ if (!b.isComplete()) {
+ final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)b;
+ final int numNodes = b.numNodes();
+ LOG.info("BLOCK* " + b + " is not COMPLETE (ucState = "
+ + uc.getBlockUCState() + ", replication# = " + numNodes
+ + (numNodes < minRepl? " < ": " >= ")
+ + " minimum = " + minRepl + ") in file " + src);
+ return false;
+ }
+ return true;
+ }
+
////////////////////////////////////////////////////////////////
// Here's how to handle block-copy failure during client write:
// -- As usual, the client's write should result in a streaming
@@ -5160,9 +5163,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if(!nameNodeHasResourcesAvailable()) {
String lowResourcesMsg = "NameNode low on available disk space. ";
if (!isInSafeMode()) {
- FSNamesystem.LOG.warn(lowResourcesMsg + "Entering safe mode.");
+ LOG.warn(lowResourcesMsg + "Entering safe mode.");
} else {
- FSNamesystem.LOG.warn(lowResourcesMsg + "Already in safe mode.");
+ LOG.warn(lowResourcesMsg + "Already in safe mode.");
}
enterSafeMode(true);
}
@@ -7022,11 +7025,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
- LOG.info("updatePipeline(block=" + oldBlock
- + ", newGenerationStamp=" + newBlock.getGenerationStamp()
+ LOG.info("updatePipeline(" + oldBlock.getLocalBlock()
+ + ", newGS=" + newBlock.getGenerationStamp()
+ ", newLength=" + newBlock.getNumBytes()
+ ", newNodes=" + Arrays.asList(newNodes)
- + ", clientName=" + clientName
+ + ", client=" + clientName
+ ")");
waitForLoadingFSImage();
writeLock();
@@ -7044,7 +7047,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
RetryCache.setState(cacheEntry, success);
}
getEditLog().logSync();
- LOG.info("updatePipeline(" + oldBlock + ") successfully to " + newBlock);
+ LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + " => "
+ + newBlock.getLocalBlock() + ") success");
}
/**