You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/03/14 21:00:28 UTC
[19/20] hadoop git commit: HDFS-9941. Do not log StandbyException on
NN, other minor logging fixes. Contributed by Arpit Agarwal.
HDFS-9941. Do not log StandbyException on NN, other minor logging fixes. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5644137a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5644137a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5644137a
Branch: refs/heads/HDFS-1312
Commit: 5644137adad30c84e40d2c4719627b3aabc73628
Parents: f291d82
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon Mar 14 09:54:54 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon Mar 14 09:54:54 2016 -0700
----------------------------------------------------------------------
.../BlockUnderConstructionFeature.java | 45 ++++++++++++--------
.../blockmanagement/DecommissionManager.java | 8 +++-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 18 +++++++-
.../hdfs/server/namenode/NameNodeRpcServer.java | 4 ++
4 files changed, 55 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
index b46b470..1a93033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
@@ -33,11 +33,13 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCSt
*/
public class BlockUnderConstructionFeature {
private BlockUCState blockUCState;
+ private static final ReplicaUnderConstruction[] NO_REPLICAS =
+ new ReplicaUnderConstruction[0];
/**
* Block replicas as assigned when the block was allocated.
*/
- private ReplicaUnderConstruction[] replicas;
+ private ReplicaUnderConstruction[] replicas = NO_REPLICAS;
/**
* Index of the primary data node doing the recovery. Useful for log
@@ -120,7 +122,7 @@ public class BlockUnderConstructionFeature {
}
public int getNumExpectedLocations() {
- return replicas == null ? 0 : replicas.length;
+ return replicas.length;
}
/**
@@ -130,7 +132,7 @@ public class BlockUnderConstructionFeature {
*/
void updateStorageScheduledSize(BlockInfoStriped storedBlock) {
assert storedBlock.getUnderConstructionFeature() == this;
- if (replicas == null) {
+ if (replicas.length == 0) {
return;
}
final int dataBlockNum = storedBlock.getDataBlockNum();
@@ -182,12 +184,10 @@ public class BlockUnderConstructionFeature {
List<ReplicaUnderConstruction> getStaleReplicas(long genStamp) {
List<ReplicaUnderConstruction> staleReplicas = new ArrayList<>();
- if (replicas != null) {
- // Remove replicas with wrong gen stamp. The replica list is unchanged.
- for (ReplicaUnderConstruction r : replicas) {
- if (genStamp != r.getGenerationStamp()) {
- staleReplicas.add(r);
- }
+ // Remove replicas with wrong gen stamp. The replica list is unchanged.
+ for (ReplicaUnderConstruction r : replicas) {
+ if (genStamp != r.getGenerationStamp()) {
+ staleReplicas.add(r);
}
}
return staleReplicas;
@@ -201,7 +201,7 @@ public class BlockUnderConstructionFeature {
public void initializeBlockRecovery(BlockInfo blockInfo, long recoveryId) {
setBlockUCState(BlockUCState.UNDER_RECOVERY);
blockRecoveryId = recoveryId;
- if (replicas == null || replicas.length == 0) {
+ if (replicas.length == 0) {
NameNode.blockStateChangeLog.warn("BLOCK*" +
" BlockUnderConstructionFeature.initializeBlockRecovery:" +
" No blocks found, lease removed.");
@@ -252,7 +252,7 @@ public class BlockUnderConstructionFeature {
/** Add the reported replica if it is not already in the replica list. */
void addReplicaIfNotPresent(DatanodeStorageInfo storage,
Block reportedBlock, ReplicaState rState) {
- if (replicas == null) {
+ if (replicas.length == 0) {
replicas = new ReplicaUnderConstruction[1];
replicas[0] = new ReplicaUnderConstruction(reportedBlock, storage,
rState);
@@ -295,15 +295,24 @@ public class BlockUnderConstructionFeature {
.append(", truncateBlock=").append(truncateBlock)
.append(", primaryNodeIndex=").append(primaryNodeIndex)
.append(", replicas=[");
- if (replicas != null) {
- int i = 0;
- for (ReplicaUnderConstruction r : replicas) {
- r.appendStringTo(sb);
- if (++i < replicas.length) {
- sb.append(", ");
- }
+ int i = 0;
+ for (ReplicaUnderConstruction r : replicas) {
+ r.appendStringTo(sb);
+ if (++i < replicas.length) {
+ sb.append(", ");
}
}
sb.append("]}");
}
+
+ public void appendUCPartsConcise(StringBuilder sb) {
+ sb.append("replicas=");
+ int i = 0;
+ for (ReplicaUnderConstruction r : replicas) {
+ sb.append(r.getExpectedStorageLocation().getDatanodeDescriptor());
+ if (++i < replicas.length) {
+ sb.append(", ");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 2a5d63c..480670a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.util.CyclicIteration;
import org.apache.hadoop.util.ChunkedArrayList;
@@ -280,6 +281,10 @@ public class DecommissionManager {
BlockCollection bc,
DatanodeDescriptor srcNode, NumberReplicas num,
Iterable<DatanodeStorageInfo> storages) {
+ if (!NameNode.blockStateChangeLog.isInfoEnabled()) {
+ return;
+ }
+
int curReplicas = num.liveReplicas();
int curExpectedReplicas = blockManager.getExpectedReplicaNum(block);
StringBuilder nodeList = new StringBuilder();
@@ -288,7 +293,8 @@ public class DecommissionManager {
nodeList.append(node);
nodeList.append(" ");
}
- LOG.info("Block: " + block + ", Expected Replicas: "
+ NameNode.blockStateChangeLog.info(
+ "Block: " + block + ", Expected Replicas: "
+ curExpectedReplicas + ", live replicas: " + curReplicas
+ ", corrupt replicas: " + num.corruptReplicas()
+ ", decommissioned replicas: " + num.decommissioned()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index cc08528..41fd869 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -848,10 +848,26 @@ class FSDirWriteFileOp {
assert fsn.hasWriteLock();
BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock, targets,
isStriped);
- NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
+ logAllocatedBlock(src, b);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
}
+ private static void logAllocatedBlock(String src, BlockInfo b) {
+ if (!NameNode.stateChangeLog.isInfoEnabled()) {
+ return;
+ }
+ StringBuilder sb = new StringBuilder(150);
+ sb.append("BLOCK* allocate ");
+ b.appendStringTo(sb);
+ sb.append(", ");
+ BlockUnderConstructionFeature uc = b.getUnderConstructionFeature();
+ if (uc != null) {
+ uc.appendUCPartsConcise(sb);
+ }
+ sb.append(" for " + src);
+ NameNode.stateChangeLog.info(sb.toString());
+ }
+
private static void setNewINodeStoragePolicy(BlockManager bm, INodeFile
inode, INodesInPath iip, boolean isLazyPersist)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5644137a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6dff1bc..eb47580 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -163,6 +163,7 @@ import org.apache.hadoop.ipc.RetryCache;
import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
@@ -494,6 +495,9 @@ class NameNodeRpcServer implements NamenodeProtocols {
FSLimitException.PathComponentTooLongException.class,
FSLimitException.MaxDirectoryItemsExceededException.class,
UnresolvedPathException.class);
+
+ clientRpcServer.addSuppressedLoggingExceptions(StandbyException.class);
+
clientRpcServer.setTracer(nn.tracer);
if (serviceRpcServer != null) {
serviceRpcServer.setTracer(nn.tracer);