You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2016/03/14 18:47:03 UTC
[3/3] hadoop git commit: HDFS-9941. Do not log StandbyException on NN,
other minor logging fixes. Contributed by Arpit Agarwal.
HDFS-9941. Do not log StandbyException on NN, other minor logging fixes. Contributed by Arpit Agarwal.
(cherry picked from commit 5644137adad30c84e40d2c4719627b3aabc73628)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
(cherry picked from commit dd462bc32f77799827c1bd438ee8fa54dacb20eb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6c6f4dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6c6f4dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6c6f4dd
Branch: refs/heads/branch-2.8
Commit: e6c6f4ddac4f0bd4fe9b62887d12aef85ebd46a0
Parents: 4f5b496
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon Mar 14 09:54:54 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon Mar 14 09:55:34 2016 -0700
----------------------------------------------------------------------
.../BlockUnderConstructionFeature.java | 43 ++++++++++++--------
.../blockmanagement/DecommissionManager.java | 8 +++-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 18 +++++++-
.../hdfs/server/namenode/NameNodeRpcServer.java | 4 ++
4 files changed, 54 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6c6f4dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
index 88cf06d..ddcdd0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockUnderConstructionFeature.java
@@ -33,11 +33,13 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCSt
*/
public class BlockUnderConstructionFeature {
private BlockUCState blockUCState;
+ private static final ReplicaUnderConstruction[] NO_REPLICAS =
+ new ReplicaUnderConstruction[0];
/**
* Block replicas as assigned when the block was allocated.
*/
- private ReplicaUnderConstruction[] replicas;
+ private ReplicaUnderConstruction[] replicas = NO_REPLICAS;
/**
* Index of the primary data node doing the recovery. Useful for log
@@ -90,7 +92,7 @@ public class BlockUnderConstructionFeature {
/** Get the number of expected locations */
public int getNumExpectedLocations() {
- return replicas == null ? 0 : replicas.length;
+ return replicas.length;
}
/**
@@ -127,12 +129,10 @@ public class BlockUnderConstructionFeature {
List<ReplicaUnderConstruction> getStaleReplicas(long genStamp) {
List<ReplicaUnderConstruction> staleReplicas = new ArrayList<>();
- if (replicas != null) {
- // Remove replicas with wrong gen stamp. The replica list is unchanged.
- for (ReplicaUnderConstruction r : replicas) {
- if (genStamp != r.getGenerationStamp()) {
- staleReplicas.add(r);
- }
+ // Remove replicas with wrong gen stamp. The replica list is unchanged.
+ for (ReplicaUnderConstruction r : replicas) {
+ if (genStamp != r.getGenerationStamp()) {
+ staleReplicas.add(r);
}
}
return staleReplicas;
@@ -146,7 +146,7 @@ public class BlockUnderConstructionFeature {
public void initializeBlockRecovery(BlockInfo blockInfo, long recoveryId) {
setBlockUCState(BlockUCState.UNDER_RECOVERY);
blockRecoveryId = recoveryId;
- if (replicas == null || replicas.length == 0) {
+ if (replicas.length == 0) {
NameNode.blockStateChangeLog.warn("BLOCK*" +
" BlockUnderConstructionFeature.initializeBlockRecovery:" +
" No blocks found, lease removed.");
@@ -197,7 +197,7 @@ public class BlockUnderConstructionFeature {
/** Add the reported replica if it is not already in the replica list. */
void addReplicaIfNotPresent(DatanodeStorageInfo storage,
Block reportedBlock, ReplicaState rState) {
- if (replicas == null) {
+ if (replicas.length == 0) {
replicas = new ReplicaUnderConstruction[1];
replicas[0] = new ReplicaUnderConstruction(reportedBlock, storage,
rState);
@@ -240,15 +240,24 @@ public class BlockUnderConstructionFeature {
.append(", truncateBlock=").append(truncateBlock)
.append(", primaryNodeIndex=").append(primaryNodeIndex)
.append(", replicas=[");
- if (replicas != null) {
- int i = 0;
- for (ReplicaUnderConstruction r : replicas) {
- r.appendStringTo(sb);
- if (++i < replicas.length) {
- sb.append(", ");
- }
+ int i = 0;
+ for (ReplicaUnderConstruction r : replicas) {
+ r.appendStringTo(sb);
+ if (++i < replicas.length) {
+ sb.append(", ");
}
}
sb.append("]}");
}
+
+ public void appendUCPartsConcise(StringBuilder sb) {
+ sb.append("replicas=");
+ int i = 0;
+ for (ReplicaUnderConstruction r : replicas) {
+ sb.append(r.getExpectedStorageLocation().getDatanodeDescriptor());
+ if (++i < replicas.length) {
+ sb.append(", ");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6c6f4dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index ec9a710..073332b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.util.CyclicIteration;
import org.apache.hadoop.util.ChunkedArrayList;
@@ -289,6 +290,10 @@ public class DecommissionManager {
BlockCollection bc,
DatanodeDescriptor srcNode, NumberReplicas num,
Iterable<DatanodeStorageInfo> storages) {
+ if (!NameNode.blockStateChangeLog.isInfoEnabled()) {
+ return;
+ }
+
int curReplicas = num.liveReplicas();
int curExpectedReplicas = block.getReplication();
StringBuilder nodeList = new StringBuilder();
@@ -297,7 +302,8 @@ public class DecommissionManager {
nodeList.append(node);
nodeList.append(" ");
}
- LOG.info("Block: " + block + ", Expected Replicas: "
+ NameNode.blockStateChangeLog.info(
+ "Block: " + block + ", Expected Replicas: "
+ curExpectedReplicas + ", live replicas: " + curReplicas
+ ", corrupt replicas: " + num.corruptReplicas()
+ ", decommissioned replicas: " + num.decommissioned()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6c6f4dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 17e2459..b49f6bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -811,10 +811,26 @@ class FSDirWriteFileOp {
assert fsn.hasWriteLock();
BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock,
targets);
- NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
+ logAllocatedBlock(src, b);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
}
+ private static void logAllocatedBlock(String src, BlockInfo b) {
+ if (!NameNode.stateChangeLog.isInfoEnabled()) {
+ return;
+ }
+ StringBuilder sb = new StringBuilder(150);
+ sb.append("BLOCK* allocate ");
+ b.appendStringTo(sb);
+ sb.append(", ");
+ BlockUnderConstructionFeature uc = b.getUnderConstructionFeature();
+ if (uc != null) {
+ uc.appendUCPartsConcise(sb);
+ }
+ sb.append(" for " + src);
+ NameNode.stateChangeLog.info(sb.toString());
+ }
+
private static void setNewINodeStoragePolicy(BlockManager bm, INodeFile
inode, INodesInPath iip, boolean isLazyPersist)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6c6f4dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 26513a8..5f82492 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -158,6 +158,7 @@ import org.apache.hadoop.ipc.RetryCache;
import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
@@ -476,6 +477,9 @@ class NameNodeRpcServer implements NamenodeProtocols {
FSLimitException.PathComponentTooLongException.class,
FSLimitException.MaxDirectoryItemsExceededException.class,
UnresolvedPathException.class);
+
+ clientRpcServer.addSuppressedLoggingExceptions(StandbyException.class);
+
clientRpcServer.setTracer(nn.tracer);
if (serviceRpcServer != null) {
serviceRpcServer.setTracer(nn.tracer);