You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/03/11 22:20:09 UTC
[3/3] hadoop git commit: HDFS-7491. Add incremental blockreport
latency to DN metrics. Contributed by Ming Ma.
HDFS-7491. Add incremental blockreport latency to DN metrics. Contributed by Ming Ma.
(cherry picked from commit fb34f45727e63ea55377fe90241328025307d818)
(cherry picked from commit 5e99b7cd2fdf49803703f00463754aa6c56a52a8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fe5f5b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fe5f5b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fe5f5b5
Branch: refs/heads/branch-2.7
Commit: 0fe5f5b5b8b2785f5c902587c7c5c40ba0e4a8a2
Parents: a5a933a
Author: cnauroth <cn...@apache.org>
Authored: Wed Mar 11 14:11:19 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Wed Mar 11 14:12:06 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 2 ++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 2 ++
.../hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java | 5 +++++
.../apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java | 2 ++
5 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fe5f5b5/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index dbcf0d8..01a8d5b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -302,6 +302,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
| `HeartbeatsAvgTime` | Average heartbeat time in milliseconds |
| `BlockReportsNumOps` | Total number of block report operations |
| `BlockReportsAvgTime` | Average time of block report operations in milliseconds |
+| `IncrementalBlockReportsNumOps` | Total number of incremental block report operations |
+| `IncrementalBlockReportsAvgTime` | Average time of incremental block report operations in milliseconds |
| `CacheReportsNumOps` | Total number of cache report operations |
| `CacheReportsAvgTime` | Average time of cache report operations in milliseconds |
| `PacketAckRoundTripTimeNanosNumOps` | Total number of ack round trip |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fe5f5b5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6fafff7..0bfe59d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -423,6 +423,9 @@ Release 2.7.0 - UNRELEASED
HDFS-6806. HDFS Rolling upgrade document should mention the versions
available. (J.Andreina via aajisaka)
+ HDFS-7491. Add incremental blockreport latency to DN metrics.
+ (Ming Ma via cnauroth)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fe5f5b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index ff1ad78..3ba2f54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -291,12 +291,14 @@ class BPServiceActor implements Runnable {
// Send incremental block reports to the Namenode outside the lock
boolean success = false;
+ final long startTime = Time.monotonicNow();
try {
bpNamenode.blockReceivedAndDeleted(bpRegistration,
bpos.getBlockPoolId(),
reports.toArray(new StorageReceivedDeletedBlocks[reports.size()]));
success = true;
} finally {
+ dn.getMetrics().addIncrementalBlockReport(Time.monotonicNow()-startTime);
if (!success) {
synchronized (pendingIncrementalBRperStorage) {
for (StorageReceivedDeletedBlocks report : reports) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fe5f5b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 0fbc2ee..2e8eb22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -107,6 +107,7 @@ public class DataNodeMetrics {
@Metric MutableRate replaceBlockOp;
@Metric MutableRate heartbeats;
@Metric MutableRate blockReports;
+ @Metric MutableRate incrementalBlockReports;
@Metric MutableRate cacheReports;
@Metric MutableRate packetAckRoundTripTimeNanos;
final MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles;
@@ -201,6 +202,10 @@ public class DataNodeMetrics {
blockReports.add(latency);
}
+ public void addIncrementalBlockReport(long latency) {
+ incrementalBlockReports.add(latency);
+ }
+
public void addCacheReport(long latency) {
cacheReports.add(latency);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fe5f5b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 8a2bacf..5d27fe6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -72,6 +72,8 @@ public class TestDataNodeMetrics {
DataNode datanode = datanodes.get(0);
MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
assertCounter("BytesWritten", LONG_FILE_LEN, rb);
+ assertTrue("Expected non-zero number of incremental block reports",
+ getLongCounter("IncrementalBlockReportsNumOps", rb) > 0);
} finally {
if (cluster != null) {cluster.shutdown();}
}