You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2017/09/14 00:14:28 UTC
hadoop git commit: HDFS-12409. Add metrics of execution time of
different stages in EC recovery task. (Lei (Eddy) Xu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c3f35c422 -> 73aed34df
HDFS-12409. Add metrics of execution time of different stages in EC recovery task. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73aed34d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73aed34d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73aed34d
Branch: refs/heads/trunk
Commit: 73aed34dffa5e79f6f819137b69054c1dee2d4dd
Parents: c3f35c4
Author: Lei Xu <le...@apache.org>
Authored: Wed Sep 13 17:10:16 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Wed Sep 13 17:14:13 2017 -0700
----------------------------------------------------------------------
.../erasurecode/StripedBlockReconstructor.java | 11 +++++++++++
.../server/datanode/metrics/DataNodeMetrics.java | 18 ++++++++++++++++++
.../TestDataNodeErasureCodingMetrics.java | 7 +++++++
3 files changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73aed34d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index bac013a..34e58ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
+import org.apache.hadoop.util.Time;
/**
* StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -83,18 +84,28 @@ class StripedBlockReconstructor extends StripedReconstructor
final int toReconstructLen =
(int) Math.min(getStripedReader().getBufferSize(), remaining);
+ long start = Time.monotonicNow();
// step1: read from minimum source DNs required for reconstruction.
// The returned success list is the source DNs we do real read from
getStripedReader().readMinimumSources(toReconstructLen);
+ long readEnd = Time.monotonicNow();
// step2: decode to reconstruct targets
reconstructTargets(toReconstructLen);
+ long decodeEnd = Time.monotonicNow();
// step3: transfer data
if (stripedWriter.transferData2Targets() == 0) {
String error = "Transfer failed for all targets.";
throw new IOException(error);
}
+ long writeEnd = Time.monotonicNow();
+
+ // Only the succeed reconstructions are recorded.
+ final DataNodeMetrics metrics = getDatanode().getMetrics();
+ metrics.incrECReconstructionReadTime(readEnd - start);
+ metrics.incrECReconstructionDecodingTime(decodeEnd - readEnd);
+ metrics.incrECReconstructionWriteTime(writeEnd - decodeEnd);
updatePositionInBlock(toReconstructLen);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73aed34d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index a8a6919..58a2f65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -151,6 +151,12 @@ public class DataNodeMetrics {
MutableCounterLong ecReconstructionBytesWritten;
@Metric("Bytes remote read by erasure coding worker")
MutableCounterLong ecReconstructionRemoteBytesRead;
+ @Metric("Milliseconds spent on read by erasure coding worker")
+ private MutableCounterLong ecReconstructionReadTimeMillis;
+ @Metric("Milliseconds spent on decoding by erasure coding worker")
+ private MutableCounterLong ecReconstructionDecodingTimeMillis;
+ @Metric("Milliseconds spent on write by erasure coding worker")
+ private MutableCounterLong ecReconstructionWriteTimeMillis;
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
@@ -503,4 +509,16 @@ public class DataNodeMetrics {
public void incrECReconstructionBytesWritten(long bytes) {
ecReconstructionBytesWritten.incr(bytes);
}
+
+ public void incrECReconstructionReadTime(long millis) {
+ ecReconstructionReadTimeMillis.incr(millis);
+ }
+
+ public void incrECReconstructionWriteTime(long millis) {
+ ecReconstructionWriteTimeMillis.incr(millis);
+ }
+
+ public void incrECReconstructionDecodingTime(long millis) {
+ ecReconstructionDecodingTimeMillis.incr(millis);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73aed34d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
index ee2afbb..8ababfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -90,6 +90,10 @@ public class TestDataNodeErasureCodingMetrics {
@Test(timeout = 120000)
public void testFullBlock() throws Exception {
+ Assert.assertEquals(0, getLongMetric("EcReconstructionReadTimeMillis"));
+ Assert.assertEquals(0, getLongMetric("EcReconstructionDecodingTimeMillis"));
+ Assert.assertEquals(0, getLongMetric("EcReconstructionWriteTimeMillis"));
+
doTest("/testEcMetrics", blockGroupSize, 0);
Assert.assertEquals("EcReconstructionTasks should be ",
@@ -103,6 +107,9 @@ public class TestDataNodeErasureCodingMetrics {
blockSize, getLongMetric("EcReconstructionBytesWritten"));
Assert.assertEquals("EcReconstructionRemoteBytesRead should be ",
0, getLongMetricWithoutCheck("EcReconstructionRemoteBytesRead"));
+ Assert.assertTrue(getLongMetric("EcReconstructionReadTimeMillis") > 0);
+ Assert.assertTrue(getLongMetric("EcReconstructionDecodingTimeMillis") > 0);
+ Assert.assertTrue(getLongMetric("EcReconstructionWriteTimeMillis") > 0);
}
// A partial block, reconstruct the partial block
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org