You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/06/29 10:16:36 UTC
[hadoop] branch branch-3.3 updated: HDFS-16089. Add metric
EcReconstructionValidateTimeMillis for StripedBlockReconstructor (#3146)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 4ee7bd3 HDFS-16089. Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor (#3146)
4ee7bd3 is described below
commit 4ee7bd345fc04c28092150f1637c2f7f4d04bced
Author: litao <to...@gmail.com>
AuthorDate: Tue Jun 29 18:15:12 2021 +0800
HDFS-16089. Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor (#3146)
(cherry picked from commit 95454d821c6e4c08d49df195d7c1e0ea1aacafb9)
---
.../server/datanode/erasurecode/StripedBlockReconstructor.java | 7 +++++++
.../hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java | 6 ++++++
.../hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java | 2 +-
3 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index cd59f51..3ead793 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -135,9 +135,16 @@ class StripedBlockReconstructor extends StripedReconstructor
resetBuffers(inputs);
DataNodeFaultInjector.get().badDecoding(outputs);
+ long start = Time.monotonicNow();
try {
getValidator().validate(inputs, erasedIndices, outputs);
+ long validateEnd = Time.monotonicNow();
+ getDatanode().getMetrics().incrECReconstructionValidateTime(
+ validateEnd - start);
} catch (InvalidDecodingException e) {
+ long validateFailedEnd = Time.monotonicNow();
+ getDatanode().getMetrics().incrECReconstructionValidateTime(
+ validateFailedEnd - start);
getDatanode().getMetrics().incrECInvalidReconstructionTasks();
throw e;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index bd21a5a..6bccfc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -168,6 +168,8 @@ public class DataNodeMetrics {
private MutableCounterLong ecReconstructionDecodingTimeMillis;
@Metric("Milliseconds spent on write by erasure coding worker")
private MutableCounterLong ecReconstructionWriteTimeMillis;
+ @Metric("Milliseconds spent on validating by erasure coding worker")
+ private MutableCounterLong ecReconstructionValidateTimeMillis;
@Metric("Sum of all BPServiceActors command queue length")
private MutableCounterLong sumOfActorCommandQueueLength;
@Metric("Num of processed commands of all BPServiceActors")
@@ -605,6 +607,10 @@ public class DataNodeMetrics {
ecReconstructionDecodingTimeMillis.incr(millis);
}
+ public void incrECReconstructionValidateTime(long millis) {
+ ecReconstructionValidateTimeMillis.incr(millis);
+ }
+
public DataNodeUsageReport getDNUsageReport(long timeSinceLastReport) {
return dnUsageReportUtil.getUsageReport(bytesWritten.value(), bytesRead
.value(), totalWriteTime.value(), totalReadTime.value(),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index 94a9ae1..fc307bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -364,7 +364,7 @@ public class TestReconstructStripedBlocks {
}
}
- @Test(timeout=120000) // 1 min timeout
+ @Test(timeout=120000) // 2 min timeout
public void testReconstructionWork() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org