You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2016/06/23 21:27:05 UTC
hadoop git commit: HDFS-10469. Add number of active xceivers to
datanode metrics. Contributed by Hanisha Koneru.
Repository: hadoop
Updated Branches:
refs/heads/trunk e98c0c7a1 -> dca298d79
HDFS-10469. Add number of active xceivers to datanode metrics. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dca298d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dca298d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dca298d7
Branch: refs/heads/trunk
Commit: dca298d79e46e27bdf008be53dd77448d7a9c0c6
Parents: e98c0c7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Jun 23 14:13:31 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Jun 23 14:13:39 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DataNode.java | 5 +++-
.../hdfs/server/datanode/DataXceiverServer.java | 4 +++
.../datanode/metrics/DataNodeMetrics.java | 16 ++++++++++
.../server/datanode/TestDataNodeMetrics.java | 31 ++++++++++++++++++++
4 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 410472e..a59a59f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1928,7 +1928,10 @@ public class DataNode extends ReconfigurableBase
} catch (InterruptedException ie) {
}
}
-
+ if (metrics != null) {
+ metrics.setDataNodeActiveXceiversCount(0);
+ }
+
// IPC server needs to be shutdown late in the process, otherwise
// shutdown command response won't get sent.
if (ipcServer != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
index 126d5b1..4aab3f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
@@ -232,11 +232,13 @@ class DataXceiverServer implements Runnable {
}
peers.put(peer, t);
peersXceiver.put(peer, xceiver);
+ datanode.metrics.incrDataNodeActiveXceiversCount();
}
synchronized void closePeer(Peer peer) {
peers.remove(peer);
peersXceiver.remove(peer);
+ datanode.metrics.decrDataNodeActiveXceiversCount();
IOUtils.cleanup(null, peer);
}
@@ -282,6 +284,7 @@ class DataXceiverServer implements Runnable {
}
peers.clear();
peersXceiver.clear();
+ datanode.metrics.setDataNodeActiveXceiversCount(0);
}
// Return the number of peers.
@@ -303,6 +306,7 @@ class DataXceiverServer implements Runnable {
synchronized void releasePeer(Peer peer) {
peers.remove(peer);
peersXceiver.remove(peer);
+ datanode.metrics.decrDataNodeActiveXceiversCount();
}
public void updateBalancerMaxConcurrentMovers(int movers) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 3d504d6..dc12787 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import java.util.concurrent.ThreadLocalRandom;
@@ -101,6 +102,9 @@ public class DataNodeMetrics {
@Metric("Count of network errors on the datanode")
MutableCounterLong datanodeNetworkErrors;
+ @Metric("Count of active dataNode xceivers")
+ private MutableGaugeInt dataNodeActiveXceiversCount;
+
@Metric MutableRate readBlockOp;
@Metric MutableRate writeBlockOp;
@Metric MutableRate blockChecksumOp;
@@ -429,4 +433,16 @@ public class DataNodeMetrics {
ecFailedReconstructionTasks.incr();
}
+ public void incrDataNodeActiveXceiversCount() {
+ dataNodeActiveXceiversCount.incr();
+ }
+
+ public void decrDataNodeActiveXceiversCount() {
+ dataNodeActiveXceiversCount.decr();
+ }
+
+ public void setDataNodeActiveXceiversCount(int value) {
+ this.dataNodeActiveXceiversCount.set(value);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 355f7a1..a404259 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.MetricsAsserts;
import org.apache.hadoop.util.Time;
import org.junit.Test;
import org.mockito.Mockito;
@@ -331,4 +332,34 @@ public class TestDataNodeMetrics {
}
}
}
+
+ @Test
+ public void testDatanodeActiveXceiversCount() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ FileSystem fs = cluster.getFileSystem();
+ List<DataNode> datanodes = cluster.getDataNodes();
+ assertEquals(datanodes.size(), 1);
+ DataNode datanode = datanodes.get(0);
+
+ MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
+ long dataNodeActiveXceiversCount = MetricsAsserts.getIntGauge(
+ "DataNodeActiveXceiversCount", rb);
+ assertEquals(dataNodeActiveXceiversCount, 0);
+
+ Path path = new Path("/counter.txt");
+ DFSTestUtil.createFile(fs, path, 204800000, (short) 3, Time
+ .monotonicNow());
+
+ MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
+ dataNodeActiveXceiversCount = MetricsAsserts.getIntGauge(
+ "DataNodeActiveXceiversCount", rbNew);
+ assertTrue(dataNodeActiveXceiversCount >= 0);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org