You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/23 21:53:20 UTC
[1/3] hadoop git commit: HDFS-10469. Add number of active xceivers to
datanode metrics. Contributed by Hanisha Koneru.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-1312 ea8bfc370 -> 509072312
HDFS-10469. Add number of active xceivers to datanode metrics. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dca298d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dca298d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dca298d7
Branch: refs/heads/HDFS-1312
Commit: dca298d79e46e27bdf008be53dd77448d7a9c0c6
Parents: e98c0c7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Jun 23 14:13:31 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Jun 23 14:13:39 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DataNode.java | 5 +++-
.../hdfs/server/datanode/DataXceiverServer.java | 4 +++
.../datanode/metrics/DataNodeMetrics.java | 16 ++++++++++
.../server/datanode/TestDataNodeMetrics.java | 31 ++++++++++++++++++++
4 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 410472e..a59a59f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1928,7 +1928,10 @@ public class DataNode extends ReconfigurableBase
} catch (InterruptedException ie) {
}
}
-
+ if (metrics != null) {
+ metrics.setDataNodeActiveXceiversCount(0);
+ }
+
// IPC server needs to be shutdown late in the process, otherwise
// shutdown command response won't get sent.
if (ipcServer != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
index 126d5b1..4aab3f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
@@ -232,11 +232,13 @@ class DataXceiverServer implements Runnable {
}
peers.put(peer, t);
peersXceiver.put(peer, xceiver);
+ datanode.metrics.incrDataNodeActiveXceiversCount();
}
synchronized void closePeer(Peer peer) {
peers.remove(peer);
peersXceiver.remove(peer);
+ datanode.metrics.decrDataNodeActiveXceiversCount();
IOUtils.cleanup(null, peer);
}
@@ -282,6 +284,7 @@ class DataXceiverServer implements Runnable {
}
peers.clear();
peersXceiver.clear();
+ datanode.metrics.setDataNodeActiveXceiversCount(0);
}
// Return the number of peers.
@@ -303,6 +306,7 @@ class DataXceiverServer implements Runnable {
synchronized void releasePeer(Peer peer) {
peers.remove(peer);
peersXceiver.remove(peer);
+ datanode.metrics.decrDataNodeActiveXceiversCount();
}
public void updateBalancerMaxConcurrentMovers(int movers) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 3d504d6..dc12787 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import java.util.concurrent.ThreadLocalRandom;
@@ -101,6 +102,9 @@ public class DataNodeMetrics {
@Metric("Count of network errors on the datanode")
MutableCounterLong datanodeNetworkErrors;
+ @Metric("Count of active dataNode xceivers")
+ private MutableGaugeInt dataNodeActiveXceiversCount;
+
@Metric MutableRate readBlockOp;
@Metric MutableRate writeBlockOp;
@Metric MutableRate blockChecksumOp;
@@ -429,4 +433,16 @@ public class DataNodeMetrics {
ecFailedReconstructionTasks.incr();
}
+ public void incrDataNodeActiveXceiversCount() {
+ dataNodeActiveXceiversCount.incr();
+ }
+
+ public void decrDataNodeActiveXceiversCount() {
+ dataNodeActiveXceiversCount.decr();
+ }
+
+ public void setDataNodeActiveXceiversCount(int value) {
+ this.dataNodeActiveXceiversCount.set(value);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dca298d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index 355f7a1..a404259 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.MetricsAsserts;
import org.apache.hadoop.util.Time;
import org.junit.Test;
import org.mockito.Mockito;
@@ -331,4 +332,34 @@ public class TestDataNodeMetrics {
}
}
}
+
+ @Test
+ public void testDatanodeActiveXceiversCount() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ FileSystem fs = cluster.getFileSystem();
+ List<DataNode> datanodes = cluster.getDataNodes();
+ assertEquals(datanodes.size(), 1);
+ DataNode datanode = datanodes.get(0);
+
+ MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
+ long dataNodeActiveXceiversCount = MetricsAsserts.getIntGauge(
+ "DataNodeActiveXceiversCount", rb);
+ assertEquals(dataNodeActiveXceiversCount, 0);
+
+ Path path = new Path("/counter.txt");
+ DFSTestUtil.createFile(fs, path, 204800000, (short) 3, Time
+ .monotonicNow());
+
+ MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
+ dataNodeActiveXceiversCount = MetricsAsserts.getIntGauge(
+ "DataNodeActiveXceiversCount", rbNew);
+ assertTrue(dataNodeActiveXceiversCount >= 0);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HDFS-10571.
TestDiskBalancerCommand#testPlanNode failed with IllegalArgumentException.
Contributed by Xiaobing Zhou.
Posted by ae...@apache.org.
HDFS-10571. TestDiskBalancerCommand#testPlanNode failed with IllegalArgumentException. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67fec0c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67fec0c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67fec0c4
Branch: refs/heads/HDFS-1312
Commit: 67fec0c4363a480eaaabcb1b5fbe0cbbabc19a86
Parents: ea8bfc3
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Jun 23 14:48:40 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Jun 23 14:48:40 2016 -0700
----------------------------------------------------------------------
.../command/TestDiskBalancerCommand.java | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67fec0c4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index b0821e2..e55c418 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -280,7 +280,7 @@ public class TestDiskBalancerCommand {
final String cmdLine = String
.format(
"hdfs diskbalancer %s", planArg);
- runCommand(cmdLine);
+ runCommand(cmdLine, cluster);
}
/* Test that illegal arguments are handled correctly*/
@@ -335,12 +335,12 @@ public class TestDiskBalancerCommand {
runCommand(cmdLine);
}
- private List<String> runCommand(final String cmdLine) throws Exception {
+ private List<String> runCommandInternal(final String cmdLine) throws
+ Exception {
String[] cmds = StringUtils.split(cmdLine, ' ');
org.apache.hadoop.hdfs.tools.DiskBalancer db =
new org.apache.hadoop.hdfs.tools.DiskBalancer(conf);
- FileSystem.setDefaultUri(conf, clusterJson);
ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bufOut);
db.run(cmds, out);
@@ -353,6 +353,17 @@ public class TestDiskBalancerCommand {
return outputs;
}
+ private List<String> runCommand(final String cmdLine) throws Exception {
+ FileSystem.setDefaultUri(conf, clusterJson);
+ return runCommandInternal(cmdLine);
+ }
+
+ private List<String> runCommand(final String cmdLine,
+ MiniDFSCluster miniCluster) throws Exception {
+ FileSystem.setDefaultUri(conf, miniCluster.getURI());
+ return runCommandInternal(cmdLine);
+ }
+
/**
* Making sure that we can query the node without having done a submit.
* @throws Exception
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50907231
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50907231
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50907231
Branch: refs/heads/HDFS-1312
Commit: 509072312eb924ee1603a8ffb32b0573ffd35e0d
Parents: 67fec0c dca298d
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Jun 23 14:48:47 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Jun 23 14:48:47 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/DataNode.java | 5 +++-
.../hdfs/server/datanode/DataXceiverServer.java | 4 +++
.../datanode/metrics/DataNodeMetrics.java | 16 ++++++++++
.../server/datanode/TestDataNodeMetrics.java | 31 ++++++++++++++++++++
4 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50907231/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org