You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/04/17 20:12:04 UTC
hadoop git commit: HDFS-11657. Ozone: Add unit test for storage
container metrics. Contributed by Yiqun Lin.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 1a418ef2c -> 49c92ea73
HDFS-11657. Ozone: Add unit test for storage container metrics. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49c92ea7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49c92ea7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49c92ea7
Branch: refs/heads/HDFS-7240
Commit: 49c92ea73d9ac0ec45cf5ac682071cf8aa36bf36
Parents: 1a418ef
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Apr 17 12:44:29 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Apr 17 12:44:29 2017 -0700
----------------------------------------------------------------------
.../common/helpers/ContainerMetrics.java | 29 ++---
.../container/metrics/TestContainerMetrics.java | 122 +++++++++++++++++++
2 files changed, 137 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c92ea7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
index 55c0fbc..23ba00b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
@@ -62,23 +62,24 @@ public class ContainerMetrics {
this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
this.registry = new MetricsRegistry("StorageContainerMetrics");
for (int i = 0; i < numEnumEntries; i++) {
- numOpsArray[i] =
- registry.newCounter("num" + ContainerProtos.Type.valueOf(i),
- "number of " + ContainerProtos.Type.valueOf(i) + " ops", (long)0);
- opsBytesArray[i] =
- registry.newCounter("bytes" + ContainerProtos.Type.valueOf(i),
- "bytes used by " + ContainerProtos.Type.valueOf(i)+"op", (long)0);
- opsLatency[i] =
- registry.newRate("latency" + ContainerProtos.Type.valueOf(i),
- ContainerProtos.Type.valueOf(i) + " op");
+ numOpsArray[i] = registry.newCounter(
+ "num" + ContainerProtos.Type.valueOf(i + 1),
+ "number of " + ContainerProtos.Type.valueOf(i + 1) + " ops",
+ (long) 0);
+ opsBytesArray[i] = registry.newCounter(
+ "bytes" + ContainerProtos.Type.valueOf(i + 1),
+ "bytes used by " + ContainerProtos.Type.valueOf(i + 1) + "op",
+ (long) 0);
+ opsLatency[i] = registry.newRate(
+ "latency" + ContainerProtos.Type.valueOf(i + 1),
+ ContainerProtos.Type.valueOf(i + 1) + " op");
for (int j = 0; j < len; j++) {
int interval = intervals[j];
- String quantileName = ContainerProtos.Type.valueOf(i) + "Nanos" +
- interval + "s";
- opsLatQuantiles[i][j] =
- registry.newQuantiles(quantileName, "latency of Container ops",
- "ops", "latency", interval);
+ String quantileName = ContainerProtos.Type.valueOf(i + 1) + "Nanos"
+ + interval + "s";
+ opsLatQuantiles[i][j] = registry.newQuantiles(quantileName,
+ "latency of Container ops", "ops", "latency", interval);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c92ea7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
new file mode 100644
index 0000000..df99577
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.impl;
+
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
+import org.apache.hadoop.ozone.container.common.transport.server.XceiverServer;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.scm.XceiverClient;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test for metrics published by storage containers.
+ */
+public class TestContainerMetrics {
+
+ @Test
+ public void testContainerMetrics() throws Exception {
+ XceiverServer server = null;
+ XceiverClient client = null;
+ String containerName = OzoneUtils.getRequestID();
+ String keyName = OzoneUtils.getRequestID();
+
+ try {
+ final int interval = 1;
+ Pipeline pipeline = ContainerTestHelper
+ .createSingleNodePipeline(containerName);
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ pipeline.getLeader().getContainerPort());
+ conf.setInt(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
+ interval);
+
+ // Since we are only testing for container metrics and we can just
+ // mock the ContainerManager and ChunkManager instances instead of
+ // starting the whole cluster.
+ ContainerManager containerManager = mock(ContainerManager.class);
+ ChunkManager chunkManager = mock(ChunkManager.class);
+ Mockito.doNothing().when(chunkManager).writeChunk(
+ Mockito.any(Pipeline.class), Mockito.anyString(),
+ Mockito.any(ChunkInfo.class), Mockito.any(byte[].class));
+
+ Mockito.doReturn(chunkManager).when(containerManager).getChunkManager();
+ Mockito.doReturn(true).when(containerManager).isOpen(containerName);
+
+ Dispatcher dispatcher = new Dispatcher(containerManager, conf);
+ dispatcher.init();
+ server = new XceiverServer(conf, dispatcher);
+ client = new XceiverClient(pipeline, conf);
+
+ server.start();
+ client.connect();
+
+ // Create container
+ ContainerCommandRequestProto request = ContainerTestHelper
+ .getCreateContainerRequest(containerName);
+ ContainerCommandResponseProto response = client.sendCommand(request);
+ Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+ Assert.assertEquals(ContainerProtos.Result.SUCCESS,
+ response.getResult());
+
+ // Write Chunk
+ ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
+ ContainerTestHelper.getWriteChunkRequest(
+ pipeline, containerName, keyName, 1024);
+ response = client.sendCommand(writeChunkRequest);
+ Assert.assertEquals(ContainerProtos.Result.SUCCESS,
+ response.getResult());
+
+ MetricsRecordBuilder containerMetrics = getMetrics(
+ "StorageContainerMetrics");
+ assertCounter("NumOps", 2L, containerMetrics);
+ assertCounter("numCreateContainer", 1L, containerMetrics);
+ assertCounter("numWriteChunk", 1L, containerMetrics);
+ assertCounter("bytesWriteChunk", 1024L, containerMetrics);
+ assertCounter("LatencyWriteChunkNumOps", 1L, containerMetrics);
+
+ String sec = interval + "s";
+ Thread.sleep((interval + 1) * 1000);
+ assertQuantileGauges("WriteChunkNanos" + sec, containerMetrics);
+ } finally {
+ if (client != null) {
+ client.close();
+ }
+ if (server != null) {
+ server.stop();
+ }
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org